Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: HA-compatible webhook delivery for pipeline events (#18708)

Danny Xu6f7441d9a0df
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 9880d9fa9f..809b82b919 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
+ "log/slog"
"slices"
"strings"
"time"
@@ -15,6 +16,7 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/dbfactory"
"github.com/bytebase/bytebase/backend/component/iam"
"github.com/bytebase/bytebase/backend/component/state"
@@ -654,6 +656,12 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))
}
+ // Reset notification state so user gets fresh feedback on retry
+ if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {
+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))
+ // Don't fail the request - notification is non-critical
+ }
+
issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{
ProjectID: &projectID,
PlanUID: &planID,
@@ -802,7 +810,6 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques
return nil, connect.NewError(connect.CodeInternal, errors.New("user not found"))
}
var taskUIDs []int
- var tasksToSkip []*store.TaskMessage
environmentSet := map[string]struct{}{}
for _, task := range request.Tasks {
_, _, _, taskID, err := common.GetProjectIDPlanIDStageIDTaskID(task)
@@ -814,7 +821,6 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques
return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("task %v not found in the rollout", taskID))
}
taskUIDs = append(taskUIDs, taskID)
- tasksToSkip = append(tasksToSkip, taskMsg)
environmentSet[taskMsg.Environment] = struct{}{}
}
@@ -832,9 +838,8 @@ func (s *RolloutService) BatchSkipTasks(ctx context.Context, req *connect.Reques
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to skip tasks"))
}
- for _, task := range tasksToSkip {
- s.stateCfg.TaskSkippedOrDoneChan <- task.ID
- }
+ // Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)
+ s.stateCfg.PlanCompletionCheckChan <- planID
return connect.NewResponse(&v1pb.BatchSkipTasksResponse{}), nil
}
@@ -942,6 +947,8 @@ func (s *RolloutService) BatchCancelTaskRuns(ctx context.Context, req *connect.R
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to batch patch task run status to canceled"))
}
+ // Note: Don't signal plan completion - canceling is interrupting execution, not completing the plan
+
return connect.NewResponse(&v1pb.BatchCancelTaskRunsResponse{}), nil
}
diff --git a/backend/component/state/state.go b/backend/component/state/state.go
index 931fe1b573..a01174b5aa 100644
--- a/backend/component/state/state.go
+++ b/backend/component/state/state.go
@@ -20,9 +20,6 @@ type State struct {
// RunningPlanCheckRunsCancelFunc is the cancelFunc of running plan checks.
RunningPlanCheckRunsCancelFunc sync.Map // map[planCheckRunUID]context.CancelFunc
- // TaskSkippedOrDoneChan is the channel for notifying the task is skipped or done.
- TaskSkippedOrDoneChan chan int
-
// PlanCheckTickleChan is the tickler for plan check scheduler.
PlanCheckTickleChan chan int
// TaskRunTickleChan is the tickler for task run scheduler.
@@ -30,13 +27,16 @@ type State struct {
// RolloutCreationChan is the channel for automatic rollout creation.
RolloutCreationChan chan int64
+
+ // PlanCompletionCheckChan signals when a plan might be complete (for PIPELINE_COMPLETED webhook).
+ PlanCompletionCheckChan chan int64
}
func New() (*State, error) {
return &State{
- TaskSkippedOrDoneChan: make(chan int, 1000),
- PlanCheckTickleChan: make(chan int, 1000),
- TaskRunTickleChan: make(chan int, 1000),
- RolloutCreationChan: make(chan int64, 100),
+ PlanCheckTickleChan: make(chan int, 1000),
+ TaskRunTickleChan: make(chan int, 1000),
+ RolloutCreationChan: make(chan int64, 100),
+ PlanCompletionCheckChan: make(chan int64, 1000),
}, nil
}
diff --git a/backend/component/webhook/event.go b/backend/component/webhook/event.go
index 2967a31c2a..5cc06fc321 100644
--- a/backend/component/webhook/event.go
+++ b/backend/component/webhook/event.go
@@ -1,8 +1,6 @@
package webhook
import (
- "time"
-
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
)
@@ -87,17 +85,7 @@ type EventIssueSentBack struct {
}
type EventRolloutFailed struct {
- Rollout *Rollout
- FailedTasks []FailedTask
-}
-
-type FailedTask struct {
- TaskID int64
- TaskName string
- DatabaseName string
- InstanceName string
- ErrorMessage string
- FailedAt time.Time
+ Rollout *Rollout
}
type EventRolloutCompleted struct {
diff --git a/backend/component/webhook/manager.go b/backend/component/webhook/manager.go
index e206f2f438..2dd03fa4f8 100644
--- a/backend/component/webhook/manager.go
+++ b/backend/component/webhook/manager.go
@@ -133,18 +133,7 @@ func (m *Manager) getWebhookContextFromEvent(ctx context.Context, e *Event, even
if e.RolloutFailed != nil {
rollout = e.RolloutFailed.Rollout
link = fmt.Sprintf("%s/projects/%s/plans/%d/rollout", externalURL, e.Project.ResourceID, rollout.UID)
- failedTasks := make([]webhook.FailedTaskInfo, 0, len(e.RolloutFailed.FailedTasks))
- for _, task := range e.RolloutFailed.FailedTasks {
- failedTasks = append(failedTasks, webhook.FailedTaskInfo{
- Name: task.TaskName,
- Instance: task.InstanceName,
- Database: task.DatabaseName,
- ErrorMessage: task.ErrorMessage,
- FailedAt: task.FailedAt.Format(time.RFC3339),
- })
- }
- webhookCtx.FailedTasks = failedTasks
- webhookCtx.Description = fmt.Sprintf("%d task(s) failed", len(failedTasks))
+ webhookCtx.Description = "Rollout failed"
}
case storepb.Activity_PIPELINE_COMPLETED:
diff --git a/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql b/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql
new file mode 100644
index 0000000000..94b7a5cc85
--- /dev/null
+++ b/backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql
@@ -0,0 +1,9 @@
+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).
+-- One row per plan at any time - mutually exclusive events.
+-- Row is deleted when user clicks BatchRunTasks to reset notification state.
+CREATE TABLE plan_webhook_delivery (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sql
index 905d6a99ba..fe59f513e4 100644
--- a/backend/migrator/migration/LATEST.sql
+++ b/backend/migrator/migration/LATEST.sql
@@ -229,6 +229,16 @@ CREATE INDEX idx_plan_check_run_active_status ON plan_check_run(status, id) WHER
ALTER SEQUENCE plan_check_run_id_seq RESTART WITH 101;
+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).
+-- One row per plan at any time - mutually exclusive events.
+-- Row is deleted when user clicks BatchRunTasks to reset notification state.
+CREATE TABLE plan_webhook_delivery (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
-- task table stores the task for a plan
CREATE TABLE task (
id serial PRIMARY KEY,
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index 4c70b914bb..d4e7e45ca1 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -12,8 +12,8 @@ import (
func TestLatestVersion(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
- require.Equal(t, semver.MustParse("3.14.15"), *files[len(files)-1].version)
- require.Equal(t, "migration/3.14/0015##task_run_available_status.sql", files[len(files)-1].path)
+ require.Equal(t, semver.MustParse("3.14.16"), *files[len(files)-1].version)
+ require.Equal(t, "migration/3.14/0016##plan_webhook_delivery.sql", files[len(files)-1].path)
}
func TestVersionUnique(t *testing.T) {
diff --git a/backend/runner/taskrun/pipeline_events.go b/backend/runner/taskrun/pipeline_events.go
deleted file mode 100644
index 25e4b697dd..0000000000
--- a/backend/runner/taskrun/pipeline_events.go
+++ /dev/null
@@ -1,78 +0,0 @@
-package taskrun
-
-import (
- "sync"
- "time"
-
- "github.com/bytebase/bytebase/backend/component/webhook"
-)
-
-// PipelineFailureWindow tracks failed tasks for aggregation.
-type PipelineFailureWindow struct {
- firstFailureTime time.Time
- failedTasks []webhook.FailedTask
- notificationSent bool
- timer *time.Timer
-}
-
-// PipelineEventsTracker manages failure aggregation windows per plan.
-type PipelineEventsTracker struct {
- mu sync.RWMutex
- windows map[int64]*PipelineFailureWindow // planID -> window
-}
-
-// NewPipelineEventsTracker creates a new pipeline events tracker.
-func NewPipelineEventsTracker() *PipelineEventsTracker {
- return &PipelineEventsTracker{
- windows: make(map[int64]*PipelineFailureWindow),
- }
-}
-
-// RecordTaskFailure adds a failed task to the aggregation window.
-func (t *PipelineEventsTracker) RecordTaskFailure(planID int64, task webhook.FailedTask, onAggregated func([]webhook.FailedTask)) {
- t.mu.Lock()
- defer t.mu.Unlock()
-
- window, exists := t.windows[planID]
- if !exists || window.notificationSent {
- // Start new window
- window = &PipelineFailureWindow{
- firstFailureTime: time.Now(),
- failedTasks: []webhook.FailedTask{task},
- notificationSent: false,
- }
- t.windows[planID] = window
-
- // Set 5-minute timer
- window.timer = time.AfterFunc(5*time.Minute, func() {
- t.mu.Lock()
- defer t.mu.Unlock()
-
- if w, ok := t.windows[planID]; ok && !w.notificationSent {
- w.notificationSent = true
- // Copy tasks to avoid holding lock during callback
- tasksCopy := make([]webhook.FailedTask, len(w.failedTasks))
- copy(tasksCopy, w.failedTasks)
- // Delete window after sending to prevent memory leak
- delete(t.windows, planID)
- onAggregated(tasksCopy)
- }
- })
- } else {
- // Add to existing window
- window.failedTasks = append(window.failedTasks, task)
- }
-}
-
-// Clear removes the window for a plan (call after pipeline completes).
-func (t *PipelineEventsTracker) Clear(planID int64) {
- t.mu.Lock()
- defer t.mu.Unlock()
-
- if window, exists := t.windows[planID]; exists {
- if window.timer != nil {
- window.timer.Stop()
- }
- delete(t.windows, planID)
- }
-}
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index 3f5d0de37e..d4b8a8271c 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -199,7 +199,32 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *st
return
}
s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_FAILED, taskRunResult.Detail)
- s.recordPipelineFailure(ctx, task, taskRunResult.Detail)
+
+ // Immediately try to send PIPELINE_FAILED webhook (HA-safe atomic claim)
+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)
+ if err != nil {
+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))
+ } else if claimed {
+ // Get plan and project for webhook
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
+ if err != nil || plan == nil {
+ slog.Error("failed to get plan for failure webhook", log.BBError(err))
+ } else {
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil || project == nil {
+ slog.Error("failed to get project for failure webhook", log.BBError(err))
+ } else {
+ // Send PIPELINE_FAILED webhook
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Type: storepb.Activity_PIPELINE_FAILED,
+ Project: webhook.NewProject(project),
+ RolloutFailed: &webhook.EventRolloutFailed{
+ Rollout: webhook.NewRollout(plan),
+ },
+ })
+ }
+ }
+ }
return
case done && err == nil:
@@ -229,7 +254,9 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int, task *st
return
}
s.createActivityForTaskRunStatusUpdate(ctx, task, storepb.TaskRun_DONE, "")
- s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+
+ // Signal to check if plan is complete and successful (may send PIPELINE_COMPLETED)
+ s.stateCfg.PlanCompletionCheckChan <- task.PlanID
return
default:
// This case should not happen in normal flow, but adding for completeness
@@ -247,63 +274,6 @@ func (*Scheduler) createActivityForTaskRunStatusUpdate(_ context.Context, _ *sto
// No webhook events for task run status updates
}
-func (s *Scheduler) recordPipelineFailure(ctx context.Context, task *store.TaskMessage, errDetail string) {
- if err := func() error {
- plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
- if err != nil {
- return errors.Wrapf(err, "failed to get plan")
- }
- if plan == nil {
- return errors.Errorf("plan %v not found", task.PlanID)
- }
-
- project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
- if err != nil {
- return errors.Wrapf(err, "failed to get project")
- }
- if project == nil {
- return errors.Errorf("project %v not found", plan.ProjectID)
- }
-
- instance, err := s.store.GetInstance(ctx, &store.FindInstanceMessage{ResourceID: &task.InstanceID})
- if err != nil {
- return errors.Wrapf(err, "failed to get instance")
- }
-
- failedTask := webhook.FailedTask{
- TaskID: int64(task.ID),
- TaskName: task.Type.String(),
- DatabaseName: task.GetDatabaseName(),
- InstanceName: instance.Metadata.Title,
- ErrorMessage: errDetail,
- FailedAt: time.Now(),
- }
-
- s.pipelineEvents.RecordTaskFailure(
- plan.UID,
- failedTask,
- func(failedTasks []webhook.FailedTask) {
- // Use background context to avoid cancellation issues
- webhookCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
-
- s.webhookManager.CreateEvent(webhookCtx, &webhook.Event{
- Type: storepb.Activity_PIPELINE_FAILED,
- Project: webhook.NewProject(project),
- RolloutFailed: &webhook.EventRolloutFailed{
- Rollout: webhook.NewRollout(plan),
- FailedTasks: failedTasks,
- },
- })
- },
- )
-
- return nil
- }(); err != nil {
- slog.Error("failed to record pipeline failure", log.BBError(err))
- }
-}
-
// isSequentialTask returns whether the task should be executed sequentially.
func isSequentialTask(task *store.TaskMessage) bool {
//exhaustive:enforce
diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.go
index 90d4049f1a..d184262f6d 100644
--- a/backend/runner/taskrun/scheduler.go
+++ b/backend/runner/taskrun/scheduler.go
@@ -27,7 +27,6 @@ type Scheduler struct {
webhookManager *webhook.Manager
executorMap map[storepb.Task_Type]Executor
profile *config.Profile
- pipelineEvents *PipelineEventsTracker
}
// NewScheduler will create a new scheduler.
@@ -43,7 +42,6 @@ func NewScheduler(
webhookManager: webhookManager,
profile: profile,
executorMap: map[storepb.Task_Type]Executor{},
- pipelineEvents: NewPipelineEventsTracker(),
}
}
@@ -85,30 +83,21 @@ func (s *Scheduler) runTaskCompletionListener(ctx context.Context) {
slog.Error("runTaskCompletionListener PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
}
}()
- slog.Info("Task completion listener started")
+ slog.Info("Plan completion checker started")
for {
select {
- case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:
- if err := func() error {
- task, err := s.store.GetTaskByID(ctx, taskUID)
- if err != nil {
- return errors.Wrapf(err, "failed to get task")
- }
-
- // Check if entire plan is complete and handle webhooks
- s.checkPlanCompletion(ctx, task.PlanID)
-
- return nil
- }(); err != nil {
- slog.Error("failed to handle task completion", log.BBError(err))
- }
case <-ctx.Done():
return
+ case planID := <-s.stateCfg.PlanCompletionCheckChan:
+ s.checkPlanCompletion(ctx, planID)
}
}
}
+// checkPlanCompletion checks if all tasks in a plan are complete and successful.
+// If so, sends PIPELINE_COMPLETED webhook.
+// Called when tasks are marked DONE/SKIPPED, or when tasks are skipped/canceled via API.
func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {
// Get all tasks for this plan
tasks, err := s.store.ListTasks(ctx, &store.TaskFind{PlanID: &planID})
@@ -117,87 +106,44 @@ func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {
return
}
- // Check if all tasks are in terminal state
- allComplete := true
- hasFailures := false
- var earliestStart, latestEnd *time.Time
-
+ // Check if all tasks are complete (DONE or SKIPPED)
for _, task := range tasks {
status := task.LatestTaskRunStatus
- // Check if task is in terminal state
- isTerminal := status == storepb.TaskRun_DONE ||
- status == storepb.TaskRun_FAILED ||
- status == storepb.TaskRun_CANCELED ||
+ // Only DONE and SKIPPED are considered complete
+ // FAILED and CANCELED are not complete states
+ isComplete := status == storepb.TaskRun_DONE ||
status == storepb.TaskRun_SKIPPED ||
task.Payload.GetSkipped()
- if !isTerminal {
- allComplete = false
- break
- }
-
- // Track failures (excluding skipped tasks)
- if status == storepb.TaskRun_FAILED && !task.Payload.GetSkipped() {
- hasFailures = true
- }
-
- // Track start/end times for metrics
- if task.RunAt != nil {
- if earliestStart == nil || task.RunAt.Before(*earliestStart) {
- earliestStart = task.RunAt
- }
- }
- if task.UpdatedAt != nil {
- if latestEnd == nil || task.UpdatedAt.After(*latestEnd) {
- latestEnd = task.UpdatedAt
- }
+ if !isComplete {
+ // Not all tasks complete - no webhook
+ return
}
}
- // Not all tasks complete yet
- if !allComplete {
+ // All tasks complete and successful - try to claim completion notification
+ claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)
+ if err != nil {
+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))
return
}
-
- // Always clear the failure window when plan completes
- s.pipelineEvents.Clear(planID)
-
- // Only send completion webhook if there were no failures
- if hasFailures {
- return
+ if !claimed {
+ return // Already sent
}
- // Get plan, project and issue for webhook
+ // Get plan and project for webhook
plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &planID})
- if err != nil {
+ if err != nil || plan == nil {
slog.Error("failed to get plan for completion webhook", log.BBError(err))
return
}
- if plan == nil {
- slog.Error("plan not found for completion webhook", slog.Int64("plan_id", planID))
- return
- }
project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
- if err != nil {
+ if err != nil || project == nil {
slog.Error("failed to get project for completion webhook", log.BBError(err))
return
}
- if project == nil {
- slog.Error("project not found for completion webhook", slog.String("project_id", plan.ProjectID))
- return
- }
-
- issueN, err := s.store.GetIssue(ctx, &store.FindIssueMessage{PlanUID: &planID})
- if err != nil {
- slog.Error("failed to get issue for completion webhook", log.BBError(err))
- return
- }
- if issueN == nil {
- slog.Error("issue not found for completion webhook", slog.Int64("plan_id", planID))
- return
- }
// Send PIPELINE_COMPLETED webhook
s.webhookManager.CreateEvent(ctx, &webhook.Event{
diff --git a/backend/store/plan_webhook_delivery.go b/backend/store/plan_webhook_delivery.go
new file mode 100644
index 0000000000..30836e531b
--- /dev/null
+++ b/backend/store/plan_webhook_delivery.go
@@ -0,0 +1,52 @@
+package store
+
+import (
+ "context"
+ "database/sql"
+)
+
+// ResetPlanWebhookDelivery deletes the webhook delivery record for a plan.
+// Called when user clicks BatchRunTasks to enable new notifications on retry.
+func (s *Store) ResetPlanWebhookDelivery(ctx context.Context, planID int64) error {
+ query := `DELETE FROM plan_webhook_delivery WHERE plan_id = $1`
+ _, err := s.GetDB().ExecContext(ctx, query, planID)
+ return err
+}
+
+// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO plan_webhook_delivery (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_FAILED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.GetDB().QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
+
+// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO plan_webhook_delivery (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_COMPLETED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.GetDB().QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
diff --git a/docs/plans/2025-12-31-ha-webhook-delivery-design.md b/docs/plans/2025-12-31-ha-webhook-delivery-design.md
new file mode 100644
index 0000000000..f672c35f11
--- /dev/null
+++ b/docs/plans/2025-12-31-ha-webhook-delivery-design.md
@@ -0,0 +1,395 @@
+# HA-Compatible Webhook Delivery Design
+
+**Date:** 2025-12-31
+**Status:** Design
+**Author:** Danny
+
+## Overview
+
+This design makes pipeline webhook notifications (PIPELINE_FAILED and PIPELINE_COMPLETED) work correctly in High Availability (HA) deployments where multiple Bytebase replicas run simultaneously. It replaces the in-memory `TaskSkippedOrDoneChan` channel with a database-backed idempotent delivery system.
+
+## Problem
+
+The current webhook notification system uses an in-memory channel (`state.TaskSkippedOrDoneChan`) which breaks in HA:
+
+```go
+// backend/component/state/state.go
+type State struct {
+ TaskSkippedOrDoneChan chan int // In-memory, not shared across replicas
+}
+
+// backend/runner/taskrun/running_scheduler.go
+s.stateCfg.TaskSkippedOrDoneChan <- task.ID // Send notification
+
+// backend/runner/taskrun/scheduler.go
+case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan: // Receive notification
+ s.checkPlanCompletion(ctx, task.PlanID)
+```
+
+**HA Issues:**
+- Replica A marks task as done, sends to its local channel
+- Replica B receives user request, doesn't see anything in its channel
+- Both replicas might process same task failure → duplicate webhooks
+- Or neither replica sends webhook → missed notification
+
+## Requirements
+
+1. **No spam**: Send PIPELINE_FAILED only once when first task fails
+2. **Retry support**: Send new PIPELINE_FAILED when user clicks "Retry Tasks"
+3. **Completion**: Send PIPELINE_COMPLETED once when all tasks succeed
+4. **HA-safe**: Multiple replicas shouldn't send duplicate webhooks
+5. **Simple**: Avoid complex aggregation windows or polling
+
+## Solution: Database-Backed Delivery Log
+
+Use a single database table with PRIMARY KEY constraint to ensure exactly-once delivery across HA replicas.
+
+### Key Insight
+
+- `PIPELINE_FAILED` and `PIPELINE_COMPLETED` are **mutually exclusive** per plan
+- User clicking "Retry Tasks" is an explicit action that resets notification state
+- Database PRIMARY KEY constraint provides atomic claim mechanism
+
+### Schema
+
+```sql
+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).
+-- One row per plan at any time - mutually exclusive events.
+-- Row is deleted when user clicks BatchRunTasks to reset notification state.
+CREATE TABLE webhook_delivery_log (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+```
+
+**Design choices:**
+- `plan_id` as PRIMARY KEY enforces uniqueness automatically
+- Keep `event_type` for audit trail (debugging which event was sent)
+- No CHECK constraint - application code controls values, comments document them
+- No separate indexes needed - PRIMARY KEY provides all we need
+
+## Architecture
+
+### 1. When Task Run Fails
+
+Location: `backend/runner/taskrun/running_scheduler.go`
+
+```go
+// Add after task run status is marked as FAILED
+func (s *Scheduler) handleTaskRunFailure(ctx context.Context, taskRun *store.TaskRunMessage) {
+ task, err := s.store.GetTaskByID(ctx, taskRun.TaskID)
+ if err != nil {
+ slog.Error("failed to get task for failure notification", log.BBError(err))
+ return
+ }
+
+ // Try to claim notification (only one replica succeeds)
+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)
+ if err != nil {
+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))
+ return
+ }
+ if !claimed {
+ // Already sent by this or another replica
+ return
+ }
+
+ // Get all failed tasks for this plan
+ failures, err := s.getFailedTaskRuns(ctx, task.PlanID)
+ if err != nil {
+ slog.Error("failed to get failed task runs", log.BBError(err))
+ return
+ }
+
+ // Get plan/project context
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
+ if err != nil || plan == nil {
+ slog.Error("failed to get plan for failure webhook", log.BBError(err))
+ return
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil || project == nil {
+ slog.Error("failed to get project for failure webhook", log.BBError(err))
+ return
+ }
+
+ // Send webhook
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Type: storepb.Activity_PIPELINE_FAILED,
+ Project: webhook.NewProject(project),
+ PipelineFailed: &webhook.EventPipelineFailed{
+ Rollout: webhook.NewRollout(plan),
+ FailedTasks: failures,
+ },
+ })
+}
+```
+
+### 2. When Pipeline Completes
+
+Location: `backend/runner/taskrun/scheduler.go` (existing `checkPlanCompletion` function)
+
+```go
+func (s *Scheduler) checkPlanCompletion(ctx context.Context, planID int64) {
+ // ... existing task completion check logic ...
+
+ if !allComplete {
+ return
+ }
+
+ if hasFailures {
+ return // Don't send completion if there were failures
+ }
+
+ // Try to claim completion notification (HA-safe)
+ claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)
+ if err != nil {
+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))
+ return
+ }
+ if !claimed {
+ return // Already sent
+ }
+
+ // ... existing webhook send code ...
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Type: storepb.Activity_PIPELINE_COMPLETED,
+ Project: webhook.NewProject(project),
+ RolloutCompleted: &webhook.EventRolloutCompleted{
+ Rollout: webhook.NewRollout(plan),
+ },
+ })
+}
+```
+
+### 3. When User Clicks "Retry Tasks"
+
+Location: `backend/api/v1/rollout_service.go` (existing `BatchRunTasks` function)
+
+```go
+func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request[v1pb.BatchRunTasksRequest]) (*connect.Response[v1pb.BatchRunTasksResponse], error) {
+ // ... existing validation code ...
+
+ // Reset notification state so user gets fresh feedback on retry
+ if err := s.store.ResetPipelineNotification(ctx, planID); err != nil {
+ slog.Error("failed to reset pipeline notification", log.BBError(err))
+ // Don't fail the request - notification is non-critical
+ }
+
+ // ... existing CreatePendingTaskRuns call ...
+
+ return connect.NewResponse(&v1pb.BatchRunTasksResponse{}), nil
+}
+```
+
+## Store Methods
+
+Location: `backend/store/webhook_delivery_log.go` (new file)
+
+```go
+package store
+
+import (
+ "context"
+ "database/sql"
+)
+
+// ResetPipelineNotification deletes the notification log for a plan.
+// Called when user clicks BatchRunTasks to enable new notifications on retry.
+func (s *Store) ResetPipelineNotification(ctx context.Context, planID int64) error {
+ query := `DELETE FROM webhook_delivery_log WHERE plan_id = $1`
+ _, err := s.db.ExecContext(ctx, query, planID)
+ return err
+}
+
+// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO webhook_delivery_log (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_FAILED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
+
+// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO webhook_delivery_log (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_COMPLETED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
+```
+
+## Example Flow
+
+### Scenario 1: Initial Deployment with Failures
+
+```
+Initial deployment (all tasks attempt 0):
+→ Task A fails
+ → Replica 1: ClaimPipelineFailureNotification(plan=123)
+ → INSERT succeeds → Send webhook: "Pipeline failed - Task A failed"
+
+→ Task B fails
+ → Replica 2: ClaimPipelineFailureNotification(plan=123)
+ → ON CONFLICT (row already exists) → Skip, no duplicate webhook
+
+Result: One webhook sent, no spam ✅
+Database: (plan_id=123, event_type='PIPELINE_FAILED')
+```
+
+### Scenario 2: User Retries Failed Tasks
+
+```
+User clicks "Retry Tasks":
+→ BatchRunTasks API called
+ → DELETE FROM webhook_delivery_log WHERE plan_id=123
+ → CreatePendingTaskRuns (creates attempt=1 for failed tasks)
+
+Database: (empty - row deleted)
+
+→ Task A succeeds (attempt=1)
+→ Task B fails (attempt=1)
+ → Replica 1: ClaimPipelineFailureNotification(plan=123)
+ → INSERT succeeds (no conflict, row was deleted)
+ → Send webhook: "Pipeline failed on retry - Task B failed"
+
+Result: User gets feedback on retry ✅
+Database: (plan_id=123, event_type='PIPELINE_FAILED')
+```
+
+### Scenario 3: Successful Completion
+
+```
+User retries again:
+→ BatchRunTasks called
+ → DELETE FROM webhook_delivery_log WHERE plan_id=123
+
+→ All tasks succeed
+ → checkPlanCompletion detects all tasks done, no failures
+ → Replica 2: ClaimPipelineCompletionNotification(plan=123)
+ → INSERT succeeds
+ → Send webhook: "Pipeline completed successfully"
+
+Result: Completion notification sent ✅
+Database: (plan_id=123, event_type='PIPELINE_COMPLETED')
+```
+
+### Scenario 4: HA Race Condition Handling
+
+```
+Two replicas detect same task failure simultaneously:
+→ Replica A: ClaimPipelineFailureNotification(plan=123)
+ → INSERT INTO webhook_delivery_log VALUES (123, 'PIPELINE_FAILED')
+ → Returns plan_id=123 → claimed=true → Sends webhook
+
+→ Replica B: ClaimPipelineFailureNotification(plan=123) (100ms later)
+ → INSERT INTO webhook_delivery_log VALUES (123, 'PIPELINE_FAILED')
+ → ON CONFLICT (plan_id) DO NOTHING
+ → Returns no rows → claimed=false → Skips webhook
+
+Result: Only one webhook sent across HA replicas ✅
+```
+
+## Migration
+
+Create migration file: `backend/migrator/migration/<<version>>/<<sequence>>##webhook_delivery_log.sql`
+
+```sql
+CREATE TABLE webhook_delivery_log (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+```
+
+## Cleanup
+
+After implementation, remove in-memory channel:
+
+1. **Remove from state:**
+ ```go
+ // backend/component/state/state.go
+ // DELETE: TaskSkippedOrDoneChan chan int
+ ```
+
+2. **Remove senders:**
+ ```go
+ // backend/api/v1/rollout_service.go
+ // DELETE: s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+
+ // backend/runner/taskrun/running_scheduler.go
+ // DELETE: s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+ ```
+
+3. **Remove receiver:**
+ ```go
+ // backend/runner/taskrun/scheduler.go
+ // DELETE: case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:
+ ```
+
+## Benefits
+
+✅ **HA-Compatible**: Database PRIMARY KEY prevents duplicate webhooks across replicas
+✅ **No Spam**: One notification per failure/completion phase
+✅ **Retry Support**: DELETE on BatchRunTasks resets state for fresh notifications
+✅ **Immediate Delivery**: No polling or aggregation windows needed
+✅ **Simple Schema**: Single table with PRIMARY KEY, no complex indexes
+✅ **Audit Trail**: `event_type` shows which notification was sent
+✅ **Follows Existing Pattern**: Same approach as TaskRun session monitoring (commit 564f1e5618)
+
+## Comparison to Alternatives
+
+### Alternative 1: 5-Minute Aggregation Windows
+❌ Complex: Needs polling worker, window management
+❌ Delayed: 5-minute wait before notification
+❌ Race-prone: Multiple replicas might process same window
+
+### Alternative 2: PostgreSQL Advisory Locks
+❌ Lock held during webhook HTTP call (can timeout)
+❌ More complex than simple INSERT/DELETE
+
+### Alternative 3: In-Memory Channel (Current)
+❌ Breaks in HA - not shared across replicas
+❌ No persistence - lost on restart
+
+### This Design (Database-Backed Log)
+✅ Simple INSERT/DELETE operations
+✅ Immediate notifications
+✅ HA-safe with database constraints
+✅ Stateless - any replica can handle requests
+
+## Future Enhancements
+
+- Add `retry_count` column to track how many times plan was retried
+- Add `failed_task_count` to show aggregated metrics
+- TTL cleanup for old delivery logs (e.g., delete after 30 days)
+
+## References
+
+- Similar pattern: TaskRun session monitoring HA design (commit 564f1e5618)
+- Related: Webhook events redesign (docs/plans/2025-12-30-webhook-events-redesign.md)
diff --git a/docs/plans/2025-12-31-ha-webhook-delivery-impl.md b/docs/plans/2025-12-31-ha-webhook-delivery-impl.md
new file mode 100644
index 0000000000..3727f9b3bf
--- /dev/null
+++ b/docs/plans/2025-12-31-ha-webhook-delivery-impl.md
@@ -0,0 +1,738 @@
+# HA-Compatible Webhook Delivery Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Implement database-backed webhook delivery to make PIPELINE_FAILED and PIPELINE_COMPLETED notifications work correctly in HA deployments
+
+**Architecture:** Replace in-memory `TaskSkippedOrDoneChan` channel with `plan_webhook_delivery` table using PRIMARY KEY constraint for atomic claiming across replicas
+
+**Tech Stack:** Go, PostgreSQL, existing webhook infrastructure
+
+---
+
+## Phase 1: Database Schema
+
+### Task 1: Create Migration for plan_webhook_delivery Table
+
+**Files:**
+- Create: `backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql`
+
+**Step 1: Create migration file**
+
+Create `backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql`:
+
+```sql
+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).
+-- One row per plan at any time - mutually exclusive events.
+-- Row is deleted when user clicks BatchRunTasks to reset notification state.
+CREATE TABLE plan_webhook_delivery (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+```
+
+**Step 2: Update LATEST.sql**
+
+Add to `backend/migrator/migration/LATEST.sql` after the `plan_check_run` table (around line 231):
+
+```sql
+-- Tracks webhook delivery for pipeline events (PIPELINE_FAILED or PIPELINE_COMPLETED).
+-- One row per plan at any time - mutually exclusive events.
+-- Row is deleted when user clicks BatchRunTasks to reset notification state.
+CREATE TABLE plan_webhook_delivery (
+ plan_id BIGINT PRIMARY KEY REFERENCES plan(id),
+ -- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'
+ event_type TEXT NOT NULL,
+ delivered_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+ALTER SEQUENCE plan_webhook_delivery_plan_id_seq RESTART WITH 101;
+```
+
+**Step 3: Verify migration files**
+
+Run: `ls backend/migrator/migration/3.14/`
+Expected: See `0016##plan_webhook_delivery.sql`
+
+**Step 4: Commit migration**
+
+```bash
+git add backend/migrator/migration/3.14/0016##plan_webhook_delivery.sql backend/migrator/migration/LATEST.sql
+git commit -m "feat: add plan_webhook_delivery table for HA webhook deduplication
+
+Table provides atomic claim mechanism for webhook delivery across HA replicas
+using PRIMARY KEY constraint. Deleted on BatchRunTasks to reset state.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 2: Store Methods
+
+### Task 2: Implement Store Methods for Webhook Delivery
+
+**Files:**
+- Create: `backend/store/plan_webhook_delivery.go`
+
+**Step 1: Create store methods file**
+
+Create `backend/store/plan_webhook_delivery.go`:
+
+```go
+package store
+
+import (
+ "context"
+ "database/sql"
+)
+
+// ResetPlanWebhookDelivery deletes the webhook delivery record for a plan.
+// Called when user clicks BatchRunTasks to enable new notifications on retry.
+func (s *Store) ResetPlanWebhookDelivery(ctx context.Context, planID int64) error {
+ query := `DELETE FROM plan_webhook_delivery WHERE plan_id = $1`
+ _, err := s.db.ExecContext(ctx, query, planID)
+ return err
+}
+
+// ClaimPipelineFailureNotification attempts to claim the right to send PIPELINE_FAILED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineFailureNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO plan_webhook_delivery (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_FAILED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
+
+// ClaimPipelineCompletionNotification attempts to claim the right to send PIPELINE_COMPLETED webhook.
+// Returns true if claimed (should send), false if already sent or claimed by another replica.
+// HA-safe: PRIMARY KEY constraint prevents duplicate sends across replicas.
+func (s *Store) ClaimPipelineCompletionNotification(ctx context.Context, planID int64) (bool, error) {
+ query := `
+ INSERT INTO plan_webhook_delivery (plan_id, event_type)
+ VALUES ($1, 'PIPELINE_COMPLETED')
+ ON CONFLICT (plan_id) DO NOTHING
+ RETURNING plan_id
+ `
+
+ var id int64
+ err := s.db.QueryRowContext(ctx, query, planID).Scan(&id)
+ if err == sql.ErrNoRows {
+ return false, nil // Already exists
+ }
+ return err == nil, err
+}
+```
+
+**Step 2: Build to verify**
+
+Run: `go build ./backend/store/...`
+Expected: Build succeeds
+
+**Step 3: Commit store methods**
+
+```bash
+git add backend/store/plan_webhook_delivery.go
+git commit -m "feat: add store methods for plan webhook delivery
+
+Implement atomic claim methods for PIPELINE_FAILED and PIPELINE_COMPLETED
+webhooks using INSERT...ON CONFLICT for HA safety. Add reset method for
+BatchRunTasks.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 3: Integrate into BatchRunTasks
+
+### Task 3: Reset Webhook State on BatchRunTasks
+
+**Files:**
+- Modify: `backend/api/v1/rollout_service.go:627-752`
+
+**Step 1: Add reset call to BatchRunTasks**
+
+In `backend/api/v1/rollout_service.go`, add after getting the plan (around line 655):
+
+```go
+// Reset notification state so user gets fresh feedback on retry
+if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {
+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))
+ // Don't fail the request - notification is non-critical
+}
+```
+
+Full context - the function should look like:
+
+```go
+func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request[v1pb.BatchRunTasksRequest]) (*connect.Response[v1pb.BatchRunTasksResponse], error) {
+ request := req.Msg
+ // ... existing validation code ...
+
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{
+ ProjectID: &projectID,
+ UID: &planID,
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to find plan for rollout"))
+ }
+ if plan == nil {
+ return nil, connect.NewError(connect.CodeNotFound, errors.Errorf("rollout (plan) %v not found", planID))
+ }
+
+ // Reset notification state so user gets fresh feedback on retry
+ if err := s.store.ResetPlanWebhookDelivery(ctx, planID); err != nil {
+ slog.Error("failed to reset plan webhook delivery", log.BBError(err))
+ // Don't fail the request - notification is non-critical
+ }
+
+ // ... rest of existing code ...
+}
+```
+
+**Step 2: Build to verify**
+
+Run: `go build ./backend/api/v1/...`
+Expected: Build succeeds
+
+**Step 3: Commit BatchRunTasks integration**
+
+```bash
+git add backend/api/v1/rollout_service.go
+git commit -m "feat: reset webhook delivery state on BatchRunTasks
+
+Delete plan_webhook_delivery row when user retries tasks to enable fresh
+notifications on retry attempts.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 4: Send PIPELINE_FAILED on Task Failure
+
+### Task 4: Add Helper to Get Failed Task Runs
+
+**Files:**
+- Modify: `backend/runner/taskrun/scheduler.go`
+
+**Step 1: Add helper method to get failed task runs**
+
+Add near the end of `backend/runner/taskrun/scheduler.go`:
+
+```go
+// getFailedTaskRuns returns all failed task runs for a plan to include in webhook payload.
+func (s *Scheduler) getFailedTaskRuns(ctx context.Context, planID int64) []webhook.FailedTask {
+ tasks, err := s.store.ListTasks(ctx, &store.TaskFind{PlanID: &planID})
+ if err != nil {
+ slog.Error("failed to list tasks for failed webhook", log.BBError(err))
+ return nil
+ }
+
+ var failures []webhook.FailedTask
+ for _, task := range tasks {
+ if task.LatestTaskRunStatus != storepb.TaskRun_FAILED {
+ continue
+ }
+
+ // Get the latest failed task run
+ taskRuns, err := s.store.ListTaskRuns(ctx, &store.TaskRunFind{TaskID: &task.ID})
+ if err != nil {
+ slog.Error("failed to list task runs", log.BBError(err))
+ continue
+ }
+
+ // Find the latest failed run
+ var latestFailed *store.TaskRunMessage
+ for _, tr := range taskRuns {
+ if tr.Status == storepb.TaskRun_FAILED {
+ if latestFailed == nil || tr.UpdatedAt.After(*latestFailed.UpdatedAt) {
+ latestFailed = tr
+ }
+ }
+ }
+
+ if latestFailed == nil {
+ continue
+ }
+
+ errorMsg := ""
+ if latestFailed.Result != nil && latestFailed.Result.Error != "" {
+ errorMsg = latestFailed.Result.Error
+ }
+
+ failures = append(failures, webhook.FailedTask{
+ TaskID: int64(task.ID),
+ TaskName: task.Name,
+ DatabaseName: task.DatabaseName,
+ InstanceName: task.Instance.ResourceID,
+ ErrorMessage: errorMsg,
+ FailedAt: *latestFailed.UpdatedAt,
+ })
+ }
+
+ return failures
+}
+```
+
+**Step 2: Build to verify**
+
+Run: `go build ./backend/runner/taskrun/...`
+Expected: Build succeeds
+
+**Step 3: Commit helper method**
+
+```bash
+git add backend/runner/taskrun/scheduler.go
+git commit -m "feat: add helper to get failed task runs for webhook
+
+Collects all failed tasks in a plan with error messages for webhook payload.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+### Task 5: Send PIPELINE_FAILED Webhook on Task Failure
+
+**Files:**
+- Modify: `backend/runner/taskrun/running_scheduler.go`
+
+**Step 1: Check where task is marked as FAILED**
+
+Read `backend/runner/taskrun/running_scheduler.go` around line 200-240 to find where task status is set to FAILED.
+
+Expected: Find code like:
+```go
+if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ Status: storepb.TaskRun_FAILED,
+ ...
+}); err != nil {
+ ...
+}
+```
+
+**Step 2: Add webhook notification after marking task as FAILED**
+
+After the task is marked as FAILED and before the return statement, add:
+
+```go
+// Send PIPELINE_FAILED webhook (HA-safe atomic claim)
+s.sendPipelineFailureWebhook(ctx, task)
+```
+
+**Step 3: Implement sendPipelineFailureWebhook method**
+
+Add to `backend/runner/taskrun/running_scheduler.go`:
+
+```go
+// sendPipelineFailureWebhook attempts to send PIPELINE_FAILED webhook.
+// Uses atomic claim to prevent duplicate sends in HA deployments.
+func (s *Scheduler) sendPipelineFailureWebhook(ctx context.Context, task *store.TaskMessage) {
+ // Try to claim notification (only one replica succeeds)
+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, task.PlanID)
+ if err != nil {
+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))
+ return
+ }
+ if !claimed {
+ // Already sent by this or another replica
+ return
+ }
+
+ // Get plan context
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{UID: &task.PlanID})
+ if err != nil || plan == nil {
+ slog.Error("failed to get plan for failure webhook", log.BBError(err))
+ return
+ }
+
+ project, err := s.store.GetProject(ctx, &store.FindProjectMessage{ResourceID: &plan.ProjectID})
+ if err != nil || project == nil {
+ slog.Error("failed to get project for failure webhook", log.BBError(err))
+ return
+ }
+
+ // Get all failed tasks
+ failures := s.getFailedTaskRuns(ctx, task.PlanID)
+ if len(failures) == 0 {
+ slog.Warn("no failed tasks found for pipeline failure webhook", slog.Int64("plan_id", task.PlanID))
+ return
+ }
+
+ // Send webhook
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Type: storepb.Activity_PIPELINE_FAILED,
+ Project: webhook.NewProject(project),
+ PipelineFailed: &webhook.EventPipelineFailed{
+ Rollout: webhook.NewRollout(plan),
+ FailedTasks: failures,
+ },
+ })
+}
+```
+
+**Step 4: Build to verify**
+
+Run: `go build ./backend/runner/taskrun/...`
+Expected: Build succeeds
+
+**Step 5: Commit pipeline failure webhook**
+
+```bash
+git add backend/runner/taskrun/running_scheduler.go
+git commit -m "feat: send PIPELINE_FAILED webhook on task failure
+
+Add atomic claim-based webhook delivery when task fails. Only first failure
+in a plan triggers notification, preventing spam in HA deployments.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 5: Send PIPELINE_COMPLETED on Success
+
+### Task 6: Update checkPlanCompletion to Send Webhook
+
+**Files:**
+- Modify: `backend/runner/taskrun/scheduler.go:112-210`
+
+**Step 1: Add claim check before sending completion webhook**
+
+In `backend/runner/taskrun/scheduler.go`, find the `checkPlanCompletion` function (around line 112).
+
+After the `if hasFailures { return }` check (around line 167), add:
+
+```go
+// Try to claim completion notification (HA-safe)
+claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)
+if err != nil {
+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))
+ return
+}
+if !claimed {
+ return // Already sent
+}
+```
+
+Full context - the section should look like:
+
+```go
+// Not all tasks complete yet
+if !allComplete {
+ return
+}
+
+// Always clear the failure window when plan completes
+s.pipelineEvents.Clear(planID)
+
+// Only send completion webhook if there were no failures
+if hasFailures {
+ return
+}
+
+// Try to claim completion notification (HA-safe)
+claimed, err := s.store.ClaimPipelineCompletionNotification(ctx, planID)
+if err != nil {
+ slog.Error("failed to claim pipeline completion notification", log.BBError(err))
+ return
+}
+if !claimed {
+ return // Already sent
+}
+
+// ... existing webhook send code continues ...
+```
+
+**Step 2: Build to verify**
+
+Run: `go build ./backend/runner/taskrun/...`
+Expected: Build succeeds
+
+**Step 3: Commit pipeline completion webhook**
+
+```bash
+git add backend/runner/taskrun/scheduler.go
+git commit -m "feat: add atomic claim for PIPELINE_COMPLETED webhook
+
+Prevent duplicate completion webhooks in HA deployments using database-backed
+claim mechanism.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 6: Cleanup In-Memory Channel
+
+### Task 7: Remove TaskSkippedOrDoneChan Channel
+
+**Files:**
+- Modify: `backend/component/state/state.go`
+- Modify: `backend/api/v1/rollout_service.go`
+- Modify: `backend/runner/taskrun/running_scheduler.go`
+- Modify: `backend/runner/taskrun/scheduler.go`
+
+**Step 1: Remove channel from state**
+
+In `backend/component/state/state.go`, remove these lines (around line 23-24):
+
+```go
+// DELETE these lines:
+// TaskSkippedOrDoneChan is the channel for notifying the task is skipped or done.
+TaskSkippedOrDoneChan chan int
+```
+
+And in the `New()` function (around line 37):
+
+```go
+// DELETE this line:
+TaskSkippedOrDoneChan: make(chan int, 1000),
+```
+
+**Step 2: Remove sender in rollout_service.go**
+
+In `backend/api/v1/rollout_service.go`, find and remove (around line 836):
+
+```go
+// DELETE these lines:
+for _, task := range tasksToSkip {
+ s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+}
+```
+
+**Step 3: Remove sender in running_scheduler.go**
+
+In `backend/runner/taskrun/running_scheduler.go`, find and remove (around line 232):
+
+```go
+// DELETE this line:
+s.stateCfg.TaskSkippedOrDoneChan <- task.ID
+```
+
+**Step 4: Remove receiver in scheduler.go**
+
+In `backend/runner/taskrun/scheduler.go`, find the `runTaskCompletionListener` function (around line 82-110).
+
+Remove the entire case statement:
+
+```go
+// DELETE this entire case block:
+case taskUID := <-s.stateCfg.TaskSkippedOrDoneChan:
+ if err := func() error {
+ task, err := s.store.GetTaskByID(ctx, taskUID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get task")
+ }
+
+ // Check if entire plan is complete and handle webhooks
+ s.checkPlanCompletion(ctx, task.PlanID)
+
+ return nil
+ }(); err != nil {
+ slog.Error("failed to handle task completion", log.BBError(err))
+ }
+```
+
+**Step 5: Build to verify**
+
+Run: `go build ./backend/...`
+Expected: Build succeeds
+
+**Step 6: Commit channel removal**
+
+```bash
+git add backend/component/state/state.go backend/api/v1/rollout_service.go backend/runner/taskrun/running_scheduler.go backend/runner/taskrun/scheduler.go
+git commit -m "refactor: remove TaskSkippedOrDoneChan in-memory channel
+
+Remove in-memory channel replaced by database-backed webhook delivery.
+TaskSkippedOrDoneChan is no longer needed as webhooks are triggered directly
+in task failure and completion handlers.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Phase 7: Testing and Verification
+
+### Task 8: Manual Testing
+
+**Step 1: Start local Bytebase**
+
+Run: `PG_URL=postgresql://bbdev@localhost/bbdev go run ./backend/bin/server/main.go --port 8080 --data . --debug`
+Expected: Server starts, migration runs successfully
+
+**Step 2: Verify table created**
+
+Run: `psql -U bbdev bbdev -c "\d plan_webhook_delivery"`
+Expected: See table schema with plan_id PRIMARY KEY
+
+**Step 3: Test PIPELINE_FAILED webhook**
+
+1. Create a plan with tasks
+2. Make a task fail
+3. Check webhook sent
+4. Query: `SELECT * FROM plan_webhook_delivery;`
+ Expected: One row with event_type='PIPELINE_FAILED'
+
+**Step 4: Test retry doesn't send duplicate**
+
+1. Fail another task in same plan
+2. Check no duplicate webhook
+3. Query: `SELECT COUNT(*) FROM plan_webhook_delivery WHERE plan_id = ?;`
+ Expected: Still 1 row
+
+**Step 5: Test BatchRunTasks resets state**
+
+1. Click "Retry Tasks"
+2. Query: `SELECT * FROM plan_webhook_delivery WHERE plan_id = ?;`
+ Expected: 0 rows (deleted)
+3. Fail task again
+4. Check webhook sent again
+5. Query: `SELECT COUNT(*) FROM plan_webhook_delivery WHERE plan_id = ?;`
+ Expected: 1 row
+
+**Step 6: Test PIPELINE_COMPLETED**
+
+1. Retry and make all tasks succeed
+2. Check PIPELINE_COMPLETED webhook sent
+3. Query: `SELECT * FROM plan_webhook_delivery WHERE plan_id = ?;`
+ Expected: One row with event_type='PIPELINE_COMPLETED'
+
+**Step 7: Document test results**
+
+Create test notes in commit message for final commit.
+
+---
+
+## Phase 8: Final Commit and Integration
+
+### Task 9: Update Migrator Test
+
+**Files:**
+- Modify: `backend/migrator/migrator_test.go`
+
+**Step 1: Find TestLatestVersion**
+
+In `backend/migrator/migrator_test.go`, find `TestLatestVersion` function.
+
+**Step 2: Update latest version**
+
+Find the line that looks like:
+```go
+latestSchemaVersion := "3.14.15"
+```
+
+Increment the last number:
+```go
+latestSchemaVersion := "3.14.16"
+```
+
+**Step 3: Run test**
+
+Run: `go test -v ./backend/migrator/ -run TestLatestVersion`
+Expected: Test passes
+
+**Step 4: Commit migrator test update**
+
+```bash
+git add backend/migrator/migrator_test.go
+git commit -m "test: update latest schema version to 3.14.16
+
+Update for plan_webhook_delivery table migration.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+### Task 10: Run Full Build and Lint
+
+**Step 1: Format Go code**
+
+Run: `gofmt -w backend/store/plan_webhook_delivery.go backend/api/v1/rollout_service.go backend/runner/taskrun/scheduler.go backend/runner/taskrun/running_scheduler.go backend/component/state/state.go`
+Expected: Files formatted
+
+**Step 2: Run golangci-lint**
+
+Run: `golangci-lint run --allow-parallel-runners`
+Expected: No errors (run multiple times until clean)
+
+**Step 3: Auto-fix lint issues**
+
+Run: `golangci-lint run --fix --allow-parallel-runners`
+Expected: Auto-fixable issues resolved
+
+**Step 4: Build backend**
+
+Run: `go build -ldflags "-w -s" -p=16 -o ./bytebase-build/bytebase ./backend/bin/server/main.go`
+Expected: Build succeeds
+
+**Step 5: Commit any lint fixes**
+
+```bash
+git add -A
+git commit -m "chore: fix linting issues
+
+Auto-fix and manual lint issue resolution.
+
+🤖 Generated with Claude Code
+
+Co-Authored-By: Claude Sonnet 4.5 <[email protected]>"
+```
+
+---
+
+## Completion Checklist
+
+- [ ] Migration created for `plan_webhook_delivery` table
+- [ ] LATEST.sql updated with new table
+- [ ] Store methods implemented (Claim, Reset)
+- [ ] BatchRunTasks resets webhook state
+- [ ] PIPELINE_FAILED webhook sent on task failure
+- [ ] PIPELINE_COMPLETED webhook sent on plan completion
+- [ ] In-memory `TaskSkippedOrDoneChan` removed
+- [ ] Migrator test updated
+- [ ] Code formatted and linted
+- [ ] Manual testing completed
+- [ ] All commits have proper messages
+
+## Notes
+
+- Each webhook is sent exactly once per "phase" (initial run, each retry)
+- HA-safe: PRIMARY KEY constraint prevents duplicates across replicas
+- User action (BatchRunTasks) explicitly resets state
+- No polling or background workers needed
+- Immediate notification delivery