Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: add task run heartbeat for HA stale detection (#18793)

p0ny970da7381aeb
diff --git a/backend/bin/server/cmd/profile.go b/backend/bin/server/cmd/profile.go
index fcb054d53a..dc7dec6362 100644
--- a/backend/bin/server/cmd/profile.go
+++ b/backend/bin/server/cmd/profile.go
@@ -23,7 +23,7 @@ func getBaseProfile(dataDir string) *config.Profile {
Version: version,
GitCommit: gitcommit,
PgURL: os.Getenv("PG_URL"),
- DeployID: uuid.NewString()[:8],
+ DeployID: uuid.NewString(),
}
config.LastActiveTS.Store(time.Now().Unix())
diff --git a/backend/migrator/migration/3.14/0027##replica_heartbeat.sql b/backend/migrator/migration/3.14/0027##replica_heartbeat.sql
new file mode 100644
index 0000000000..72f76bcd7f
--- /dev/null
+++ b/backend/migrator/migration/3.14/0027##replica_heartbeat.sql
@@ -0,0 +1,16 @@
+-- Create replica heartbeat table
+CREATE TABLE replica_heartbeat (
+ replica_id TEXT PRIMARY KEY,
+ last_heartbeat TIMESTAMPTZ NOT NULL
+);
+
+-- Add replica_id column to task_run
+ALTER TABLE task_run ADD COLUMN replica_id TEXT;
+
+CREATE INDEX idx_task_run_running_replica ON task_run(replica_id) WHERE status = 'RUNNING' AND replica_id IS NOT NULL;
+
+-- Mark existing RUNNING task runs as FAILED
+UPDATE task_run
+SET status = 'FAILED',
+ result = '{"detail": "Marked as failed during migration"}'
+WHERE status = 'RUNNING';
diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sql
index cef814dffb..1762b8e07e 100644
--- a/backend/migrator/migration/LATEST.sql
+++ b/backend/migrator/migration/LATEST.sql
@@ -265,7 +265,8 @@ CREATE TABLE task_run (
run_at timestamptz,
-- result saves the task run result in json format
-- Stored as TaskRunResult (proto/store/store/task_run.proto)
- result jsonb NOT NULL DEFAULT '{}'
+ result jsonb NOT NULL DEFAULT '{}',
+ replica_id TEXT
);
CREATE INDEX idx_task_run_task_id ON task_run(task_id);
@@ -279,6 +280,15 @@ CREATE INDEX idx_task_run_active_status_id ON task_run(status, id) WHERE status
ALTER SEQUENCE task_run_id_seq RESTART WITH 101;
+CREATE INDEX idx_task_run_running_replica ON task_run(replica_id) WHERE status = 'RUNNING' AND replica_id IS NOT NULL;
+
+-- replica_heartbeat tracks active replicas in HA deployments.
+-- Used to detect and clean up stale RUNNING task runs from crashed replicas.
+CREATE TABLE replica_heartbeat (
+ replica_id TEXT PRIMARY KEY,
+ last_heartbeat TIMESTAMPTZ NOT NULL
+);
+
CREATE TABLE task_run_log (
id bigserial PRIMARY KEY,
task_run_id integer NOT NULL REFERENCES task_run(id),
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index 9bafbb3d1a..1b3284600f 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -12,8 +12,8 @@ import (
func TestLatestVersion(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
- require.Equal(t, semver.MustParse("3.14.26"), *files[len(files)-1].version)
- require.Equal(t, "migration/3.14/0026##drop_release_digest_unique_constraint.sql", files[len(files)-1].path)
+ require.Equal(t, semver.MustParse("3.14.27"), *files[len(files)-1].version)
+ require.Equal(t, "migration/3.14/0027##replica_heartbeat.sql", files[len(files)-1].path)
}
func TestVersionUnique(t *testing.T) {
diff --git a/backend/runner/cleaner/data_cleaner.go b/backend/runner/cleaner/data_cleaner.go
index 0710b938ff..6e9f36e301 100644
--- a/backend/runner/cleaner/data_cleaner.go
+++ b/backend/runner/cleaner/data_cleaner.go
@@ -2,7 +2,6 @@ package cleaner
import (
"context"
- "fmt"
"log/slog"
"sync"
"time"
@@ -13,6 +12,9 @@ import (
const (
cleanupInterval = 1 * time.Hour
+ staleDetectionInterval = 30 * time.Second
+ stalenessThreshold = 1 * time.Minute
+ heartbeatRetentionPeriod = 1 * time.Hour
exportArchiveRetentionPeriod = 24 * time.Hour
oauth2ClientRetentionPeriod = 30 * 24 * time.Hour // 30 days of inactivity
)
@@ -32,17 +34,27 @@ func NewDataCleaner(store *store.Store) *DataCleaner {
// Run starts the DataCleaner.
func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done()
- ticker := time.NewTicker(cleanupInterval)
- defer ticker.Stop()
- slog.Debug(fmt.Sprintf("Data cleaner started and will run every %v", cleanupInterval))
- // Run once immediately on startup
+ cleanupTicker := time.NewTicker(cleanupInterval)
+ defer cleanupTicker.Stop()
+
+ staleTicker := time.NewTicker(staleDetectionInterval)
+ defer staleTicker.Stop()
+
+ slog.Debug("Data cleaner started",
+ slog.Duration("cleanupInterval", cleanupInterval),
+ slog.Duration("staleDetectionInterval", staleDetectionInterval))
+
+ // Run cleanup immediately on startup
c.cleanup(ctx)
+ c.detectStaleTaskRuns(ctx)
for {
select {
- case <-ticker.C:
+ case <-cleanupTicker.C:
c.cleanup(ctx)
+ case <-staleTicker.C:
+ c.detectStaleTaskRuns(ctx)
case <-ctx.Done():
return
}
@@ -53,6 +65,29 @@ func (c *DataCleaner) cleanup(ctx context.Context) {
c.cleanupExportArchives(ctx)
c.cleanupOAuth2Data(ctx)
c.cleanupWebRefreshTokens(ctx)
+ c.cleanupStaleHeartbeats(ctx)
+}
+
+func (c *DataCleaner) detectStaleTaskRuns(ctx context.Context) {
+ rowsAffected, err := c.store.FailStaleTaskRuns(ctx, stalenessThreshold)
+ if err != nil {
+ slog.Error("Failed to detect stale task runs", log.BBError(err))
+ return
+ }
+ if rowsAffected > 0 {
+ slog.Info("Marked stale task runs as failed", slog.Int64("count", rowsAffected))
+ }
+}
+
+func (c *DataCleaner) cleanupStaleHeartbeats(ctx context.Context) {
+ rowsAffected, err := c.store.DeleteStaleReplicaHeartbeats(ctx, heartbeatRetentionPeriod)
+ if err != nil {
+ slog.Error("Failed to clean up stale replica heartbeats", log.BBError(err))
+ return
+ }
+ if rowsAffected > 0 {
+ slog.Info("Cleaned up stale replica heartbeats", slog.Int64("count", rowsAffected))
+ }
}
func (c *DataCleaner) cleanupExportArchives(ctx context.Context) {
diff --git a/backend/runner/heartbeat/runner.go b/backend/runner/heartbeat/runner.go
new file mode 100644
index 0000000000..b305bdf96e
--- /dev/null
+++ b/backend/runner/heartbeat/runner.go
@@ -0,0 +1,58 @@
+package heartbeat
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+const (
+ heartbeatInterval = 10 * time.Second
+)
+
+// Runner sends periodic heartbeats to indicate this replica is alive.
+type Runner struct {
+ store *store.Store
+ profile *config.Profile
+}
+
+// NewRunner creates a new heartbeat runner.
+func NewRunner(store *store.Store, profile *config.Profile) *Runner {
+ return &Runner{
+ store: store,
+ profile: profile,
+ }
+}
+
+// Run starts the heartbeat runner.
+func (r *Runner) Run(ctx context.Context, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ ticker := time.NewTicker(heartbeatInterval)
+ defer ticker.Stop()
+
+ slog.Debug("Heartbeat runner started", slog.String("replicaID", r.profile.DeployID))
+
+ // Send heartbeat immediately on startup
+ r.sendHeartbeat(ctx)
+
+ for {
+ select {
+ case <-ticker.C:
+ r.sendHeartbeat(ctx)
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (r *Runner) sendHeartbeat(ctx context.Context) {
+ if err := r.store.UpsertReplicaHeartbeat(ctx, r.profile.DeployID); err != nil {
+ slog.Error("Failed to send heartbeat", log.BBError(err))
+ }
+}
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index 15e79546fd..f9b09d17be 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -44,7 +44,7 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
func (s *Scheduler) scheduleRunningTaskRuns(ctx context.Context) error {
// Atomically claim all AVAILABLE task runs
- claimed, err := s.store.ClaimAvailableTaskRuns(ctx)
+ claimed, err := s.store.ClaimAvailableTaskRuns(ctx, s.profile.DeployID)
if err != nil {
return errors.Wrapf(err, "failed to claim available task runs")
}
diff --git a/backend/server/server.go b/backend/server/server.go
index d2401ed26c..5827504f1d 100644
--- a/backend/server/server.go
+++ b/backend/server/server.go
@@ -36,6 +36,7 @@ import (
"github.com/bytebase/bytebase/backend/resources/postgres"
"github.com/bytebase/bytebase/backend/runner/approval"
"github.com/bytebase/bytebase/backend/runner/cleaner"
+ "github.com/bytebase/bytebase/backend/runner/heartbeat"
"github.com/bytebase/bytebase/backend/runner/monitor"
"github.com/bytebase/bytebase/backend/runner/notifylistener"
"github.com/bytebase/bytebase/backend/runner/plancheck"
@@ -62,6 +63,7 @@ type Server struct {
approvalRunner *approval.Runner
notifyListener *notifylistener.Listener
dataCleaner *cleaner.DataCleaner
+ heartbeatRunner *heartbeat.Runner
runnerWG sync.WaitGroup
webhookManager *webhook.Manager
@@ -201,6 +203,9 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
// Data cleaner
s.dataCleaner = cleaner.NewDataCleaner(stores)
+ // Heartbeat runner
+ s.heartbeatRunner = heartbeat.NewRunner(stores, profile)
+
// LSP server.
s.lspServer = lsp.NewServer(s.store, profile, secret, s.bus, s.iamManager, s.licenseService)
@@ -238,6 +243,9 @@ func (s *Server) Run(ctx context.Context, port int) error {
s.runnerWG.Add(1)
go s.dataCleaner.Run(ctx, &s.runnerWG)
+ s.runnerWG.Add(1)
+ go s.heartbeatRunner.Run(ctx, &s.runnerWG)
+
s.runnerWG.Add(1)
go s.notifyListener.Run(ctx, &s.runnerWG)
diff --git a/backend/store/replica_heartbeat.go b/backend/store/replica_heartbeat.go
new file mode 100644
index 0000000000..52771be508
--- /dev/null
+++ b/backend/store/replica_heartbeat.go
@@ -0,0 +1,49 @@
+package store
+
+import (
+ "context"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/qb"
+)
+
+// UpsertReplicaHeartbeat updates or inserts a replica heartbeat.
+func (s *Store) UpsertReplicaHeartbeat(ctx context.Context, replicaID string) error {
+ q := qb.Q().Space(`
+ INSERT INTO replica_heartbeat (replica_id, last_heartbeat)
+ VALUES (?, now())
+ ON CONFLICT (replica_id)
+ DO UPDATE SET last_heartbeat = now()
+ `, replicaID)
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return errors.Wrapf(err, "failed to build sql")
+ }
+
+ if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {
+ return errors.Wrapf(err, "failed to upsert replica heartbeat")
+ }
+ return nil
+}
+
+// DeleteStaleReplicaHeartbeats deletes heartbeat rows older than the given duration.
+func (s *Store) DeleteStaleReplicaHeartbeats(ctx context.Context, olderThan time.Duration) (int64, error) {
+ q := qb.Q().Space(`
+ DELETE FROM replica_heartbeat
+ WHERE last_heartbeat < now() - ?::INTERVAL
+ `, olderThan.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to build sql")
+ }
+
+ result, err := s.GetDB().ExecContext(ctx, query, args...)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to delete stale replica heartbeats")
+ }
+ return result.RowsAffected()
+}
diff --git a/backend/store/task_run.go b/backend/store/task_run.go
index 9debc7e5ba..076177c603 100644
--- a/backend/store/task_run.go
+++ b/backend/store/task_run.go
@@ -220,17 +220,17 @@ type ClaimedTaskRun struct {
// ClaimAvailableTaskRuns atomically claims all AVAILABLE task runs by updating them to RUNNING
// and returns the claimed task run and task UIDs. This combines list + claim into a single atomic operation.
// Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different tasks.
-func (s *Store) ClaimAvailableTaskRuns(ctx context.Context) ([]*ClaimedTaskRun, error) {
+func (s *Store) ClaimAvailableTaskRuns(ctx context.Context, replicaID string) ([]*ClaimedTaskRun, error) {
q := qb.Q().Space(`
UPDATE task_run
- SET status = ?, updated_at = now()
+ SET status = ?, updated_at = now(), replica_id = ?
WHERE id IN (
SELECT task_run.id FROM task_run
WHERE task_run.status = ?
FOR UPDATE SKIP LOCKED
)
RETURNING id, task_id
- `, storepb.TaskRun_RUNNING.String(), storepb.TaskRun_AVAILABLE.String())
+ `, storepb.TaskRun_RUNNING.String(), replicaID, storepb.TaskRun_AVAILABLE.String())
query, args, err := q.ToSQL()
if err != nil {
@@ -418,3 +418,35 @@ func (s *Store) BatchCancelTaskRuns(ctx context.Context, taskRunIDs []int) error
}
return nil
}
+
+// FailStaleTaskRuns marks RUNNING task runs as FAILED if their replica is dead.
+// A replica is considered dead if:
+// 1. Its replica_id is not in the replica_heartbeat table, OR
+// 2. Its last_heartbeat is older than the staleness threshold
+// Returns the number of task runs marked as failed.
+func (s *Store) FailStaleTaskRuns(ctx context.Context, stalenessThreshold time.Duration) (int64, error) {
+ q := qb.Q().Space(`
+ UPDATE task_run
+ SET status = ?,
+ result = '{"detail": "Task run abandoned: owning replica stopped responding"}',
+ updated_at = now()
+ WHERE status = ?
+ AND replica_id IS NOT NULL
+ AND NOT EXISTS (
+ SELECT 1 FROM replica_heartbeat rh
+ WHERE rh.replica_id = task_run.replica_id
+ AND rh.last_heartbeat >= now() - ?::INTERVAL
+ )
+ `, storepb.TaskRun_FAILED.String(), storepb.TaskRun_RUNNING.String(), stalenessThreshold.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to build sql")
+ }
+
+ result, err := s.GetDB().ExecContext(ctx, query, args...)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to fail stale task runs")
+ }
+ return result.RowsAffected()
+}
diff --git a/docs/plans/2026-01-07-task-run-heartbeat-design.md b/docs/plans/2026-01-07-task-run-heartbeat-design.md
new file mode 100644
index 0000000000..264225bb9f
--- /dev/null
+++ b/docs/plans/2026-01-07-task-run-heartbeat-design.md
@@ -0,0 +1,621 @@
+# Task Run Heartbeat Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Detect stale RUNNING task runs in HA deployments by tracking replica heartbeats.
+
+**Architecture:** Each Bytebase replica sends heartbeats to a `replica_heartbeat` table using its `deployId`. Task runs record which replica claimed them. Data cleaner detects stale tasks (replica missing or heartbeat too old) and marks them FAILED.
+
+**Tech Stack:** Go, PostgreSQL, existing runner infrastructure
+
+---
+
+## Task 1: Add Database Migration
+
+**Files:**
+- Create: `backend/migrator/migration/3.10/0000##replica_heartbeat.sql`
+- Modify: `backend/migrator/migration/LATEST.sql:256-280`
+- Modify: `backend/migrator/migrator_test.go` (update version)
+
+**Step 1: Create migration file**
+
+Create `backend/migrator/migration/3.10/0000##replica_heartbeat.sql`:
+
+```sql
+-- Create replica heartbeat table
+CREATE TABLE replica_heartbeat (
+ replica_id TEXT PRIMARY KEY,
+ last_heartbeat TIMESTAMPTZ NOT NULL
+);
+
+-- Add replica_id column to task_run
+ALTER TABLE task_run ADD COLUMN replica_id TEXT;
+
+-- Mark existing RUNNING task runs as FAILED
+UPDATE task_run
+SET status = 'FAILED',
+ result = '{"detail": "Marked as failed during heartbeat migration"}'
+WHERE status = 'RUNNING';
+```
+
+**Step 2: Update LATEST.sql**
+
+Add to `backend/migrator/migration/LATEST.sql` after task_run table definition (around line 269):
+
+```sql
+-- Add after: result jsonb NOT NULL DEFAULT '{}'
+ replica_id TEXT
+```
+
+Add new table before task_run_log (around line 280):
+
+```sql
+CREATE TABLE replica_heartbeat (
+ replica_id TEXT PRIMARY KEY,
+ last_heartbeat TIMESTAMPTZ NOT NULL
+);
+```
+
+**Step 3: Update migrator_test.go**
+
+Update `TestLatestVersion` to include version "3.10".
+
+**Step 4: Run migration locally**
+
+```bash
+# Verify migration syntax
+psql -U bbdev bbdev -f backend/migrator/migration/3.10/0000##replica_heartbeat.sql
+```
+
+**Step 5: Commit**
+
+```bash
+but commit <branch> -m "chore: add replica heartbeat migration"
+```
+
+---
+
+## Task 2: Add Store Methods for Heartbeat
+
+**Files:**
+- Create: `backend/store/replica_heartbeat.go`
+
+**Step 1: Create store file**
+
+Create `backend/store/replica_heartbeat.go`:
+
+```go
+package store
+
+import (
+ "context"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common/qb"
+)
+
+// UpsertReplicaHeartbeat updates or inserts a replica heartbeat.
+func (s *Store) UpsertReplicaHeartbeat(ctx context.Context, replicaID string) error {
+ q := qb.Q().Space(`
+ INSERT INTO replica_heartbeat (replica_id, last_heartbeat)
+ VALUES (?, now())
+ ON CONFLICT (replica_id)
+ DO UPDATE SET last_heartbeat = now()
+ `, replicaID)
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return errors.Wrapf(err, "failed to build sql")
+ }
+
+ if _, err := s.GetDB().ExecContext(ctx, query, args...); err != nil {
+ return errors.Wrapf(err, "failed to upsert replica heartbeat")
+ }
+ return nil
+}
+
+// DeleteStaleReplicaHeartbeats deletes heartbeat rows older than the given duration.
+func (s *Store) DeleteStaleReplicaHeartbeats(ctx context.Context, olderThan time.Duration) (int64, error) {
+ q := qb.Q().Space(`
+ DELETE FROM replica_heartbeat
+ WHERE last_heartbeat < now() - ?::INTERVAL
+ `, olderThan.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to build sql")
+ }
+
+ result, err := s.GetDB().ExecContext(ctx, query, args...)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to delete stale replica heartbeats")
+ }
+ return result.RowsAffected()
+}
+```
+
+**Step 2: Verify it compiles**
+
+```bash
+go build ./backend/store/...
+```
+
+**Step 3: Commit**
+
+```bash
+but commit <branch> -m "feat: add replica heartbeat store methods"
+```
+
+---
+
+## Task 3: Add Store Method for Stale Task Detection
+
+**Files:**
+- Modify: `backend/store/task_run.go`
+
+**Step 1: Add FailStaleTaskRuns method**
+
+Add to `backend/store/task_run.go`:
+
+```go
+// FailStaleTaskRuns marks RUNNING task runs as FAILED if their replica is dead.
+// A replica is considered dead if:
+// 1. Its replica_id is not in the replica_heartbeat table, OR
+// 2. Its last_heartbeat is older than the staleness threshold
+// Returns the number of task runs marked as failed.
+func (s *Store) FailStaleTaskRuns(ctx context.Context, stalenessThreshold time.Duration) (int64, error) {
+ q := qb.Q().Space(`
+ UPDATE task_run
+ SET status = ?,
+ result = '{"detail": "Task run abandoned: owning replica stopped responding"}',
+ updated_at = now()
+ WHERE status = ?
+ AND replica_id IS NOT NULL
+ AND (
+ replica_id NOT IN (SELECT replica_id FROM replica_heartbeat)
+ OR replica_id IN (
+ SELECT replica_id FROM replica_heartbeat
+ WHERE last_heartbeat < now() - ?::INTERVAL
+ )
+ )
+ `, storepb.TaskRun_FAILED.String(), storepb.TaskRun_RUNNING.String(), stalenessThreshold.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to build sql")
+ }
+
+ result, err := s.GetDB().ExecContext(ctx, query, args...)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to fail stale task runs")
+ }
+ return result.RowsAffected()
+}
+```
+
+**Step 2: Verify it compiles**
+
+```bash
+go build ./backend/store/...
+```
+
+**Step 3: Commit**
+
+```bash
+but commit <branch> -m "feat: add store method to fail stale task runs"
+```
+
+---
+
+## Task 4: Modify ClaimAvailableTaskRuns to Set replica_id
+
+**Files:**
+- Modify: `backend/store/task_run.go:220-258`
+
+**Step 1: Update ClaimAvailableTaskRuns signature and query**
+
+Modify `ClaimAvailableTaskRuns` in `backend/store/task_run.go`:
+
+```go
+// ClaimAvailableTaskRuns atomically claims all AVAILABLE task runs by updating them to RUNNING
+// and returns the claimed task run and task UIDs. This combines list + claim into a single atomic operation.
+// Uses FOR UPDATE SKIP LOCKED to allow concurrent schedulers to claim different tasks.
+func (s *Store) ClaimAvailableTaskRuns(ctx context.Context, replicaID string) ([]*ClaimedTaskRun, error) {
+ q := qb.Q().Space(`
+ UPDATE task_run
+ SET status = ?, updated_at = now(), replica_id = ?
+ WHERE id IN (
+ SELECT task_run.id FROM task_run
+ WHERE task_run.status = ?
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, task_id
+ `, storepb.TaskRun_RUNNING.String(), replicaID, storepb.TaskRun_AVAILABLE.String())
+
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to build sql")
+ }
+
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to claim task runs")
+ }
+ defer rows.Close()
+
+ var claimed []*ClaimedTaskRun
+ for rows.Next() {
+ var c ClaimedTaskRun
+ if err := rows.Scan(&c.TaskRunUID, &c.TaskUID); err != nil {
+ return nil, err
+ }
+ claimed = append(claimed, &c)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return claimed, nil
+}
+```
+
+**Step 2: Update caller in running_scheduler.go**
+
+Find the call to `ClaimAvailableTaskRuns` in `backend/runner/taskrun/running_scheduler.go` and add the replicaID parameter:
+
+```go
+// Change from:
+claimed, err := s.store.ClaimAvailableTaskRuns(ctx)
+// To:
+claimed, err := s.store.ClaimAvailableTaskRuns(ctx, s.profile.DeployID)
+```
+
+**Step 3: Verify it compiles**
+
+```bash
+go build ./backend/...
+```
+
+**Step 4: Commit**
+
+```bash
+but commit <branch> -m "feat: set replica_id when claiming task runs"
+```
+
+---
+
+## Task 5: Create Heartbeat Runner
+
+**Files:**
+- Create: `backend/runner/heartbeat/runner.go`
+
+**Step 1: Create heartbeat runner**
+
+Create `backend/runner/heartbeat/runner.go`:
+
+```go
+package heartbeat
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/config"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+const (
+ heartbeatInterval = 10 * time.Second
+)
+
+// Runner sends periodic heartbeats to indicate this replica is alive.
+type Runner struct {
+ store *store.Store
+ profile *config.Profile
+}
+
+// NewRunner creates a new heartbeat runner.
+func NewRunner(store *store.Store, profile *config.Profile) *Runner {
+ return &Runner{
+ store: store,
+ profile: profile,
+ }
+}
+
+// Run starts the heartbeat runner.
+func (r *Runner) Run(ctx context.Context, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ ticker := time.NewTicker(heartbeatInterval)
+ defer ticker.Stop()
+
+ slog.Debug("Heartbeat runner started", slog.String("replicaID", r.profile.DeployID))
+
+ // Send heartbeat immediately on startup
+ r.sendHeartbeat(ctx)
+
+ for {
+ select {
+ case <-ticker.C:
+ r.sendHeartbeat(ctx)
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+
+func (r *Runner) sendHeartbeat(ctx context.Context) {
+ if err := r.store.UpsertReplicaHeartbeat(ctx, r.profile.DeployID); err != nil {
+ slog.Error("Failed to send heartbeat", log.BBError(err))
+ }
+}
+```
+
+**Step 2: Verify it compiles**
+
+```bash
+go build ./backend/runner/heartbeat/...
+```
+
+**Step 3: Commit**
+
+```bash
+but commit <branch> -m "feat: add heartbeat runner"
+```
+
+---
+
+## Task 6: Add Stale Detection to Data Cleaner
+
+**Files:**
+- Modify: `backend/runner/cleaner/data_cleaner.go`
+
+**Step 1: Add constants and stale detection ticker**
+
+Modify `backend/runner/cleaner/data_cleaner.go`:
+
+```go
+const (
+ cleanupInterval = 1 * time.Hour
+ staleDetectionInterval = 30 * time.Second
+ stalenessThreshold = 1 * time.Minute
+ heartbeatRetentionPeriod = 1 * time.Hour
+ exportArchiveRetentionPeriod = 24 * time.Hour
+ oauth2ClientRetentionPeriod = 30 * 24 * time.Hour // 30 days of inactivity
+)
+```
+
+**Step 2: Update Run method with dual tickers**
+
+Replace the `Run` method:
+
+```go
+// Run starts the DataCleaner.
+func (c *DataCleaner) Run(ctx context.Context, wg *sync.WaitGroup) {
+ defer wg.Done()
+
+ cleanupTicker := time.NewTicker(cleanupInterval)
+ defer cleanupTicker.Stop()
+
+ staleTicker := time.NewTicker(staleDetectionInterval)
+ defer staleTicker.Stop()
+
+ slog.Debug("Data cleaner started",
+ slog.Duration("cleanupInterval", cleanupInterval),
+ slog.Duration("staleDetectionInterval", staleDetectionInterval))
+
+ // Run cleanup immediately on startup
+ c.cleanup(ctx)
+ c.detectStaleTaskRuns(ctx)
+
+ for {
+ select {
+ case <-cleanupTicker.C:
+ c.cleanup(ctx)
+ case <-staleTicker.C:
+ c.detectStaleTaskRuns(ctx)
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+```
+
+**Step 3: Add stale detection and heartbeat cleanup methods**
+
+Add these methods:
+
+```go
+func (c *DataCleaner) detectStaleTaskRuns(ctx context.Context) {
+ rowsAffected, err := c.store.FailStaleTaskRuns(ctx, stalenessThreshold)
+ if err != nil {
+ slog.Error("Failed to detect stale task runs", log.BBError(err))
+ return
+ }
+ if rowsAffected > 0 {
+ slog.Info("Marked stale task runs as failed", slog.Int64("count", rowsAffected))
+ }
+}
+
+func (c *DataCleaner) cleanupStaleHeartbeats(ctx context.Context) {
+ rowsAffected, err := c.store.DeleteStaleReplicaHeartbeats(ctx, heartbeatRetentionPeriod)
+ if err != nil {
+ slog.Error("Failed to clean up stale replica heartbeats", log.BBError(err))
+ return
+ }
+ if rowsAffected > 0 {
+ slog.Info("Cleaned up stale replica heartbeats", slog.Int64("count", rowsAffected))
+ }
+}
+```
+
+**Step 4: Update cleanup method to include heartbeat cleanup**
+
+```go
+func (c *DataCleaner) cleanup(ctx context.Context) {
+ c.cleanupExportArchives(ctx)
+ c.cleanupOAuth2Data(ctx)
+ c.cleanupWebRefreshTokens(ctx)
+ c.cleanupStaleHeartbeats(ctx)
+}
+```
+
+**Step 5: Verify it compiles**
+
+```bash
+go build ./backend/runner/cleaner/...
+```
+
+**Step 6: Commit**
+
+```bash
+but commit <branch> -m "feat: add stale task detection to data cleaner"
+```
+
+---
+
+## Task 7: Register Heartbeat Runner in Server
+
+**Files:**
+- Modify: `backend/server/server.go`
+
+**Step 1: Add import**
+
+Add to imports:
+
+```go
+"github.com/bytebase/bytebase/backend/runner/heartbeat"
+```
+
+**Step 2: Add field to Server struct**
+
+Add field (around line 64):
+
+```go
+heartbeatRunner *heartbeat.Runner
+```
+
+**Step 3: Initialize runner**
+
+Add after dataCleaner initialization (around line 202):
+
+```go
+// Heartbeat runner
+s.heartbeatRunner = heartbeat.NewRunner(stores, profile)
+```
+
+**Step 4: Start runner**
+
+Add after dataCleaner.Run (around line 239):
+
+```go
+s.runnerWG.Add(1)
+go s.heartbeatRunner.Run(ctx, &s.runnerWG)
+```
+
+**Step 5: Verify it compiles**
+
+```bash
+go build ./backend/...
+```
+
+**Step 6: Commit**
+
+```bash
+but commit <branch> -m "feat: register heartbeat runner in server"
+```
+
+---
+
+## Task 8: Run Linter and Fix Issues
+
+**Step 1: Run golangci-lint**
+
+```bash
+golangci-lint run --allow-parallel-runners
+```
+
+**Step 2: Fix any issues**
+
+Common issues to watch for:
+- Unused imports
+- Missing error checks
+- Formatting issues
+
+**Step 3: Run again until clean**
+
+```bash
+golangci-lint run --allow-parallel-runners
+```
+
+**Step 4: Commit fixes if any**
+
+```bash
+but commit <branch> -m "fix: address linter issues"
+```
+
+---
+
+## Task 9: Manual Testing
+
+**Step 1: Start the server**
+
+```bash
+PG_URL=postgresql://bbdev@localhost/bbdev go run ./backend/bin/server/main.go --port 8080 --data . --debug
+```
+
+**Step 2: Verify heartbeat table is populated**
+
+```bash
+psql -U bbdev bbdev -c "SELECT * FROM replica_heartbeat;"
+```
+
+**Step 3: Verify task runs get replica_id**
+
+Create a task run and check:
+
+```bash
+psql -U bbdev bbdev -c "SELECT id, status, replica_id FROM task_run ORDER BY id DESC LIMIT 5;"
+```
+
+**Step 4: Test stale detection**
+
+Manually insert a stale task run and verify it gets marked as FAILED:
+
+```bash
+# Insert a fake running task with unknown replica
+psql -U bbdev bbdev -c "
+ UPDATE task_run
+ SET status = 'RUNNING', replica_id = 'fake-replica-id'
+ WHERE id = (SELECT id FROM task_run LIMIT 1);
+"
+
+# Wait 30 seconds for stale detection to run
+sleep 35
+
+# Verify it was marked as FAILED
+psql -U bbdev bbdev -c "
+ SELECT id, status, result FROM task_run
+ WHERE replica_id = 'fake-replica-id';
+"
+```
+
+---
+
+## Summary
+
+| Task | Description |
+|------|-------------|
+| 1 | Database migration (replica_heartbeat table, task_run.replica_id) |
+| 2 | Store methods for heartbeat upsert and cleanup |
+| 3 | Store method for stale task detection |
+| 4 | Modify ClaimAvailableTaskRuns to set replica_id |
+| 5 | Create heartbeat runner |
+| 6 | Add stale detection to data cleaner |
+| 7 | Register heartbeat runner in server |
+| 8 | Lint and fix issues |
+| 9 | Manual testing |