Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(ha): fail task runs with error instead of frontend banner (#19912)

Danny Xue8449bbd5df1
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 64b17dd5db..d5fa45779f 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -26,17 +26,24 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Wa
select {
case <-ticker.C:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
- slog.Warn("Pending task runs scheduler skipped due to HA license restriction", log.BBError(err))
+ if s.haFailSince.IsZero() {
+ s.haFailSince = time.Now()
+ }
+ if time.Since(s.haFailSince) >= haFailGracePeriod {
+ s.failTaskRunsForHA(ctx, err)
+ }
continue
}
+ s.haFailSince = time.Time{}
if err := s.schedulePendingTaskRuns(ctx); err != nil {
slog.Error("failed to schedule pending task runs", log.BBError(err))
}
case <-s.bus.TaskRunTickleChan:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
- slog.Warn("Pending task runs scheduler skipped due to HA license restriction", log.BBError(err))
+ // Grace period is tracked by the ticker branch; just skip here.
continue
}
+ s.haFailSince = time.Time{}
if err := s.schedulePendingTaskRuns(ctx); err != nil {
slog.Error("failed to schedule pending task runs", log.BBError(err))
}
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index c921059382..491b772c9e 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -30,7 +30,7 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
select {
case <-ticker.C:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
- slog.Warn("Running task runs scheduler skipped due to HA license restriction", log.BBError(err))
+ // Pending scheduler handles failing task runs; just skip here.
continue
}
if err := s.scheduleRunningTaskRuns(ctx); err != nil {
@@ -38,7 +38,6 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa
}
case <-s.bus.TaskRunTickleChan:
if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
- slog.Warn("Running task runs scheduler skipped due to HA license restriction", log.BBError(err))
continue
}
if err := s.scheduleRunningTaskRuns(ctx); err != nil {
diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.go
index 048bf44b94..67d21899d4 100644
--- a/backend/runner/taskrun/scheduler.go
+++ b/backend/runner/taskrun/scheduler.go
@@ -19,6 +19,11 @@ import (
const (
taskSchedulerInterval = 5 * time.Second
+ // haFailGracePeriod is the duration to wait before failing task runs after
+ // the HA license check starts failing. This avoids permanently failing task
+ // runs during rolling restarts where stale heartbeats cause transient
+ // over-counts.
+ haFailGracePeriod = 10 * time.Minute
)
// Scheduler is the scheduler for task run.
@@ -29,6 +34,9 @@ type Scheduler struct {
licenseService *enterprise.LicenseService
executorMap map[storepb.Task_Type]Executor
profile *config.Profile
+ // haFailSince is when CheckReplicaLimit first started failing.
+ // Zero means the check is currently passing.
+ haFailSince time.Time
}
// NewScheduler will create a new scheduler.
@@ -60,6 +68,79 @@ func (s *Scheduler) Register(taskType storepb.Task_Type, executorGetter Executor
s.executorMap[taskType] = executorGetter
}
+// failTaskRunsForHA fails all PENDING and AVAILABLE task runs because HA is not licensed.
+func (s *Scheduler) failTaskRunsForHA(ctx context.Context, haErr error) {
+ taskRuns, err := s.store.ListTaskRunsByStatus(ctx, []storepb.TaskRun_Status{storepb.TaskRun_PENDING, storepb.TaskRun_AVAILABLE})
+ if err != nil {
+ slog.Error("failed to list task runs for HA limit check", log.BBError(err))
+ return
+ }
+ if len(taskRuns) == 0 {
+ return
+ }
+
+ // Track affected plans to send failure webhooks.
+ // Key by (projectID, planID) to match ClaimPipelineFailureNotification's dedup key.
+ type planKey struct {
+ projectID string
+ planID int64
+ }
+ affectedPlans := map[planKey]string{} // value = first environment seen
+
+ for _, taskRun := range taskRuns {
+ if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
+ ID: taskRun.ID,
+ ProjectID: taskRun.ProjectID,
+ Status: storepb.TaskRun_FAILED,
+ ResultProto: &storepb.TaskRunResult{
+ Detail: haErr.Error(),
+ },
+ }); err != nil {
+ slog.Error("failed to fail task run for HA limit",
+ slog.Int64("taskRunID", taskRun.ID),
+ log.BBError(err),
+ )
+ continue
+ }
+ key := planKey{projectID: taskRun.ProjectID, planID: taskRun.PlanUID}
+ if _, ok := affectedPlans[key]; !ok {
+ affectedPlans[key] = taskRun.Environment
+ }
+ }
+
+ slog.Warn("Failed task runs due to HA license restriction", slog.Int64("count", int64(len(taskRuns))), log.BBError(haErr))
+
+ // Send PIPELINE_FAILED webhook for each affected plan.
+ for key, environment := range affectedPlans {
+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, key.projectID, key.planID)
+ if err != nil {
+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))
+ continue
+ }
+ if !claimed {
+ continue
+ }
+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{ProjectID: key.projectID, UID: &key.planID})
+ if err != nil || plan == nil {
+ slog.Error("failed to get plan for HA failure webhook", log.BBError(err))
+ continue
+ }
+ project, err := s.store.GetProjectByResourceID(ctx, plan.ProjectID)
+ if err != nil || project == nil {
+ slog.Error("failed to get project for HA failure webhook", log.BBError(err))
+ continue
+ }
+ s.webhookManager.CreateEvent(ctx, &webhook.Event{
+ Type: storepb.Activity_PIPELINE_FAILED,
+ Project: webhook.NewProject(project),
+ RolloutFailed: &webhook.EventRolloutFailed{
+ Rollout: webhook.NewRollout(plan),
+ Environment: environment,
+ },
+ })
+ }
+}
+
// Run will start the scheduler.
func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done()
diff --git a/frontend/src/components/BannersWrapper.vue b/frontend/src/components/BannersWrapper.vue
index dd588dc9b5..0ac57901cc 100644
--- a/frontend/src/components/BannersWrapper.vue
+++ b/frontend/src/components/BannersWrapper.vue
@@ -3,7 +3,6 @@
<BannerDemo v-if="shouldShowDemoBanner" />
<BannerSubscription v-if="shouldShowSubscriptionBanner" />
<BannerExternalUrl v-if="shouldShowExternalUrlBanner" />
- <BannerHA v-if="shouldShowHABanner" />
<BannerAnnouncement />
</template>
@@ -20,15 +19,14 @@ import { isDev } from "@/utils";
import BannerAnnouncement from "@/views/BannerAnnouncement.vue";
import BannerDemo from "@/views/BannerDemo.vue";
import BannerExternalUrl from "@/views/BannerExternalUrl.vue";
-import BannerHA from "@/views/BannerHA.vue";
import BannerSubscription from "@/views/BannerSubscription.vue";
import BannerUpgradeSubscription from "@/views/BannerUpgradeSubscription.vue";
const actuatorStore = useActuatorV1Store();
const subscriptionStore = useSubscriptionV1Store();
-const { needConfigureExternalUrl, replicaCount } = storeToRefs(actuatorStore);
-const { isExpired, isTrialing, currentPlan, daysBeforeExpire, isHAAllowed } =
+const { needConfigureExternalUrl } = storeToRefs(actuatorStore);
+const { isExpired, isTrialing, currentPlan, daysBeforeExpire } =
storeToRefs(subscriptionStore);
const shouldShowDemoBanner = computed(() => {
@@ -48,8 +46,4 @@ const shouldShowSubscriptionBanner = computed(() => {
const shouldShowExternalUrlBanner = computed(() => {
return !isDev() && needConfigureExternalUrl.value;
});
-
-const shouldShowHABanner = computed(() => {
- return !isHAAllowed.value && replicaCount.value > 1;
-});
</script>
diff --git a/frontend/src/locales/en-US.json b/frontend/src/locales/en-US.json
index 2aca417549..76bf68518b 100644
--- a/frontend/src/locales/en-US.json
+++ b/frontend/src/locales/en-US.json
@@ -144,7 +144,6 @@
"cloud": "Cloud",
"deploy": "Self Host",
"external-url": "Bytebase has not configured --external-url",
- "ha-not-licensed": "Multiple replicas detected without HA license. Backend runners are disabled.",
"license-expired": "Your license for {plan} has expired on {expireAt}.",
"license-expires": "Your license for {plan} will expire in {days} days on {expireAt}.",
"request-demo": "Book a 30-minute product demo.",
diff --git a/frontend/src/locales/es-ES.json b/frontend/src/locales/es-ES.json
index 9ee4113efd..15be46c214 100644
--- a/frontend/src/locales/es-ES.json
+++ b/frontend/src/locales/es-ES.json
@@ -144,7 +144,6 @@
"cloud": "Cloud",
"deploy": "Auto Hospedaje",
"external-url": "Bytebase no ha configurado --external-url",
- "ha-not-licensed": "Se detectaron múltiples réplicas sin licencia HA. Los ejecutores del backend están deshabilitados.",
"license-expired": "Su licencia para {plan} ha expirado el {expireAt}.",
"license-expires": "Su licencia para {plan} ha caducado el {expireAt}.",
"request-demo": "Reserva una demo de producto de 30 minutos.",
diff --git a/frontend/src/locales/ja-JP.json b/frontend/src/locales/ja-JP.json
index 2b7bec8b9b..8a7967948b 100644
--- a/frontend/src/locales/ja-JP.json
+++ b/frontend/src/locales/ja-JP.json
@@ -144,7 +144,6 @@
"cloud": "クラウドサービス",
"deploy": "ローカル展開",
"external-url": "Bytebase は --extern-url を設定していません",
- "ha-not-licensed": "HAライセンスなしで複数のレプリカが検出されました。バックエンドランナーは無効です。",
"license-expired": "{plan} のライセンスは {expireAt} に期限切れになりました。",
"license-expires": "{plan} ライセンスは {expireAt} に期限切れになりました",
"request-demo": "30 分間の製品デモを予約してください。",
diff --git a/frontend/src/locales/vi-VN.json b/frontend/src/locales/vi-VN.json
index e3188a66f3..6477653331 100644
--- a/frontend/src/locales/vi-VN.json
+++ b/frontend/src/locales/vi-VN.json
@@ -144,7 +144,6 @@
"cloud": "Đám mây",
"deploy": "Tự lưu trữ",
"external-url": "Bytebase chưa được cấu hình --external-url",
- "ha-not-licensed": "Phát hiện nhiều bản sao mà không có giấy phép HA. Trình chạy backend đã bị tắt.",
"license-expired": "Giấy phép {plan} của bạn đã hết hạn vào {expireAt}.",
"license-expires": "Giấy phép {plan} của bạn sẽ hết hạn sau {days} ngày vào {expireAt}.",
"request-demo": "Đặt lịch giới thiệu sản phẩm 30 phút.",
diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.json
index 3b036c0d8b..ce7f00f169 100644
--- a/frontend/src/locales/zh-CN.json
+++ b/frontend/src/locales/zh-CN.json
@@ -144,7 +144,6 @@
"cloud": "云服务",
"deploy": "本地部署",
"external-url": "Bytebase 还没有配置 --external-url",
- "ha-not-licensed": "检测到多个副本但没有 HA 许可证。后端任务运行器已禁用。",
"license-expired": "您的{plan}证书已在 {expireAt} 过期",
"license-expires": "您的{plan}证书将在 {days} 天后({expireAt})过期",
"request-demo": "预约一个 30 分钟的产品演示。",
diff --git a/frontend/src/views/BannerHA.vue b/frontend/src/views/BannerHA.vue
deleted file mode 100644
index 93d77b94dd..0000000000
--- a/frontend/src/views/BannerHA.vue
+++ /dev/null
@@ -1,16 +0,0 @@
-<template>
- <div class="bg-warning flex items-center justify-center p-3">
- <TriangleAlertIcon
- class="h-6 w-6 text-white shrink-0"
- />
- <p class="ml-3 font-medium text-white truncate">
- <span>
- {{ $t("banner.ha-not-licensed") }}
- </span>
- </p>
- </div>
-</template>
-
-<script lang="ts" setup>
-import { TriangleAlertIcon } from "lucide-vue-next";
-</script>