Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(ha): fail task runs with error instead of frontend banner (#19912)
Danny Xue8449bbd5df1
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex 64b17dd5db..d5fa45779f 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -26,17 +26,24 @@ func (s *Scheduler) runPendingTaskRunsScheduler(ctx context.Context, wg *sync.Waselect {case <-ticker.C:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {- slog.Warn("Pending task runs scheduler skipped due to HA license restriction", log.BBError(err))+ if s.haFailSince.IsZero() {+ s.haFailSince = time.Now()+ }+ if time.Since(s.haFailSince) >= haFailGracePeriod {+ s.failTaskRunsForHA(ctx, err)+ }continue}+ s.haFailSince = time.Time{}if err := s.schedulePendingTaskRuns(ctx); err != nil {slog.Error("failed to schedule pending task runs", log.BBError(err))}case <-s.bus.TaskRunTickleChan:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {- slog.Warn("Pending task runs scheduler skipped due to HA license restriction", log.BBError(err))+ // Grace period is tracked by the ticker branch; just skip here.continue}+ s.haFailSince = time.Time{}if err := s.schedulePendingTaskRuns(ctx); err != nil {slog.Error("failed to schedule pending task runs", log.BBError(err))}diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex c921059382..491b772c9e 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -30,7 +30,7 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Waselect {case <-ticker.C:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {- slog.Warn("Running task runs scheduler skipped due to HA license restriction", log.BBError(err))+ // Pending scheduler handles failing task runs; just skip here.continue}if err := s.scheduleRunningTaskRuns(ctx); err != nil {@@ -38,7 +38,6 @@ func (s *Scheduler) runRunningTaskRunsScheduler(ctx context.Context, wg *sync.Wa}case <-s.bus.TaskRunTickleChan:if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {- slog.Warn("Running task runs scheduler skipped due to HA license restriction", log.BBError(err))continue}if err := s.scheduleRunningTaskRuns(ctx); err != nil {diff --git a/backend/runner/taskrun/scheduler.go b/backend/runner/taskrun/scheduler.goindex 048bf44b94..67d21899d4 100644--- a/backend/runner/taskrun/scheduler.go+++ b/backend/runner/taskrun/scheduler.go@@ -19,6 +19,11 @@ import (const (taskSchedulerInterval = 5 * time.Second+ // haFailGracePeriod is the duration to wait before failing task runs after+ // the HA license check starts failing. This avoids permanently failing task+ // runs during rolling restarts where stale heartbeats cause transient+ // over-counts.+ haFailGracePeriod = 10 * time.Minute)// Scheduler is the scheduler for task run.@@ -29,6 +34,9 @@ type Scheduler struct {licenseService *enterprise.LicenseServiceexecutorMap map[storepb.Task_Type]Executorprofile *config.Profile+ // haFailSince is when CheckReplicaLimit first started failing.+ // Zero means the check is currently passing.+ haFailSince time.Time}// NewScheduler will create a new scheduler.@@ -60,6 +68,79 @@ func (s *Scheduler) Register(taskType storepb.Task_Type, executorGetter Executors.executorMap[taskType] = executorGetter}+// failTaskRunsForHA fails all PENDING and AVAILABLE task runs because HA is not licensed.+func (s *Scheduler) failTaskRunsForHA(ctx context.Context, haErr error) {+ taskRuns, err := s.store.ListTaskRunsByStatus(ctx, []storepb.TaskRun_Status{storepb.TaskRun_PENDING, storepb.TaskRun_AVAILABLE})+ if err != nil {+ slog.Error("failed to list task runs for HA limit check", log.BBError(err))+ return+ }+ if len(taskRuns) == 0 {+ return+ }++ // Track affected plans to send failure webhooks.+ // Key by (projectID, planID) to match ClaimPipelineFailureNotification's dedup key.+ type planKey struct {+ projectID string+ planID int64+ }+ affectedPlans := map[planKey]string{} // value = first environment seen++ for _, taskRun := range taskRuns {+ if _, err := s.store.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{+ ID: taskRun.ID,+ ProjectID: taskRun.ProjectID,+ Status: storepb.TaskRun_FAILED,+ ResultProto: &storepb.TaskRunResult{+ Detail: haErr.Error(),+ },+ }); err != nil {+ slog.Error("failed to fail task run for HA limit",+ slog.Int64("taskRunID", taskRun.ID),+ log.BBError(err),+ )+ continue+ }+ key := planKey{projectID: taskRun.ProjectID, planID: taskRun.PlanUID}+ if _, ok := affectedPlans[key]; !ok {+ affectedPlans[key] = taskRun.Environment+ }+ }++ slog.Warn("Failed task runs due to HA license restriction", slog.Int64("count", int64(len(taskRuns))), log.BBError(haErr))++ // Send PIPELINE_FAILED webhook for each affected plan.+ for key, environment := range affectedPlans {+ claimed, err := s.store.ClaimPipelineFailureNotification(ctx, key.projectID, key.planID)+ if err != nil {+ slog.Error("failed to claim pipeline failure notification", log.BBError(err))+ continue+ }+ if !claimed {+ continue+ }+ plan, err := s.store.GetPlan(ctx, &store.FindPlanMessage{ProjectID: key.projectID, UID: &key.planID})+ if err != nil || plan == nil {+ slog.Error("failed to get plan for HA failure webhook", log.BBError(err))+ continue+ }+ project, err := s.store.GetProjectByResourceID(ctx, plan.ProjectID)+ if err != nil || project == nil {+ slog.Error("failed to get project for HA failure webhook", log.BBError(err))+ continue+ }+ s.webhookManager.CreateEvent(ctx, &webhook.Event{+ Type: storepb.Activity_PIPELINE_FAILED,+ Project: webhook.NewProject(project),+ RolloutFailed: &webhook.EventRolloutFailed{+ Rollout: webhook.NewRollout(plan),+ Environment: environment,+ },+ })+ }+}+// Run will start the scheduler.func (s *Scheduler) Run(ctx context.Context, wg *sync.WaitGroup) {defer wg.Done()diff --git a/frontend/src/components/BannersWrapper.vue b/frontend/src/components/BannersWrapper.vueindex dd588dc9b5..0ac57901cc 100644--- a/frontend/src/components/BannersWrapper.vue+++ b/frontend/src/components/BannersWrapper.vue@@ -3,7 +3,6 @@<BannerDemo v-if="shouldShowDemoBanner" /><BannerSubscription v-if="shouldShowSubscriptionBanner" /><BannerExternalUrl v-if="shouldShowExternalUrlBanner" />- <BannerHA v-if="shouldShowHABanner" /><BannerAnnouncement /></template>@@ -20,15 +19,14 @@ import { isDev } from "@/utils";import BannerAnnouncement from "@/views/BannerAnnouncement.vue";import BannerDemo from "@/views/BannerDemo.vue";import BannerExternalUrl from "@/views/BannerExternalUrl.vue";-import BannerHA from "@/views/BannerHA.vue";import BannerSubscription from "@/views/BannerSubscription.vue";import BannerUpgradeSubscription from "@/views/BannerUpgradeSubscription.vue";const actuatorStore = useActuatorV1Store();const subscriptionStore = useSubscriptionV1Store();-const { needConfigureExternalUrl, replicaCount } = storeToRefs(actuatorStore);-const { isExpired, isTrialing, currentPlan, daysBeforeExpire, isHAAllowed } =+const { needConfigureExternalUrl } = storeToRefs(actuatorStore);+const { isExpired, isTrialing, currentPlan, daysBeforeExpire } =storeToRefs(subscriptionStore);const shouldShowDemoBanner = computed(() => {@@ -48,8 +46,4 @@ const shouldShowSubscriptionBanner = computed(() => {const shouldShowExternalUrlBanner = computed(() => {return !isDev() && needConfigureExternalUrl.value;});--const shouldShowHABanner = computed(() => {- return !isHAAllowed.value && replicaCount.value > 1;-});</script>diff --git a/frontend/src/locales/en-US.json b/frontend/src/locales/en-US.jsonindex 2aca417549..76bf68518b 100644--- a/frontend/src/locales/en-US.json+++ b/frontend/src/locales/en-US.json@@ -144,7 +144,6 @@"cloud": "Cloud","deploy": "Self Host","external-url": "Bytebase has not configured --external-url",- "ha-not-licensed": "Multiple replicas detected without HA license. Backend runners are disabled.","license-expired": "Your license for {plan} has expired on {expireAt}.","license-expires": "Your license for {plan} will expire in {days} days on {expireAt}.","request-demo": "Book a 30-minute product demo.",diff --git a/frontend/src/locales/es-ES.json b/frontend/src/locales/es-ES.jsonindex 9ee4113efd..15be46c214 100644--- a/frontend/src/locales/es-ES.json+++ b/frontend/src/locales/es-ES.json@@ -144,7 +144,6 @@"cloud": "Cloud","deploy": "Auto Hospedaje","external-url": "Bytebase no ha configurado --external-url",- "ha-not-licensed": "Se detectaron múltiples réplicas sin licencia HA. Los ejecutores del backend están deshabilitados.","license-expired": "Su licencia para {plan} ha expirado el {expireAt}.","license-expires": "Su licencia para {plan} ha caducado el {expireAt}.","request-demo": "Reserva una demo de producto de 30 minutos.",diff --git a/frontend/src/locales/ja-JP.json b/frontend/src/locales/ja-JP.jsonindex 2b7bec8b9b..8a7967948b 100644--- a/frontend/src/locales/ja-JP.json+++ b/frontend/src/locales/ja-JP.json@@ -144,7 +144,6 @@"cloud": "クラウドサービス","deploy": "ローカル展開","external-url": "Bytebase は --extern-url を設定していません",- "ha-not-licensed": "HAライセンスなしで複数のレプリカが検出されました。バックエンドランナーは無効です。","license-expired": "{plan} のライセンスは {expireAt} に期限切れになりました。","license-expires": "{plan} ライセンスは {expireAt} に期限切れになりました","request-demo": "30 分間の製品デモを予約してください。",diff --git a/frontend/src/locales/vi-VN.json b/frontend/src/locales/vi-VN.jsonindex e3188a66f3..6477653331 100644--- a/frontend/src/locales/vi-VN.json+++ b/frontend/src/locales/vi-VN.json@@ -144,7 +144,6 @@"cloud": "Đám mây","deploy": "Tự lưu trữ","external-url": "Bytebase chưa được cấu hình --external-url",- "ha-not-licensed": "Phát hiện nhiều bản sao mà không có giấy phép HA. Trình chạy backend đã bị tắt.","license-expired": "Giấy phép {plan} của bạn đã hết hạn vào {expireAt}.","license-expires": "Giấy phép {plan} của bạn sẽ hết hạn sau {days} ngày vào {expireAt}.","request-demo": "Đặt lịch giới thiệu sản phẩm 30 phút.",diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.jsonindex 3b036c0d8b..ce7f00f169 100644--- a/frontend/src/locales/zh-CN.json+++ b/frontend/src/locales/zh-CN.json@@ -144,7 +144,6 @@"cloud": "云服务","deploy": "本地部署","external-url": "Bytebase 还没有配置 --external-url",- "ha-not-licensed": "检测到多个副本但没有 HA 许可证。后端任务运行器已禁用。","license-expired": "您的{plan}证书已在 {expireAt} 过期","license-expires": "您的{plan}证书将在 {days} 天后({expireAt})过期","request-demo": "预约一个 30 分钟的产品演示。",diff --git a/frontend/src/views/BannerHA.vue b/frontend/src/views/BannerHA.vuedeleted file mode 100644index 93d77b94dd..0000000000--- a/frontend/src/views/BannerHA.vue+++ /dev/null@@ -1,16 +0,0 @@-<template>- <div class="bg-warning flex items-center justify-center p-3">- <TriangleAlertIcon- class="h-6 w-6 text-white shrink-0"- />- <p class="ml-3 font-medium text-white truncate">- <span>- {{ $t("banner.ha-not-licensed") }}- </span>- </p>- </div>-</template>--<script lang="ts" setup>-import { TriangleAlertIcon } from "lucide-vue-next";-</script>