Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

23.6 KB
Raw Blame
// Package schemasync is a runner that synchronize database schemas.
package schemasync

import (
	"bytes"
	"context"
	"database/sql"
	"fmt"
	"log/slog"
	"slices"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	metadatapb "github.com/bytebase/omni/metadata"
	"github.com/pkg/errors"
	"github.com/sourcegraph/conc/pool"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/bytebase/bytebase/backend/common"
	"github.com/bytebase/bytebase/backend/common/log"
	"github.com/bytebase/bytebase/backend/component/dbfactory"
	"github.com/bytebase/bytebase/backend/component/productmetrics"
	"github.com/bytebase/bytebase/backend/enterprise"
	storepb "github.com/bytebase/bytebase/backend/generated-go/store"
	"github.com/bytebase/bytebase/backend/plugin/db"
	"github.com/bytebase/bytebase/backend/store"
)

const (
	instanceSyncInterval        = 15 * time.Minute
	databaseSyncCheckerInterval = 10 * time.Second
	syncTimeout                 = 15 * time.Minute
	// defaultSyncInterval means never sync.
	defaultSyncInterval = 0 * time.Second
	MaximumOutstanding  = 100
)

// NewSyncer creates a schema syncer.
func NewSyncer(stores *store.Store, dbFactory *dbfactory.DBFactory, licenseService *enterprise.LicenseService, productMetrics *productmetrics.ProductMetrics) *Syncer {
	return &Syncer{
		store:          stores,
		dbFactory:      dbFactory,
		licenseService: licenseService,
		productMetrics: productMetrics,
	}
}

// Syncer is the schema syncer.
type Syncer struct {
	sync.Mutex

	store           *store.Store
	dbFactory       *dbfactory.DBFactory
	licenseService  *enterprise.LicenseService
	productMetrics  *productmetrics.ProductMetrics
	databaseSyncMap sync.Map // map[string]*store.DatabaseMessage
}

// Run will run the schema syncer once.
func (s *Syncer) Run(ctx context.Context, wg *sync.WaitGroup) {
	defer wg.Done()

	sp := pool.New()
	sp.Go(func() {
		slog.Debug(fmt.Sprintf("Schema syncer started and will run every %v", instanceSyncInterval))
		ticker := time.NewTicker(instanceSyncInterval)
		defer ticker.Stop()

		for {
			select {
			case <-ticker.C:
				if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
					slog.Warn("Schema syncer skipped due to HA license restriction", log.BBError(err))
					continue
				}
				s.trySyncAll(ctx)
			case <-ctx.Done(): // if cancel() execute
				return
			}
		}
	})

	sp.Go(func() {
		ticker := time.NewTicker(databaseSyncCheckerInterval)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
				if err := s.licenseService.CheckReplicaLimit(ctx); err != nil {
					slog.Warn("Database sync checker skipped due to HA license restriction", log.BBError(err))
					continue
				}
				if err := s.syncQueuedDatabases(ctx); err != nil {
					slog.Error("Failed to run database sync cycle", log.BBError(err))
					continue
				}
			case <-ctx.Done(): // if cancel() execute
				return
			}
		}
	})
	sp.Wait()
}

func (s *Syncer) trySyncAll(ctx context.Context) {
	startedAt := time.Now()
	result := productmetrics.ResultFailure
	defer func() {
		if r := recover(); r != nil {
			err, ok := r.(error)
			if !ok {
				err = errors.Errorf("%v", r)
			}
			slog.Error("Instance syncer PANIC RECOVER", log.BBError(err), log.BBStack("panic-stack"))
		}
		if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
			s.productMetrics.RecordRunnerRun(productmetrics.RunnerInstanceSync, result, time.Since(startedAt))
		}
	}()

	tx, err := s.store.GetDB().BeginTx(ctx, nil)
	if err != nil {
		slog.Error("Failed to begin schema syncer transaction", log.BBError(err))
		return
	}
	defer func() {
		if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) {
			result = productmetrics.ResultFailure
			slog.Error("Failed to rollback schema syncer transaction", log.BBError(err))
		}
	}()

	if _, err := tx.ExecContext(ctx, "SET LOCAL idle_in_transaction_session_timeout = 0"); err != nil {
		slog.Error("Failed to configure schema syncer transaction", log.BBError(err))
		return
	}

	acquired, err := store.TryAdvisoryXactLock(ctx, tx, store.AdvisoryLockKeySchemaSyncer)
	if err != nil {
		slog.Error("Failed to acquire schema syncer advisory lock", log.BBError(err))
		return
	}
	if !acquired {
		slog.Debug("Schema syncer advisory lock held by another replica, skipping")
		result = productmetrics.ResultSkipped
		return
	}

	wp := pool.New().WithMaxGoroutines(MaximumOutstanding)
	var syncFailed atomic.Bool
	instances, err := s.store.ListAllInstances(ctx, false)
	if err != nil {
		slog.Error("Failed to retrieve instances", log.BBError(err))
		return
	}
	now := time.Now()
	for _, instance := range instances {
		instance := instance
		canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
		if err != nil {
			syncFailed.Store(true)
			slog.Error("Failed to determine whether instance sync can be scheduled",
				slog.String("instance", instance.ResourceID),
				log.BBError(err))
			continue
		}
		if !canSchedule {
			continue
		}
		interval := s.getOrDefaultSyncInterval(ctx, instance)
		if interval == defaultSyncInterval {
			continue
		}
		lastSyncTime := getOrDefaultLastSyncTime(instance.Metadata.LastSyncTime)
		// lastSyncTime + syncInterval > now
		// Next round not started yet.
		nextSyncTime := lastSyncTime.Add(interval)
		if now.Before(nextSyncTime) {
			continue
		}

		wp.Go(func() {
			slog.Debug("Sync instance schema", slog.String("instance", instance.ResourceID))
			if _, _, _, err := s.SyncInstance(ctx, instance); err != nil {
				syncFailed.Store(true)
				slog.Debug("Failed to sync instance",
					slog.String("instance", instance.ResourceID),
					slog.String("error", err.Error()))
			}
		})
	}
	wp.Wait()

	instancesMap := map[string]*store.InstanceMessage{}
	for _, instance := range instances {
		instancesMap[instance.ResourceID] = instance
	}

	databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{})
	if err != nil {
		slog.Error("Failed to retrieve databases", log.BBError(err))
		return
	}
	for _, database := range databases {
		database := database
		if database.Deleted {
			continue
		}
		instance, ok := instancesMap[database.InstanceID]
		if !ok {
			continue
		}
		canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
		if err != nil {
			syncFailed.Store(true)
			slog.Error("Failed to determine whether database sync can be scheduled",
				slog.String("instance", database.InstanceID),
				slog.String("database", database.DatabaseName),
				log.BBError(err))
			continue
		}
		if !canSchedule {
			continue
		}
		// The database inherits the sync interval from the instance.
		interval := s.getOrDefaultSyncInterval(ctx, instance)
		if interval == defaultSyncInterval {
			continue
		}
		lastSyncTime := getOrDefaultLastSyncTime(database.Metadata.LastSyncTime)
		// lastSyncTime + syncInterval > now
		// Next round not started yet.
		nextSyncTime := lastSyncTime.Add(interval)
		if now.Before(nextSyncTime) {
			continue
		}

		s.databaseSyncMap.Store(database.String(), database)
	}
	if !syncFailed.Load() {
		result = productmetrics.ResultSuccess
	}
}

func (s *Syncer) syncQueuedDatabases(ctx context.Context) (retErr error) {
	startedAt := time.Now()
	result := productmetrics.ResultFailure
	defer func() {
		if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
			s.productMetrics.RecordRunnerRun(productmetrics.RunnerDatabaseSync, result, time.Since(startedAt))
		}
	}()
	defer func() {
		if r := recover(); r != nil {
			panicErr, ok := r.(error)
			if !ok {
				panicErr = errors.Errorf("%v", r)
			}
			retErr = errors.Wrap(panicErr, "database sync checker panic")
			slog.Error("Database sync checker PANIC RECOVER", log.BBError(retErr), log.BBStack("panic-stack"))
		}
	}()

	instances, err := s.store.ListAllInstances(ctx, false)
	if err != nil {
		return errors.Wrap(err, "failed to list instances")
	}
	instanceMap := make(map[string]*store.InstanceMessage)
	for _, instance := range instances {
		instanceMap[instance.ResourceID] = instance
	}
	dbwp := pool.New().WithMaxGoroutines(MaximumOutstanding)
	var syncFailed atomic.Bool
	s.databaseSyncMap.Range(func(key, value any) bool {
		database, ok := value.(*store.DatabaseMessage)
		if !ok {
			return true
		}

		instance, ok := instanceMap[database.InstanceID]
		if !ok {
			s.databaseSyncMap.Delete(key)
			return true
		}
		canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
		if err != nil {
			syncFailed.Store(true)
			slog.Error("Failed to determine whether queued database sync can be scheduled",
				slog.String("instance", database.InstanceID),
				slog.String("database", database.DatabaseName),
				log.BBError(err))
			return true
		}
		s.databaseSyncMap.Delete(key)
		if !canSchedule {
			return true
		}
		dbwp.Go(func() {
			slog.Debug("Sync database schema", slog.String("instance", database.InstanceID), slog.String("database", database.DatabaseName))
			if err := s.SyncDatabaseSchema(ctx, database); err != nil {
				syncFailed.Store(true)
				slog.Debug("Failed to sync database schema",
					slog.String("instance", database.InstanceID),
					slog.String("databaseName", database.DatabaseName),
					log.BBError(err))
				// Save sync error to database metadata
				if _, updateErr := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{
					InstanceID:   database.InstanceID,
					DatabaseName: database.DatabaseName,
					MetadataUpdates: []func(*storepb.DatabaseMetadata){
						func(md *storepb.DatabaseMetadata) {
							md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_FAILED
							md.SyncError = err.Error()
							md.LastSyncTime = timestamppb.Now()
						},
					},
				}); updateErr != nil {
					syncFailed.Store(true)
					slog.Error("Failed to update database sync error",
						slog.String("instance", database.InstanceID),
						slog.String("database", database.DatabaseName),
						log.BBError(updateErr))
				}
			}
		})
		return true
	})
	dbwp.Wait()

	if syncFailed.Load() {
		return errors.New("one or more queued database synchronizations failed")
	}
	result = productmetrics.ResultSuccess
	return nil
}

func (s *Syncer) canScheduleInstanceSync(ctx context.Context, instance *store.InstanceMessage) (bool, error) {
	if instance.ProjectID == nil {
		return true, nil
	}
	return s.isProjectActive(ctx, *instance.ProjectID)
}

func (s *Syncer) canScheduleDatabaseSync(ctx context.Context, instance *store.InstanceMessage, database *store.DatabaseMessage) (bool, error) {
	if database.Deleted {
		return false, nil
	}
	canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
	if err != nil || !canSchedule {
		return canSchedule, err
	}
	return s.isProjectActive(ctx, database.ProjectID)
}

func (s *Syncer) isProjectActive(ctx context.Context, projectID string) (bool, error) {
	project, err := s.store.GetProjectByResourceID(ctx, projectID)
	if err != nil {
		return false, errors.Wrapf(err, "failed to get project %q for schema sync", projectID)
	}
	if project == nil {
		slog.Warn("project not found for schema sync", slog.String("project", projectID))
		return false, nil
	}
	return !project.Deleted, nil
}

func (s *Syncer) SyncAllDatabases(ctx context.Context, instance *store.InstanceMessage) {
	find := &store.FindDatabaseMessage{}
	if instance != nil {
		canSchedule, err := s.canScheduleInstanceSync(ctx, instance)
		if err != nil {
			slog.Error("Failed to determine whether instance sync can be scheduled",
				slog.String("instance", instance.ResourceID),
				log.BBError(err))
			return
		}
		if !canSchedule {
			return
		}
		find.InstanceID = &instance.ResourceID
	}
	databases, err := s.store.ListDatabases(ctx, find)
	if err != nil {
		slog.Debug("Failed to find databases to sync",
			slog.String("error", err.Error()))
		return
	}

	for _, database := range databases {
		if database.Deleted {
			continue
		}
		if instance != nil {
			canSchedule, err := s.canScheduleDatabaseSync(ctx, instance, database)
			if err != nil {
				slog.Error("Failed to determine whether database sync can be scheduled",
					slog.String("instance", database.InstanceID),
					slog.String("database", database.DatabaseName),
					log.BBError(err))
				continue
			}
			if !canSchedule {
				continue
			}
		}
		s.databaseSyncMap.Store(database.String(), database)
	}
}

func (s *Syncer) SyncDatabaseAsync(database *store.DatabaseMessage) {
	if database == nil || database.Deleted {
		return
	}
	s.databaseSyncMap.Store(database.String(), database)
}

func (s *Syncer) SyncDatabasesAsync(databases []*store.DatabaseMessage) {
	for _, database := range databases {
		s.SyncDatabaseAsync(database)
	}
}

// GetInstanceMeta gets the instance metadata.
func (s *Syncer) GetInstanceMeta(ctx context.Context, instance *store.InstanceMessage) (*db.InstanceMetadata, error) {
	driver, err := s.dbFactory.GetAdminDatabaseDriver(ctx, instance, nil /* database */, db.ConnectionContext{})
	if err != nil {
		return nil, err
	}
	defer driver.Close(ctx)

	deadlineCtx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(syncTimeout))
	defer cancelFunc()
	instanceMeta, err := driver.SyncInstance(deadlineCtx)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to sync instance: %s", instance.ResourceID)
	}

	if instanceMeta.Metadata == nil {
		instanceMeta.Metadata = &storepb.Instance{}
	}

	instanceMeta.Metadata.LastSyncTime = timestamppb.Now()

	return instanceMeta, nil
}

// SyncInstance syncs the schema for all databases in an instance.
func (s *Syncer) SyncInstance(ctx context.Context, instance *store.InstanceMessage) (updatedInstance *store.InstanceMessage, allDatabases []*metadatapb.DatabaseSchemaMetadata, newDatabases []*store.DatabaseMessage, retErr error) {
	startedAt := time.Now()
	defer func() {
		panicValue := recover()
		if panicValue != nil {
			retErr = errorFromPanic(panicValue, "instance sync panic")
		}
		if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
			s.productMetrics.RecordInstanceSync(instance, time.Since(startedAt), retErr)
		}
		if panicValue != nil {
			panic(panicValue)
		}
	}()
	instanceMeta, err := s.GetInstanceMeta(ctx, instance)
	if err != nil {
		return nil, nil, nil, err
	}
	metadata := proto.CloneOf(instance.Metadata)
	metadata.LastSyncTime = instanceMeta.Metadata.LastSyncTime
	metadata.MysqlLowerCaseTableNames = instanceMeta.Metadata.MysqlLowerCaseTableNames
	metadata.Roles = instanceMeta.Metadata.Roles

	updateInstance := &store.UpdateInstanceMessage{
		ResourceID: &instance.ResourceID,
		Workspace:  instance.Workspace,
		Metadata:   metadata,
	}
	if instanceMeta.Version != instance.Metadata.GetVersion() {
		metadata.Version = instanceMeta.Version
	}
	updatedInstance, err = s.store.UpdateInstance(ctx, updateInstance)
	if err != nil {
		return nil, nil, nil, err
	}
	databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{InstanceID: &instance.ResourceID})
	if err != nil {
		return nil, nil, nil, errors.Wrapf(err, "failed to sync database for instance: %s. Failed to find database list", instance.ResourceID)
	}
	var filteredDatabaseMetadatas []*metadatapb.DatabaseSchemaMetadata
	var databaseProjectID string
	if instance.ProjectID != nil {
		databaseProjectID = *instance.ProjectID
	} else {
		databaseProjectID, err = s.store.GetDefaultProjectID(ctx, instance.Workspace)
		if err != nil {
			return nil, nil, nil, errors.Wrapf(err, "failed to get default project ID for instance %q", instance.ResourceID)
		}
	}

	for _, databaseMetadata := range instanceMeta.Databases {
		if instance.Metadata.SyncDatabases != nil && !slices.Contains(instance.Metadata.SyncDatabases.Databases, databaseMetadata.Name) {
			continue
		}
		filteredDatabaseMetadatas = append(filteredDatabaseMetadatas, databaseMetadata)
		idx := slices.IndexFunc(databases, func(db *store.DatabaseMessage) bool { return db.DatabaseName == databaseMetadata.Name })

		if idx < 0 {
			newDatabase, err := s.store.CreateDatabaseDefault(ctx, &store.DatabaseMessage{
				InstanceID:   instance.ResourceID,
				DatabaseName: databaseMetadata.Name,
				ProjectID:    databaseProjectID,
			})
			if err != nil {
				return nil, nil, nil, errors.Wrapf(err, "failed to create instance %q database %q in sync runner", instance.ResourceID, databaseMetadata.Name)
			}
			if newDatabase != nil {
				newDatabases = append(newDatabases, newDatabase)
			}
		}
	}

	for _, database := range databases {
		idx := slices.IndexFunc(filteredDatabaseMetadatas, func(db *metadatapb.DatabaseSchemaMetadata) bool { return db.Name == database.DatabaseName })
		if idx < 0 {
			if _, err := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{
				InstanceID:   instance.ResourceID,
				DatabaseName: database.DatabaseName,
				Deleted:      new(true),
			}); err != nil {
				return nil, nil, nil, errors.Errorf("failed to update database %q for instance %q", database.DatabaseName, instance.ResourceID)
			}
		}
	}

	return updatedInstance, instanceMeta.Databases, newDatabases, nil
}

// doSyncDatabaseSchema is the core implementation that syncs the schema for a database and optionally creates a sync history record.
func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.DatabaseMessage, createSyncHistory bool) (syncHistoryResourceID string, retErr error) {
	if database == nil {
		return "", errors.New("cannot sync nil database")
	}
	defer func() {
		panicValue := recover()
		if panicValue != nil {
			retErr = errorFromPanic(panicValue, "database schema sync panic")
		}
		if !errors.Is(ctx.Err(), context.Canceled) && s.productMetrics != nil {
			s.productMetrics.RecordDatabaseSync(database, retErr)
		}
		if panicValue != nil {
			panic(panicValue)
		}
	}()
	instance, err := s.store.GetInstanceByResourceID(ctx, database.InstanceID)
	if err != nil {
		return "", errors.Wrapf(err, "failed to get instance %q", database.InstanceID)
	}
	if instance == nil {
		return "", errors.Errorf("instance %q not found", database.InstanceID)
	}
	driver, err := s.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{})
	if err != nil {
		return "", err
	}
	defer driver.Close(ctx)
	// Sync database schema
	deadlineCtx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(syncTimeout))
	defer cancelFunc()
	// Everything below describes the database as of this instant, so it is what
	// orders this sync against the other syncs writing the same projection.
	readAt := time.Now()
	syncedDatabaseMetadata, err := driver.SyncDBSchema(deadlineCtx)
	if err != nil {
		return "", errors.Wrapf(err, "failed to sync database schema for database %q", database.DatabaseName)
	}
	var schemaBuf bytes.Buffer
	if err := driver.Dump(deadlineCtx, &schemaBuf, syncedDatabaseMetadata); err != nil {
		return "", errors.Wrapf(err, "failed to dump database schema for database %q", database.DatabaseName)
	}
	rawDump := schemaBuf.Bytes()

	// Resolve store reads before UpdateDatabase opens its write transaction.
	// Acquiring another pool connection inside the callback can deadlock sync bursts.
	backupAvailable := s.databaseBackupAvailable(ctx, instance, syncedDatabaseMetadata)

	stored, err := s.store.UpsertDBSchema(ctx,
		database.InstanceID, database.DatabaseName,
		syncedDatabaseMetadata, rawDump, readAt,
	)
	if err != nil {
		if strings.Contains(err.Error(), "escape sequence") || strings.Contains(err.Error(), "invalid byte sequence") {
			if metadataBytes, err := protojson.Marshal(syncedDatabaseMetadata); err == nil {
				slog.Error("unsupported Unicode escape sequence", slog.String("metadata", string(metadataBytes)), slog.String("raw_dump", string(rawDump)))
			}
		}
		return "", errors.Wrapf(err, "failed to upsert database schema for database %q", database.DatabaseName)
	}

	// These values come from the same read, so leave them to the newer sync that won
	// rather than reporting this read as the database's current state. LastSyncTime
	// stays a completion time, because the sync interval is measured from it; the
	// dump's own freshness is db_schema.synced_at.
	if stored {
		if _, err := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{
			InstanceID:   database.InstanceID,
			DatabaseName: database.DatabaseName,
			Deleted:      new(false),
			MetadataUpdates: []func(*storepb.DatabaseMetadata){
				func(md *storepb.DatabaseMetadata) {
					md.LastSyncTime = timestamppb.Now()
					md.BackupAvailable = backupAvailable
					md.Datashare = syncedDatabaseMetadata.Datashare
					md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_OK
					md.SyncError = ""
				},
			},
		}); err != nil {
			return "", errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)
		}
	} else {
		slog.Debug("Skipped stale database schema sync",
			slog.String("instance", database.InstanceID),
			slog.String("database", database.DatabaseName),
			slog.Time("readAt", readAt))
	}

	// Create sync history if requested
	if createSyncHistory {
		resourceID, err := s.store.CreateSyncHistory(ctx, database.InstanceID, database.DatabaseName, syncedDatabaseMetadata, string(rawDump))
		if err != nil {
			if strings.Contains(err.Error(), "escape sequence") {
				if metadataBytes, err := protojson.Marshal(syncedDatabaseMetadata); err == nil {
					slog.Error("unsupported Unicode escape sequence", slog.String("metadata", string(metadataBytes)), slog.String("raw_dump", string(rawDump)))
				}
			}
			return "", errors.Wrapf(err, "failed to insert sync history for database %q", database.DatabaseName)
		}
		return resourceID, nil
	}

	return "", nil
}

func errorFromPanic(value any, message string) error {
	err, ok := value.(error)
	if !ok {
		err = errors.Errorf("%v", value)
	}
	return errors.Wrap(err, message)
}

// SyncDatabaseSchemaToHistory will sync the schema for a database and create a sync history record.
func (s *Syncer) SyncDatabaseSchemaToHistory(ctx context.Context, database *store.DatabaseMessage) (string, error) {
	return s.doSyncDatabaseSchema(ctx, database, true)
}

// SyncDatabaseSchema will sync the schema for a database.
func (s *Syncer) SyncDatabaseSchema(ctx context.Context, database *store.DatabaseMessage) error {
	_, err := s.doSyncDatabaseSchema(ctx, database, false)
	return err
}

func (s *Syncer) databaseBackupAvailable(ctx context.Context, instance *store.InstanceMessage, dbMetadata *metadatapb.DatabaseSchemaMetadata) bool {
	if !common.EngineSupportPriorBackup(instance.Metadata.GetEngine()) {
		return false
	}
	switch instance.Metadata.GetEngine() {
	case storepb.Engine_POSTGRES:
		if dbMetadata == nil {
			return false
		}
		for _, schema := range dbMetadata.Schemas {
			if schema.GetName() == common.BackupDatabaseNameOfEngine(storepb.Engine_POSTGRES) {
				return true
			}
		}
	case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_MSSQL, storepb.Engine_TIDB, storepb.Engine_ORACLE:
		dbName := common.BackupDatabaseNameOfEngine(instance.Metadata.GetEngine())
		backupDB, err := s.store.GetDatabase(ctx, &store.FindDatabaseMessage{
			InstanceID:   &instance.ResourceID,
			DatabaseName: &dbName,
		})
		if err != nil {
			slog.Debug("Failed to get backup database", "err", err)
			return false
		}
		return backupDB != nil
	default:
		// Unsupported database engine for backup
		slog.Debug("Unsupported database engine for backup", "engine", instance.Metadata.GetEngine())
		return false
	}
	return false
}

func (s *Syncer) getOrDefaultSyncInterval(ctx context.Context, instance *store.InstanceMessage) time.Duration {
	activated := instance.Metadata.GetActivation()
	if s.licenseService != nil {
		activated = s.licenseService.IsInstanceEffectivelyActivated(ctx, instance.Workspace, instance)
	}
	if !activated {
		return defaultSyncInterval
	}
	if !instance.Metadata.GetSyncInterval().IsValid() {
		return defaultSyncInterval
	}
	if instance.Metadata.GetSyncInterval().GetSeconds() == 0 && instance.Metadata.GetSyncInterval().GetNanos() == 0 {
		return defaultSyncInterval
	}
	return instance.Metadata.GetSyncInterval().AsDuration()
}

func getOrDefaultLastSyncTime(t *timestamppb.Timestamp) time.Time {
	if t.IsValid() {
		return t.AsTime()
	}
	return time.Unix(0, 0)
}