Skip to content

PostgreSQL backup tool with Point-In-Time-Recovery and restore verification

!REFACTOR (tasks): Move tasks cancellation and tracking to separate package from backuping to use for restores

Rostislav Dugina1adef2261e7
diff --git a/backend/.env.development.example b/backend/.env.development.example
index 41dba6a8..8d96aee8 100644
--- a/backend/.env.development.example
+++ b/backend/.env.development.example
@@ -2,8 +2,10 @@
DEV_DB_NAME=databasus
DEV_DB_USERNAME=postgres
DEV_DB_PASSWORD=Q1234567
-#app
+# app
ENV_MODE=development
+# logging
+SHOW_DB_INSTALLATION_VERIFICATION_LOGS=true
# db
DATABASE_DSN=host=dev-db user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable
DATABASE_URL=postgres://postgres:Q1234567@dev-db:5437/databasus?sslmode=disable
diff --git a/backend/cmd/main.go b/backend/cmd/main.go
index eda1dbb6..bdc3ac1b 100644
--- a/backend/cmd/main.go
+++ b/backend/cmd/main.go
@@ -16,7 +16,6 @@ import (
"databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups"
"databasus-backend/internal/features/backups/backups/backuping"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_download "databasus-backend/internal/features/backups/backups/download"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
@@ -28,6 +27,8 @@ import (
"databasus-backend/internal/features/restores"
"databasus-backend/internal/features/storages"
system_healthcheck "databasus-backend/internal/features/system/healthcheck"
+ task_cancellation "databasus-backend/internal/features/tasks/cancellation"
+ task_registry "databasus-backend/internal/features/tasks/registry"
users_controllers "databasus-backend/internal/features/users/controllers"
users_middleware "databasus-backend/internal/features/users/middleware"
users_services "databasus-backend/internal/features/users/services"
@@ -59,6 +60,8 @@ func main() {
cache_utils.TestCacheConnection()
if config.GetEnv().IsPrimaryNode {
+ log.Info("Clearing cache...")
+
err := cache_utils.ClearAllCache()
if err != nil {
log.Error("Failed to clear cache", "error", err)
@@ -239,7 +242,7 @@ func setUpDependencies() {
notifiers.SetupDependencies()
storages.SetupDependencies()
backups_config.SetupDependencies()
- backups_cancellation.SetupDependencies()
+ task_cancellation.SetupDependencies()
}
func runBackgroundTasks(log *slog.Logger) {
@@ -257,14 +260,14 @@ func runBackgroundTasks(log *slog.Logger) {
cancel()
}()
+ err := files_utils.CleanFolder(config.GetEnv().TempFolder)
+ if err != nil {
+ log.Error("Failed to clean temp folder", "error", err)
+ }
+
if config.GetEnv().IsPrimaryNode {
log.Info("Starting primary node background tasks...")
- err := files_utils.CleanFolder(config.GetEnv().TempFolder)
- if err != nil {
- log.Error("Failed to clean temp folder", "error", err)
- }
-
go runWithPanicLogging(log, "backup background service", func() {
backuping.GetBackupsScheduler().Run(ctx)
})
@@ -284,6 +287,10 @@ func runBackgroundTasks(log *slog.Logger) {
go runWithPanicLogging(log, "download token cleanup background service", func() {
backups_download.GetDownloadTokenBackgroundService().Run(ctx)
})
+
+ go runWithPanicLogging(log, "task nodes registry background service", func() {
+ task_registry.GetTaskNodesRegistry().Run(ctx)
+ })
} else {
log.Info("Skipping primary node tasks as not primary node")
}
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index df4da549..5b4aa14a 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -30,6 +30,8 @@ type EnvVariables struct {
MariadbInstallDir string `env:"MARIADB_INSTALL_DIR"`
MongodbInstallDir string `env:"MONGODB_INSTALL_DIR"`
+ ShowDbInstallationVerificationLogs bool `env:"SHOW_DB_INSTALLATION_VERIFICATION_LOGS"`
+
NodeID string
IsManyNodesMode bool `env:"IS_MANY_NODES_MODE"`
IsPrimaryNode bool `env:"IS_PRIMARY_NODE"`
@@ -169,6 +171,11 @@ func loadEnvVariables() {
os.Exit(1)
}
+ // Set default value for ShowDbInstallationVerificationLogs if not defined
+ if os.Getenv("SHOW_DB_INSTALLATION_VERIFICATION_LOGS") == "" {
+ env.ShowDbInstallationVerificationLogs = true
+ }
+
for _, arg := range os.Args {
if strings.Contains(arg, "test") {
env.IsTesting = true
@@ -192,16 +199,36 @@ func loadEnvVariables() {
log.Info("ENV_MODE loaded", "mode", env.EnvMode)
env.PostgresesInstallDir = filepath.Join(backendRoot, "tools", "postgresql")
- tools.VerifyPostgresesInstallation(log, env.EnvMode, env.PostgresesInstallDir)
+ tools.VerifyPostgresesInstallation(
+ log,
+ env.EnvMode,
+ env.PostgresesInstallDir,
+ env.ShowDbInstallationVerificationLogs,
+ )
env.MysqlInstallDir = filepath.Join(backendRoot, "tools", "mysql")
- tools.VerifyMysqlInstallation(log, env.EnvMode, env.MysqlInstallDir)
+ tools.VerifyMysqlInstallation(
+ log,
+ env.EnvMode,
+ env.MysqlInstallDir,
+ env.ShowDbInstallationVerificationLogs,
+ )
env.MariadbInstallDir = filepath.Join(backendRoot, "tools", "mariadb")
- tools.VerifyMariadbInstallation(log, env.EnvMode, env.MariadbInstallDir)
+ tools.VerifyMariadbInstallation(
+ log,
+ env.EnvMode,
+ env.MariadbInstallDir,
+ env.ShowDbInstallationVerificationLogs,
+ )
env.MongodbInstallDir = filepath.Join(backendRoot, "tools", "mongodb")
- tools.VerifyMongodbInstallation(log, env.EnvMode, env.MongodbInstallDir)
+ tools.VerifyMongodbInstallation(
+ log,
+ env.EnvMode,
+ env.MongodbInstallDir,
+ env.ShowDbInstallationVerificationLogs,
+ )
env.NodeID = uuid.New().String()
if env.NodeNetworkThroughputMBs == 0 {
diff --git a/backend/internal/features/backups/backups/backuping/backuper.go b/backend/internal/features/backups/backups/backuping/backuper.go
index af5e92d7..efa8b1b4 100644
--- a/backend/internal/features/backups/backups/backuping/backuper.go
+++ b/backend/internal/features/backups/backups/backuping/backuper.go
@@ -3,11 +3,12 @@ package backuping
import (
"context"
"databasus-backend/internal/config"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/storages"
+ tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
+ task_registry "databasus-backend/internal/features/tasks/registry"
workspaces_services "databasus-backend/internal/features/workspaces/services"
util_encryption "databasus-backend/internal/util/encryption"
"errors"
@@ -20,6 +21,11 @@ import (
"github.com/google/uuid"
)
+const (
+ heartbeatTickerInterval = 15 * time.Second
+ backuperHeathcheckThreshold = 5 * time.Minute
+)
+
type BackuperNode struct {
databaseService *databases.DatabaseService
fieldEncryptor util_encryption.FieldEncryptor
@@ -28,8 +34,8 @@ type BackuperNode struct {
backupConfigService *backups_config.BackupConfigService
storageService *storages.StorageService
notificationSender backups_core.NotificationSender
- backupCancelManager *backups_cancellation.BackupCancelManager
- nodesRegistry *BackupNodesRegistry
+ backupCancelManager *tasks_cancellation.TaskCancelManager
+ tasksRegistry *task_registry.TaskNodesRegistry
logger *slog.Logger
createBackupUseCase backups_core.CreateBackupUsecase
nodeID uuid.UUID
@@ -42,20 +48,19 @@ func (n *BackuperNode) Run(ctx context.Context) {
throughputMBs := config.GetEnv().NodeNetworkThroughputMBs
- backupNode := BackupNode{
+ backupNode := task_registry.TaskNode{
ID: n.nodeID,
ThroughputMBs: throughputMBs,
- LastHeartbeat: time.Now().UTC(),
}
- if err := n.nodesRegistry.HearthbeatNodeInRegistry(time.Now().UTC(), backupNode); err != nil {
+ if err := n.tasksRegistry.HearthbeatNodeInRegistry(time.Now().UTC(), backupNode); err != nil {
n.logger.Error("Failed to register node in registry", "error", err)
panic(err)
}
backupHandler := func(backupID uuid.UUID, isCallNotifier bool) {
n.MakeBackup(backupID, isCallNotifier)
- if err := n.nodesRegistry.PublishBackupCompletion(n.nodeID.String(), backupID); err != nil {
+ if err := n.tasksRegistry.PublishTaskCompletion(n.nodeID.String(), backupID); err != nil {
n.logger.Error(
"Failed to publish backup completion",
"error",
@@ -66,17 +71,17 @@ func (n *BackuperNode) Run(ctx context.Context) {
}
}
- if err := n.nodesRegistry.SubscribeNodeForBackupsAssignment(n.nodeID.String(), backupHandler); err != nil {
+ if err := n.tasksRegistry.SubscribeNodeForTasksAssignment(n.nodeID.String(), backupHandler); err != nil {
n.logger.Error("Failed to subscribe to backup assignments", "error", err)
panic(err)
}
defer func() {
- if err := n.nodesRegistry.UnsubscribeNodeForBackupsAssignments(); err != nil {
+ if err := n.tasksRegistry.UnsubscribeNodeForTasksAssignments(); err != nil {
n.logger.Error("Failed to unsubscribe from backup assignments", "error", err)
}
}()
- ticker := time.NewTicker(15 * time.Second)
+ ticker := time.NewTicker(heartbeatTickerInterval)
defer ticker.Stop()
n.logger.Info("Backup node started", "nodeID", n.nodeID, "throughput", throughputMBs)
@@ -86,7 +91,7 @@ func (n *BackuperNode) Run(ctx context.Context) {
case <-ctx.Done():
n.logger.Info("Shutdown signal received, unregistering node", "nodeID", n.nodeID)
- if err := n.nodesRegistry.UnregisterNodeFromRegistry(backupNode); err != nil {
+ if err := n.tasksRegistry.UnregisterNodeFromRegistry(backupNode); err != nil {
n.logger.Error("Failed to unregister node from registry", "error", err)
}
@@ -98,7 +103,7 @@ func (n *BackuperNode) Run(ctx context.Context) {
}
func (n *BackuperNode) IsBackuperRunning() bool {
- return n.lastHeartbeat.After(time.Now().UTC().Add(-5 * time.Minute))
+ return n.lastHeartbeat.After(time.Now().UTC().Add(-backuperHeathcheckThreshold))
}
func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
@@ -147,8 +152,8 @@ func (n *BackuperNode) MakeBackup(backupID uuid.UUID, isCallNotifier bool) {
}
ctx, cancel := context.WithCancel(context.Background())
- n.backupCancelManager.RegisterBackup(backup.ID, cancel)
- defer n.backupCancelManager.UnregisterBackup(backup.ID)
+ n.backupCancelManager.RegisterTask(backup.ID, cancel)
+ defer n.backupCancelManager.UnregisterTask(backup.ID)
backupMetadata, err := n.createBackupUseCase.Execute(
ctx,
@@ -335,10 +340,9 @@ func (n *BackuperNode) SendBackupNotification(
}
}
-func (n *BackuperNode) sendHeartbeat(backupNode *BackupNode) {
+func (n *BackuperNode) sendHeartbeat(backupNode *task_registry.TaskNode) {
n.lastHeartbeat = time.Now().UTC()
- backupNode.LastHeartbeat = time.Now().UTC()
- if err := n.nodesRegistry.HearthbeatNodeInRegistry(time.Now().UTC(), *backupNode); err != nil {
+ if err := n.tasksRegistry.HearthbeatNodeInRegistry(time.Now().UTC(), *backupNode); err != nil {
n.logger.Error("Failed to send heartbeat", "error", err)
}
}
diff --git a/backend/internal/features/backups/backups/backuping/di.go b/backend/internal/features/backups/backups/backuping/di.go
index 3b95ed76..ff2b3429 100644
--- a/backend/internal/features/backups/backups/backuping/di.go
+++ b/backend/internal/features/backups/backups/backuping/di.go
@@ -2,15 +2,15 @@ package backuping
import (
"databasus-backend/internal/config"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_core "databasus-backend/internal/features/backups/backups/core"
"databasus-backend/internal/features/backups/backups/usecases"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
+ tasks_cancellation "databasus-backend/internal/features/tasks/cancellation"
+ task_registry "databasus-backend/internal/features/tasks/registry"
workspaces_services "databasus-backend/internal/features/workspaces/services"
- cache_utils "databasus-backend/internal/util/cache"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
"time"
@@ -20,15 +20,9 @@ import (
var backupRepository = &backups_core.BackupRepository{}
-var backupCancelManager = backups_cancellation.GetBackupCancelManager()
+var taskCancelManager = tasks_cancellation.GetTaskCancelManager()
-var nodesRegistry = &BackupNodesRegistry{
- cache_utils.GetValkeyClient(),
- logger.GetLogger(),
- cache_utils.DefaultCacheTimeout,
- cache_utils.NewPubSubManager(),
- cache_utils.NewPubSubManager(),
-}
+var nodesRegistry = task_registry.GetTaskNodesRegistry()
func getNodeID() uuid.UUID {
nodeIDStr := config.GetEnv().NodeID
@@ -48,7 +42,7 @@ var backuperNode = &BackuperNode{
backups_config.GetBackupConfigService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
- backupCancelManager,
+ taskCancelManager,
nodesRegistry,
logger.GetLogger(),
usecases.GetCreateBackupUsecase(),
@@ -60,7 +54,7 @@ var backupsScheduler = &BackupsScheduler{
backupRepository,
backups_config.GetBackupConfigService(),
storages.GetStorageService(),
- backupCancelManager,
+ taskCancelManager,
nodesRegistry,
time.Now().UTC(),
logger.GetLogger(),
diff --git a/backend/internal/features/backups/backups/backuping/dto.go b/backend/internal/features/backups/backups/backuping/dto.go
index d8824936..c8f2fa3e 100644
--- a/backend/internal/features/backups/backups/backuping/dto.go
+++ b/backend/internal/features/backups/backups/backuping/dto.go
@@ -1,32 +1,6 @@
package backuping
-import (
- "time"
-
- "github.com/google/uuid"
-)
-
-type BackupNode struct {
- ID uuid.UUID `json:"id"`
- ThroughputMBs int `json:"throughputMBs"`
- LastHeartbeat time.Time `json:"lastHeartbeat"`
-}
-
-type BackupNodeStats struct {
- ID uuid.UUID `json:"id"`
- ActiveBackups int `json:"activeBackups"`
-}
-
-type BackupSubmitMessage struct {
- NodeID string `json:"nodeId"`
- BackupID string `json:"backupId"`
- IsCallNotifier bool `json:"isCallNotifier"`
-}
-
-type BackupCompletionMessage struct {
- NodeID string `json:"nodeId"`
- BackupID string `json:"backupId"`
-}
+import "github.com/google/uuid"
type BackupToNodeRelation struct {
NodeID uuid.UUID `json:"nodeId"`
diff --git a/backend/internal/features/backups/backups/backuping/registry.go b/backend/internal/features/backups/backups/backuping/registry.go
deleted file mode 100644
index 21c850bd..00000000
--- a/backend/internal/features/backups/backups/backuping/registry.go
+++ /dev/null
@@ -1,448 +0,0 @@
-package backuping
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "log/slog"
- "strings"
- "time"
-
- cache_utils "databasus-backend/internal/util/cache"
-
- "github.com/google/uuid"
- "github.com/valkey-io/valkey-go"
-)
-
-const (
- nodeInfoKeyPrefix = "node:"
- nodeInfoKeySuffix = ":info"
- nodeActiveBackupsPrefix = "node:"
- nodeActiveBackupsSuffix = ":active_backups"
- backupSubmitChannel = "backup:submit"
- backupCompletionChannel = "backup:completion"
-)
-
-// BackupNodesRegistry helps to sync backups scheduler and backup nodes.
-// Features:
-// - Track node availability and load level
-// - Assign from scheduler to node backups needed to be processed
-// - Notify scheduler from node about backup completion
-type BackupNodesRegistry struct {
- client valkey.Client
- logger *slog.Logger
- timeout time.Duration
- pubsubBackups *cache_utils.PubSubManager
- pubsubCompletions *cache_utils.PubSubManager
-}
-
-func (r *BackupNodesRegistry) GetAvailableNodes() ([]BackupNode, error) {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- var allKeys []string
- cursor := uint64(0)
- pattern := nodeInfoKeyPrefix + "*" + nodeInfoKeySuffix
-
- for {
- result := r.client.Do(
- ctx,
- r.client.B().Scan().Cursor(cursor).Match(pattern).Count(1_000).Build(),
- )
-
- if result.Error() != nil {
- return nil, fmt.Errorf("failed to scan node keys: %w", result.Error())
- }
-
- scanResult, err := result.AsScanEntry()
- if err != nil {
- return nil, fmt.Errorf("failed to parse scan result: %w", err)
- }
-
- allKeys = append(allKeys, scanResult.Elements...)
-
- cursor = scanResult.Cursor
- if cursor == 0 {
- break
- }
- }
-
- if len(allKeys) == 0 {
- return []BackupNode{}, nil
- }
-
- keyDataMap, err := r.pipelineGetKeys(allKeys)
- if err != nil {
- return nil, fmt.Errorf("failed to pipeline get node keys: %w", err)
- }
-
- var nodes []BackupNode
- for key, data := range keyDataMap {
- var node BackupNode
- if err := json.Unmarshal(data, &node); err != nil {
- r.logger.Warn("Failed to unmarshal node data", "key", key, "error", err)
- continue
- }
- nodes = append(nodes, node)
- }
-
- return nodes, nil
-}
-
-func (r *BackupNodesRegistry) GetBackupNodesStats() ([]BackupNodeStats, error) {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- var allKeys []string
- cursor := uint64(0)
- pattern := nodeActiveBackupsPrefix + "*" + nodeActiveBackupsSuffix
-
- for {
- result := r.client.Do(
- ctx,
- r.client.B().Scan().Cursor(cursor).Match(pattern).Count(100).Build(),
- )
-
- if result.Error() != nil {
- return nil, fmt.Errorf("failed to scan active backups keys: %w", result.Error())
- }
-
- scanResult, err := result.AsScanEntry()
- if err != nil {
- return nil, fmt.Errorf("failed to parse scan result: %w", err)
- }
-
- allKeys = append(allKeys, scanResult.Elements...)
-
- cursor = scanResult.Cursor
- if cursor == 0 {
- break
- }
- }
-
- if len(allKeys) == 0 {
- return []BackupNodeStats{}, nil
- }
-
- keyDataMap, err := r.pipelineGetKeys(allKeys)
- if err != nil {
- return nil, fmt.Errorf("failed to pipeline get active backups keys: %w", err)
- }
-
- var stats []BackupNodeStats
- for key, data := range keyDataMap {
- nodeID := r.extractNodeIDFromKey(key, nodeActiveBackupsPrefix, nodeActiveBackupsSuffix)
-
- count, err := r.parseIntFromBytes(data)
- if err != nil {
- r.logger.Warn("Failed to parse active backups count", "key", key, "error", err)
- continue
- }
-
- stat := BackupNodeStats{
- ID: nodeID,
- ActiveBackups: int(count),
- }
- stats = append(stats, stat)
- }
-
- return stats, nil
-}
-
-func (r *BackupNodesRegistry) IncrementBackupsInProgress(nodeID string) error {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- key := fmt.Sprintf("%s%s%s", nodeActiveBackupsPrefix, nodeID, nodeActiveBackupsSuffix)
- result := r.client.Do(ctx, r.client.B().Incr().Key(key).Build())
-
- if result.Error() != nil {
- return fmt.Errorf(
- "failed to increment backups in progress for node %s: %w",
- nodeID,
- result.Error(),
- )
- }
-
- return nil
-}
-
-func (r *BackupNodesRegistry) DecrementBackupsInProgress(nodeID string) error {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- key := fmt.Sprintf("%s%s%s", nodeActiveBackupsPrefix, nodeID, nodeActiveBackupsSuffix)
- result := r.client.Do(ctx, r.client.B().Decr().Key(key).Build())
-
- if result.Error() != nil {
- return fmt.Errorf(
- "failed to decrement backups in progress for node %s: %w",
- nodeID,
- result.Error(),
- )
- }
-
- newValue, err := result.AsInt64()
- if err != nil {
- return fmt.Errorf("failed to parse decremented value for node %s: %w", nodeID, err)
- }
-
- if newValue < 0 {
- setCtx, setCancel := context.WithTimeout(context.Background(), r.timeout)
- r.client.Do(setCtx, r.client.B().Set().Key(key).Value("0").Build())
- setCancel()
- r.logger.Warn("Active backups counter went below 0, reset to 0", "nodeID", nodeID)
- }
-
- return nil
-}
-
-func (r *BackupNodesRegistry) HearthbeatNodeInRegistry(now time.Time, backupNode BackupNode) error {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- backupNode.LastHeartbeat = now
-
- data, err := json.Marshal(backupNode)
- if err != nil {
- return fmt.Errorf("failed to marshal backup node: %w", err)
- }
-
- key := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, backupNode.ID.String(), nodeInfoKeySuffix)
- result := r.client.Do(
- ctx,
- r.client.B().Set().Key(key).Value(string(data)).Build(),
- )
-
- if result.Error() != nil {
- return fmt.Errorf("failed to register node %s: %w", backupNode.ID, result.Error())
- }
-
- return nil
-}
-
-func (r *BackupNodesRegistry) UnregisterNodeFromRegistry(backupNode BackupNode) error {
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- infoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, backupNode.ID.String(), nodeInfoKeySuffix)
- counterKey := fmt.Sprintf(
- "%s%s%s",
- nodeActiveBackupsPrefix,
- backupNode.ID.String(),
- nodeActiveBackupsSuffix,
- )
-
- result := r.client.Do(
- ctx,
- r.client.B().Del().Key(infoKey, counterKey).Build(),
- )
-
- if result.Error() != nil {
- return fmt.Errorf("failed to unregister node %s: %w", backupNode.ID, result.Error())
- }
-
- r.logger.Info("Unregistered node from registry", "nodeID", backupNode.ID)
- return nil
-}
-
-func (r *BackupNodesRegistry) AssignBackupToNode(
- targetNodeID string,
- backupID uuid.UUID,
- isCallNotifier bool,
-) error {
- ctx := context.Background()
-
- message := BackupSubmitMessage{
- NodeID: targetNodeID,
- BackupID: backupID.String(),
- IsCallNotifier: isCallNotifier,
- }
-
- messageJSON, err := json.Marshal(message)
- if err != nil {
- return fmt.Errorf("failed to marshal backup submit message: %w", err)
- }
-
- err = r.pubsubBackups.Publish(ctx, backupSubmitChannel, string(messageJSON))
- if err != nil {
- return fmt.Errorf("failed to publish backup submit message: %w", err)
- }
-
- return nil
-}
-
-func (r *BackupNodesRegistry) SubscribeNodeForBackupsAssignment(
- nodeID string,
- handler func(backupID uuid.UUID, isCallNotifier bool),
-) error {
- ctx := context.Background()
-
- wrappedHandler := func(message string) {
- var msg BackupSubmitMessage
- if err := json.Unmarshal([]byte(message), &msg); err != nil {
- r.logger.Warn("Failed to unmarshal backup submit message", "error", err)
- return
- }
-
- if msg.NodeID != nodeID {
- return
- }
-
- backupID, err := uuid.Parse(msg.BackupID)
- if err != nil {
- r.logger.Warn(
- "Failed to parse backup ID from message",
- "backupId",
- msg.BackupID,
- "error",
- err,
- )
- return
- }
-
- handler(backupID, msg.IsCallNotifier)
- }
-
- err := r.pubsubBackups.Subscribe(ctx, backupSubmitChannel, wrappedHandler)
- if err != nil {
- return fmt.Errorf("failed to subscribe to backup submit channel: %w", err)
- }
-
- r.logger.Info("Subscribed to backup submit channel", "nodeID", nodeID)
- return nil
-}
-
-func (r *BackupNodesRegistry) UnsubscribeNodeForBackupsAssignments() error {
- err := r.pubsubBackups.Close()
- if err != nil {
- return fmt.Errorf("failed to unsubscribe from backup submit channel: %w", err)
- }
-
- r.logger.Info("Unsubscribed from backup submit channel")
- return nil
-}
-
-func (r *BackupNodesRegistry) PublishBackupCompletion(nodeID string, backupID uuid.UUID) error {
- ctx := context.Background()
-
- message := BackupCompletionMessage{
- NodeID: nodeID,
- BackupID: backupID.String(),
- }
-
- messageJSON, err := json.Marshal(message)
- if err != nil {
- return fmt.Errorf("failed to marshal backup completion message: %w", err)
- }
-
- err = r.pubsubCompletions.Publish(ctx, backupCompletionChannel, string(messageJSON))
- if err != nil {
- return fmt.Errorf("failed to publish backup completion message: %w", err)
- }
-
- return nil
-}
-
-func (r *BackupNodesRegistry) SubscribeForBackupsCompletions(
- handler func(nodeID string, backupID uuid.UUID),
-) error {
- ctx := context.Background()
-
- wrappedHandler := func(message string) {
- var msg BackupCompletionMessage
- if err := json.Unmarshal([]byte(message), &msg); err != nil {
- r.logger.Warn("Failed to unmarshal backup completion message", "error", err)
- return
- }
-
- backupID, err := uuid.Parse(msg.BackupID)
- if err != nil {
- r.logger.Warn(
- "Failed to parse backup ID from completion message",
- "backupId",
- msg.BackupID,
- "error",
- err,
- )
- return
- }
-
- handler(msg.NodeID, backupID)
- }
-
- err := r.pubsubCompletions.Subscribe(ctx, backupCompletionChannel, wrappedHandler)
- if err != nil {
- return fmt.Errorf("failed to subscribe to backup completion channel: %w", err)
- }
-
- r.logger.Info("Subscribed to backup completion channel")
- return nil
-}
-
-func (r *BackupNodesRegistry) UnsubscribeForBackupsCompletions() error {
- err := r.pubsubCompletions.Close()
- if err != nil {
- return fmt.Errorf("failed to unsubscribe from backup completion channel: %w", err)
- }
-
- r.logger.Info("Unsubscribed from backup completion channel")
- return nil
-}
-
-func (r *BackupNodesRegistry) extractNodeIDFromKey(key, prefix, suffix string) uuid.UUID {
- nodeIDStr := strings.TrimPrefix(key, prefix)
- nodeIDStr = strings.TrimSuffix(nodeIDStr, suffix)
-
- nodeID, err := uuid.Parse(nodeIDStr)
- if err != nil {
- r.logger.Warn("Failed to parse node ID from key", "key", key, "error", err)
- return uuid.Nil
- }
-
- return nodeID
-}
-
-func (r *BackupNodesRegistry) pipelineGetKeys(keys []string) (map[string][]byte, error) {
- if len(keys) == 0 {
- return make(map[string][]byte), nil
- }
-
- ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
- defer cancel()
-
- commands := make([]valkey.Completed, 0, len(keys))
- for _, key := range keys {
- commands = append(commands, r.client.B().Get().Key(key).Build())
- }
-
- results := r.client.DoMulti(ctx, commands...)
-
- keyDataMap := make(map[string][]byte, len(keys))
- for i, result := range results {
- if result.Error() != nil {
- r.logger.Warn("Failed to get key in pipeline", "key", keys[i], "error", result.Error())
- continue
- }
-
- data, err := result.AsBytes()
- if err != nil {
- r.logger.Warn("Failed to parse key data in pipeline", "key", keys[i], "error", err)
- continue
- }
-
- keyDataMap[keys[i]] = data
- }
-
- return keyDataMap, nil
-}
-
-func (r *BackupNodesRegistry) parseIntFromBytes(data []byte) (int64, error) {
- str := string(data)
- var count int64
- _, err := fmt.Sscanf(str, "%d", &count)
- if err != nil {
- return 0, fmt.Errorf("failed to parse integer from bytes: %w", err)
- }
- return count, nil
-}
diff --git a/backend/internal/features/backups/backups/backuping/registry_test.go b/backend/internal/features/backups/backups/backuping/registry_test.go
deleted file mode 100644
index b5438e1e..00000000
--- a/backend/internal/features/backups/backups/backuping/registry_test.go
+++ /dev/null
@@ -1,904 +0,0 @@
-package backuping
-
-import (
- "context"
- "testing"
- "time"
-
- cache_utils "databasus-backend/internal/util/cache"
- "databasus-backend/internal/util/logger"
-
- "github.com/google/uuid"
- "github.com/stretchr/testify/assert"
-)
-
-func Test_HearthbeatNodeInRegistry_RegistersNodeWithTTL(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer cleanupTestNode(registry, node)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Len(t, nodes, 1)
- assert.Equal(t, node.ID, nodes[0].ID)
- assert.Equal(t, node.ThroughputMBs, nodes[0].ThroughputMBs)
-}
-
-func Test_UnregisterNodeFromRegistry_RemovesNodeAndCounter(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- err = registry.UnregisterNodeFromRegistry(node)
- assert.NoError(t, err)
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Empty(t, nodes)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Empty(t, stats)
-}
-
-func Test_GetAvailableNodes_ReturnsAllRegisteredNodes(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- node3 := createTestBackupNode()
- defer cleanupTestNode(registry, node1)
- defer cleanupTestNode(registry, node2)
- defer cleanupTestNode(registry, node3)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
- assert.NoError(t, err)
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Len(t, nodes, 3)
-
- nodeIDs := make(map[uuid.UUID]bool)
- for _, node := range nodes {
- nodeIDs[node.ID] = true
- }
- assert.True(t, nodeIDs[node1.ID])
- assert.True(t, nodeIDs[node2.ID])
- assert.True(t, nodeIDs[node3.ID])
-}
-
-func Test_GetAvailableNodes_WhenNoNodesExist_ReturnsEmptySlice(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.NotNil(t, nodes)
- assert.Empty(t, nodes)
-}
-
-func Test_IncrementBackupsInProgress_IncrementsCounter(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer cleanupTestNode(registry, node)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Len(t, stats, 1)
- assert.Equal(t, node.ID, stats[0].ID)
- assert.Equal(t, 1, stats[0].ActiveBackups)
-
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err = registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Len(t, stats, 1)
- assert.Equal(t, 2, stats[0].ActiveBackups)
-}
-
-func Test_DecrementBackupsInProgress_DecrementsCounter(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer cleanupTestNode(registry, node)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Equal(t, 3, stats[0].ActiveBackups)
-
- err = registry.DecrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err = registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Equal(t, 2, stats[0].ActiveBackups)
-
- err = registry.DecrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err = registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Equal(t, 1, stats[0].ActiveBackups)
-}
-
-func Test_DecrementBackupsInProgress_WhenNegative_ResetsToZero(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer cleanupTestNode(registry, node)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- err = registry.DecrementBackupsInProgress(node.ID.String())
- assert.NoError(t, err)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Len(t, stats, 1)
- assert.Equal(t, 0, stats[0].ActiveBackups)
-}
-
-func Test_GetBackupNodesStats_ReturnsStatsForAllNodes(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- node3 := createTestBackupNode()
- defer cleanupTestNode(registry, node1)
- defer cleanupTestNode(registry, node2)
- defer cleanupTestNode(registry, node3)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node1.ID.String())
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node2.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node2.ID.String())
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node3.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node3.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node3.ID.String())
- assert.NoError(t, err)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Len(t, stats, 3)
-
- statsMap := make(map[uuid.UUID]int)
- for _, stat := range stats {
- statsMap[stat.ID] = stat.ActiveBackups
- }
-
- assert.Equal(t, 1, statsMap[node1.ID])
- assert.Equal(t, 2, statsMap[node2.ID])
- assert.Equal(t, 3, statsMap[node3.ID])
-}
-
-func Test_GetBackupNodesStats_WhenNoStats_ReturnsEmptySlice(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.NotNil(t, stats)
- assert.Empty(t, stats)
-}
-
-func Test_MultipleNodes_RegisteredAndQueriedCorrectly(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node1 := createTestBackupNode()
- node1.ThroughputMBs = 50
- node2 := createTestBackupNode()
- node2.ThroughputMBs = 100
- node3 := createTestBackupNode()
- node3.ThroughputMBs = 150
- defer cleanupTestNode(registry, node1)
- defer cleanupTestNode(registry, node2)
- defer cleanupTestNode(registry, node3)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
- assert.NoError(t, err)
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Len(t, nodes, 3)
-
- nodeMap := make(map[uuid.UUID]BackupNode)
- for _, node := range nodes {
- nodeMap[node.ID] = node
- }
-
- assert.Equal(t, 50, nodeMap[node1.ID].ThroughputMBs)
- assert.Equal(t, 100, nodeMap[node2.ID].ThroughputMBs)
- assert.Equal(t, 150, nodeMap[node3.ID].ThroughputMBs)
-}
-
-func Test_BackupCounters_TrackedSeparatelyPerNode(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- defer cleanupTestNode(registry, node1)
- defer cleanupTestNode(registry, node2)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
- assert.NoError(t, err)
- err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node1.ID.String())
- assert.NoError(t, err)
- err = registry.IncrementBackupsInProgress(node1.ID.String())
- assert.NoError(t, err)
-
- err = registry.IncrementBackupsInProgress(node2.ID.String())
- assert.NoError(t, err)
-
- stats, err := registry.GetBackupNodesStats()
- assert.NoError(t, err)
- assert.Len(t, stats, 2)
-
- statsMap := make(map[uuid.UUID]int)
- for _, stat := range stats {
- statsMap[stat.ID] = stat.ActiveBackups
- }
-
- assert.Equal(t, 2, statsMap[node1.ID])
- assert.Equal(t, 1, statsMap[node2.ID])
-
- err = registry.DecrementBackupsInProgress(node1.ID.String())
- assert.NoError(t, err)
-
- stats, err = registry.GetBackupNodesStats()
- assert.NoError(t, err)
-
- statsMap = make(map[uuid.UUID]int)
- for _, stat := range stats {
- statsMap[stat.ID] = stat.ActiveBackups
- }
-
- assert.Equal(t, 1, statsMap[node1.ID])
- assert.Equal(t, 1, statsMap[node2.ID])
-}
-
-func Test_GetAvailableNodes_SkipsInvalidJsonData(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer cleanupTestNode(registry, node)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- ctx, cancel := context.WithTimeout(context.Background(), registry.timeout)
- defer cancel()
-
- invalidKey := nodeInfoKeyPrefix + uuid.New().String() + nodeInfoKeySuffix
- registry.client.Do(
- ctx,
- registry.client.B().Set().Key(invalidKey).Value("invalid json data").Build(),
- )
- defer func() {
- cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), registry.timeout)
- defer cleanupCancel()
- registry.client.Do(cleanupCtx, registry.client.B().Del().Key(invalidKey).Build())
- }()
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Len(t, nodes, 1)
- assert.Equal(t, node.ID, nodes[0].ID)
-}
-
-func Test_PipelineGetKeys_HandlesEmptyKeysList(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
-
- keyDataMap, err := registry.pipelineGetKeys([]string{})
- assert.NoError(t, err)
- assert.NotNil(t, keyDataMap)
- assert.Empty(t, keyDataMap)
-}
-
-func Test_HearthbeatNodeInRegistry_UpdatesLastHeartbeat(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- originalHeartbeat := node.LastHeartbeat
- defer cleanupTestNode(registry, node)
-
- time.Sleep(10 * time.Millisecond)
-
- err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
- assert.NoError(t, err)
-
- nodes, err := registry.GetAvailableNodes()
- assert.NoError(t, err)
- assert.Len(t, nodes, 1)
- assert.True(t, nodes[0].LastHeartbeat.After(originalHeartbeat))
-}
-
-func createTestRegistry() *BackupNodesRegistry {
- return &BackupNodesRegistry{
- cache_utils.GetValkeyClient(),
- logger.GetLogger(),
- cache_utils.DefaultCacheTimeout,
- cache_utils.NewPubSubManager(),
- cache_utils.NewPubSubManager(),
- }
-}
-
-func createTestBackupNode() BackupNode {
- return BackupNode{
- ID: uuid.New(),
- ThroughputMBs: 100,
- LastHeartbeat: time.Now().UTC(),
- }
-}
-
-func cleanupTestNode(registry *BackupNodesRegistry, node BackupNode) {
- registry.UnregisterNodeFromRegistry(node)
-}
-
-func Test_AssignBackupTonode_PublishesJsonMessageToChannel(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID := uuid.New()
-
- err := registry.AssignBackupToNode(node.ID.String(), backupID, true)
- assert.NoError(t, err)
-}
-
-func Test_SubscribeNodeForBackupsAssignment_ReceivesSubmittedBackupsForMatchingNode(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID := uuid.New()
- defer registry.UnsubscribeNodeForBackupsAssignments()
-
- receivedBackupID := make(chan uuid.UUID, 1)
- handler := func(id uuid.UUID, isCallNotifier bool) {
- receivedBackupID <- id
- }
-
- err := registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.AssignBackupToNode(node.ID.String(), backupID, true)
- assert.NoError(t, err)
-
- select {
- case received := <-receivedBackupID:
- assert.Equal(t, backupID, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Timeout waiting for backup message")
- }
-}
-
-func Test_SubscribeNodeForBackupsAssignment_FiltersOutBackupsForDifferentNode(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- backupID := uuid.New()
- defer registry.UnsubscribeNodeForBackupsAssignments()
-
- receivedBackupID := make(chan uuid.UUID, 1)
- handler := func(id uuid.UUID, isCallNotifier bool) {
- receivedBackupID <- id
- }
-
- err := registry.SubscribeNodeForBackupsAssignment(node1.ID.String(), handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.AssignBackupToNode(node2.ID.String(), backupID, false)
- assert.NoError(t, err)
-
- select {
- case <-receivedBackupID:
- t.Fatal("Should not receive backup for different node")
- case <-time.After(500 * time.Millisecond):
- }
-}
-
-func Test_SubscribeNodeForBackupsAssignment_ParsesJsonAndBackupIdCorrectly(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID1 := uuid.New()
- backupID2 := uuid.New()
- defer registry.UnsubscribeNodeForBackupsAssignments()
-
- receivedBackups := make(chan uuid.UUID, 2)
- handler := func(id uuid.UUID, isCallNotifier bool) {
- receivedBackups <- id
- }
-
- err := registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.AssignBackupToNode(node.ID.String(), backupID1, true)
- assert.NoError(t, err)
-
- err = registry.AssignBackupToNode(node.ID.String(), backupID2, false)
- assert.NoError(t, err)
-
- received1 := <-receivedBackups
- received2 := <-receivedBackups
-
- receivedIDs := []uuid.UUID{received1, received2}
- assert.Contains(t, receivedIDs, backupID1)
- assert.Contains(t, receivedIDs, backupID2)
-}
-
-func Test_SubscribeNodeForBackupsAssignment_HandlesInvalidJson(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer registry.UnsubscribeNodeForBackupsAssignments()
-
- receivedBackupID := make(chan uuid.UUID, 1)
- handler := func(id uuid.UUID, isCallNotifier bool) {
- receivedBackupID <- id
- }
-
- err := registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- ctx := context.Background()
- err = registry.pubsubBackups.Publish(ctx, "backup:submit", "invalid json")
- assert.NoError(t, err)
-
- select {
- case <-receivedBackupID:
- t.Fatal("Should not receive backup for invalid JSON")
- case <-time.After(500 * time.Millisecond):
- }
-}
-
-func Test_UnsubscribeNodeForBackupsAssignments_StopsReceivingMessages(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID1 := uuid.New()
- backupID2 := uuid.New()
-
- receivedBackupID := make(chan uuid.UUID, 2)
- handler := func(id uuid.UUID, isCallNotifier bool) {
- receivedBackupID <- id
- }
-
- err := registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.AssignBackupToNode(node.ID.String(), backupID1, true)
- assert.NoError(t, err)
-
- received := <-receivedBackupID
- assert.Equal(t, backupID1, received)
-
- err = registry.UnsubscribeNodeForBackupsAssignments()
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.AssignBackupToNode(node.ID.String(), backupID2, false)
- assert.NoError(t, err)
-
- select {
- case <-receivedBackupID:
- t.Fatal("Should not receive backup after unsubscribe")
- case <-time.After(500 * time.Millisecond):
- }
-}
-
-func Test_SubscribeNodeForBackupsAssignment_WhenAlreadySubscribed_ReturnsError(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- defer registry.UnsubscribeNodeForBackupsAssignments()
-
- handler := func(id uuid.UUID, isCallNotifier bool) {}
-
- err := registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.NoError(t, err)
-
- err = registry.SubscribeNodeForBackupsAssignment(node.ID.String(), handler)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "already subscribed")
-}
-
-func Test_MultipleNodes_EachReceivesOnlyTheirBackups(t *testing.T) {
- cache_utils.ClearAllCache()
- registry1 := createTestRegistry()
- registry2 := createTestRegistry()
- registry3 := createTestRegistry()
-
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- node3 := createTestBackupNode()
-
- backupID1 := uuid.New()
- backupID2 := uuid.New()
- backupID3 := uuid.New()
-
- defer registry1.UnsubscribeNodeForBackupsAssignments()
- defer registry2.UnsubscribeNodeForBackupsAssignments()
- defer registry3.UnsubscribeNodeForBackupsAssignments()
-
- receivedBackups1 := make(chan uuid.UUID, 3)
- receivedBackups2 := make(chan uuid.UUID, 3)
- receivedBackups3 := make(chan uuid.UUID, 3)
-
- handler1 := func(id uuid.UUID, isCallNotifier bool) { receivedBackups1 <- id }
- handler2 := func(id uuid.UUID, isCallNotifier bool) { receivedBackups2 <- id }
- handler3 := func(id uuid.UUID, isCallNotifier bool) { receivedBackups3 <- id }
-
- err := registry1.SubscribeNodeForBackupsAssignment(node1.ID.String(), handler1)
- assert.NoError(t, err)
-
- err = registry2.SubscribeNodeForBackupsAssignment(node2.ID.String(), handler2)
- assert.NoError(t, err)
-
- err = registry3.SubscribeNodeForBackupsAssignment(node3.ID.String(), handler3)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- submitRegistry := createTestRegistry()
- err = submitRegistry.AssignBackupToNode(node1.ID.String(), backupID1, true)
- assert.NoError(t, err)
-
- err = submitRegistry.AssignBackupToNode(node2.ID.String(), backupID2, false)
- assert.NoError(t, err)
-
- err = submitRegistry.AssignBackupToNode(node3.ID.String(), backupID3, true)
- assert.NoError(t, err)
-
- select {
- case received := <-receivedBackups1:
- assert.Equal(t, backupID1, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Node 1 timeout waiting for backup message")
- }
-
- select {
- case received := <-receivedBackups2:
- assert.Equal(t, backupID2, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Node 2 timeout waiting for backup message")
- }
-
- select {
- case received := <-receivedBackups3:
- assert.Equal(t, backupID3, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Node 3 timeout waiting for backup message")
- }
-
- select {
- case <-receivedBackups1:
- t.Fatal("Node 1 should not receive additional backups")
- case <-time.After(300 * time.Millisecond):
- }
-
- select {
- case <-receivedBackups2:
- t.Fatal("Node 2 should not receive additional backups")
- case <-time.After(300 * time.Millisecond):
- }
-
- select {
- case <-receivedBackups3:
- t.Fatal("Node 3 should not receive additional backups")
- case <-time.After(300 * time.Millisecond):
- }
-}
-
-func Test_PublishBackupCompletion_PublishesMessageToChannel(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID := uuid.New()
-
- err := registry.PublishBackupCompletion(node.ID.String(), backupID)
- assert.NoError(t, err)
-}
-
-func Test_SubscribeForBackupsCompletions_ReceivesCompletedBackups(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID := uuid.New()
- defer registry.UnsubscribeForBackupsCompletions()
-
- receivedBackupID := make(chan uuid.UUID, 1)
- receivedNodeID := make(chan string, 1)
- handler := func(nodeID string, backupID uuid.UUID) {
- receivedNodeID <- nodeID
- receivedBackupID <- backupID
- }
-
- err := registry.SubscribeForBackupsCompletions(handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.PublishBackupCompletion(node.ID.String(), backupID)
- assert.NoError(t, err)
-
- select {
- case receivedNode := <-receivedNodeID:
- assert.Equal(t, node.ID.String(), receivedNode)
- case <-time.After(2 * time.Second):
- t.Fatal("Timeout waiting for node ID")
- }
-
- select {
- case received := <-receivedBackupID:
- assert.Equal(t, backupID, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Timeout waiting for backup completion message")
- }
-}
-
-func Test_SubscribeForBackupsCompletions_ParsesJsonCorrectly(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID1 := uuid.New()
- backupID2 := uuid.New()
- defer registry.UnsubscribeForBackupsCompletions()
-
- receivedBackups := make(chan uuid.UUID, 2)
- handler := func(nodeID string, backupID uuid.UUID) {
- receivedBackups <- backupID
- }
-
- err := registry.SubscribeForBackupsCompletions(handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.PublishBackupCompletion(node.ID.String(), backupID1)
- assert.NoError(t, err)
-
- err = registry.PublishBackupCompletion(node.ID.String(), backupID2)
- assert.NoError(t, err)
-
- received1 := <-receivedBackups
- received2 := <-receivedBackups
-
- receivedIDs := []uuid.UUID{received1, received2}
- assert.Contains(t, receivedIDs, backupID1)
- assert.Contains(t, receivedIDs, backupID2)
-}
-
-func Test_SubscribeForBackupsCompletions_HandlesInvalidJson(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- defer registry.UnsubscribeForBackupsCompletions()
-
- receivedBackupID := make(chan uuid.UUID, 1)
- handler := func(nodeID string, backupID uuid.UUID) {
- receivedBackupID <- backupID
- }
-
- err := registry.SubscribeForBackupsCompletions(handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- ctx := context.Background()
- err = registry.pubsubCompletions.Publish(ctx, "backup:completion", "invalid json")
- assert.NoError(t, err)
-
- select {
- case <-receivedBackupID:
- t.Fatal("Should not receive backup for invalid JSON")
- case <-time.After(500 * time.Millisecond):
- }
-}
-
-func Test_UnsubscribeForBackupsCompletions_StopsReceivingMessages(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- node := createTestBackupNode()
- backupID1 := uuid.New()
- backupID2 := uuid.New()
-
- receivedBackupID := make(chan uuid.UUID, 2)
- handler := func(nodeID string, backupID uuid.UUID) {
- receivedBackupID <- backupID
- }
-
- err := registry.SubscribeForBackupsCompletions(handler)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.PublishBackupCompletion(node.ID.String(), backupID1)
- assert.NoError(t, err)
-
- received := <-receivedBackupID
- assert.Equal(t, backupID1, received)
-
- err = registry.UnsubscribeForBackupsCompletions()
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- err = registry.PublishBackupCompletion(node.ID.String(), backupID2)
- assert.NoError(t, err)
-
- select {
- case <-receivedBackupID:
- t.Fatal("Should not receive backup after unsubscribe")
- case <-time.After(500 * time.Millisecond):
- }
-}
-
-func Test_SubscribeForBackupsCompletions_WhenAlreadySubscribed_ReturnsError(t *testing.T) {
- cache_utils.ClearAllCache()
- registry := createTestRegistry()
- defer registry.UnsubscribeForBackupsCompletions()
-
- handler := func(nodeID string, backupID uuid.UUID) {}
-
- err := registry.SubscribeForBackupsCompletions(handler)
- assert.NoError(t, err)
-
- err = registry.SubscribeForBackupsCompletions(handler)
- assert.Error(t, err)
- assert.Contains(t, err.Error(), "already subscribed")
-}
-
-func Test_MultipleSubscribers_EachReceivesCompletionMessages(t *testing.T) {
- cache_utils.ClearAllCache()
- registry1 := createTestRegistry()
- registry2 := createTestRegistry()
- registry3 := createTestRegistry()
-
- node1 := createTestBackupNode()
- node2 := createTestBackupNode()
- node3 := createTestBackupNode()
-
- backupID1 := uuid.New()
- backupID2 := uuid.New()
- backupID3 := uuid.New()
-
- defer registry1.UnsubscribeForBackupsCompletions()
- defer registry2.UnsubscribeForBackupsCompletions()
- defer registry3.UnsubscribeForBackupsCompletions()
-
- receivedBackups1 := make(chan uuid.UUID, 3)
- receivedBackups2 := make(chan uuid.UUID, 3)
- receivedBackups3 := make(chan uuid.UUID, 3)
-
- handler1 := func(nodeID string, backupID uuid.UUID) { receivedBackups1 <- backupID }
- handler2 := func(nodeID string, backupID uuid.UUID) { receivedBackups2 <- backupID }
- handler3 := func(nodeID string, backupID uuid.UUID) { receivedBackups3 <- backupID }
-
- err := registry1.SubscribeForBackupsCompletions(handler1)
- assert.NoError(t, err)
-
- err = registry2.SubscribeForBackupsCompletions(handler2)
- assert.NoError(t, err)
-
- err = registry3.SubscribeForBackupsCompletions(handler3)
- assert.NoError(t, err)
-
- time.Sleep(100 * time.Millisecond)
-
- publishRegistry := createTestRegistry()
- err = publishRegistry.PublishBackupCompletion(node1.ID.String(), backupID1)
- assert.NoError(t, err)
-
- err = publishRegistry.PublishBackupCompletion(node2.ID.String(), backupID2)
- assert.NoError(t, err)
-
- err = publishRegistry.PublishBackupCompletion(node3.ID.String(), backupID3)
- assert.NoError(t, err)
-
- receivedAll1 := []uuid.UUID{}
- receivedAll2 := []uuid.UUID{}
- receivedAll3 := []uuid.UUID{}
-
- for i := 0; i < 3; i++ {
- select {
- case received := <-receivedBackups1:
- receivedAll1 = append(receivedAll1, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Subscriber 1 timeout waiting for completion message")
- }
- }
-
- for i := 0; i < 3; i++ {
- select {
- case received := <-receivedBackups2:
- receivedAll2 = append(receivedAll2, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Subscriber 2 timeout waiting for completion message")
- }
- }
-
- for i := 0; i < 3; i++ {
- select {
- case received := <-receivedBackups3:
- receivedAll3 = append(receivedAll3, received)
- case <-time.After(2 * time.Second):
- t.Fatal("Subscriber 3 timeout waiting for completion message")
- }
- }
-
- assert.Contains(t, receivedAll1, backupID1)
- assert.Contains(t, receivedAll1, backupID2)
- assert.Contains(t, receivedAll1, backupID3)
-
- assert.Contains(t, receivedAll2, backupID1)
- assert.Contains(t, receivedAll2, backupID2)
- assert.Contains(t, receivedAll2, backupID3)
-
- assert.Contains(t, receivedAll3, backupID1)
- assert.Contains(t, receivedAll3, backupID2)
- assert.Contains(t, receivedAll3, backupID3)
-}
diff --git a/backend/internal/features/backups/backups/backuping/scheduler.go b/backend/internal/features/backups/backups/backuping/scheduler.go
index d723fab6..49e6fd42 100644
--- a/backend/internal/features/backups/backups/backuping/scheduler.go
+++ b/backend/internal/features/backups/backups/backuping/scheduler.go
@@ -3,10 +3,11 @@ package backuping
import (
"context"
"databasus-backend/internal/config"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_config "databasus-backend/internal/features/backups/config"
"databasus-backend/internal/features/storages"
+ task_cancellation "databasus-backend/internal/features/tasks/cancellation"
+ task_registry "databasus-backend/internal/features/tasks/registry"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/period"
"fmt"
@@ -16,12 +17,18 @@ import (
"github.com/google/uuid"
)
+const (
+ schedulerStartupDelay = 1 * time.Minute
+ schedulerTickerInterval = 1 * time.Minute
+ schedulerHealthcheckThreshold = 5 * time.Minute
+)
+
type BackupsScheduler struct {
backupRepository *backups_core.BackupRepository
backupConfigService *backups_config.BackupConfigService
storageService *storages.StorageService
- backupCancelManager *backups_cancellation.BackupCancelManager
- nodesRegistry *BackupNodesRegistry
+ taskCancelManager *task_cancellation.TaskCancelManager
+ tasksRegistry *task_registry.TaskNodesRegistry
lastBackupTime time.Time
logger *slog.Logger
@@ -35,7 +42,7 @@ func (s *BackupsScheduler) Run(ctx context.Context) {
if config.GetEnv().IsManyNodesMode {
// wait other nodes to start
- time.Sleep(1 * time.Minute)
+ time.Sleep(schedulerStartupDelay)
}
if err := s.failBackupsInProgress(); err != nil {
@@ -43,12 +50,12 @@ func (s *BackupsScheduler) Run(ctx context.Context) {
panic(err)
}
- if err := s.nodesRegistry.SubscribeForBackupsCompletions(s.onBackupCompleted); err != nil {
+ if err := s.tasksRegistry.SubscribeForTasksCompletions(s.onBackupCompleted); err != nil {
s.logger.Error("Failed to subscribe to backup completions", "error", err)
panic(err)
}
defer func() {
- if err := s.nodesRegistry.UnsubscribeForBackupsCompletions(); err != nil {
+ if err := s.tasksRegistry.UnsubscribeForTasksCompletions(); err != nil {
s.logger.Error("Failed to unsubscribe from backup completions", "error", err)
}
}()
@@ -57,7 +64,7 @@ func (s *BackupsScheduler) Run(ctx context.Context) {
return
}
- ticker := time.NewTicker(1 * time.Minute)
+ ticker := time.NewTicker(schedulerTickerInterval)
defer ticker.Stop()
for {
@@ -84,7 +91,7 @@ func (s *BackupsScheduler) Run(ctx context.Context) {
func (s *BackupsScheduler) IsSchedulerRunning() bool {
// if last backup time is more than 5 minutes ago, return false
- return s.lastBackupTime.After(time.Now().UTC().Add(-5 * time.Minute))
+ return s.lastBackupTime.After(time.Now().UTC().Add(-schedulerHealthcheckThreshold))
}
func (s *BackupsScheduler) failBackupsInProgress() error {
@@ -93,12 +100,10 @@ func (s *BackupsScheduler) failBackupsInProgress() error {
return err
}
- fmt.Println("Backups in progress", len(backupsInProgress))
-
for _, backup := range backupsInProgress {
- if err := s.backupCancelManager.CancelBackup(backup.ID); err != nil {
+ if err := s.taskCancelManager.CancelTask(backup.ID); err != nil {
s.logger.Error(
- "Failed to cancel backup via context manager",
+ "Failed to cancel backup via task cancel manager",
"backupId",
backup.ID,
"error",
@@ -175,7 +180,7 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
return
}
- if err := s.nodesRegistry.IncrementBackupsInProgress(leastBusyNodeID.String()); err != nil {
+ if err := s.tasksRegistry.IncrementTasksInProgress(leastBusyNodeID.String()); err != nil {
s.logger.Error(
"Failed to increment backups in progress",
"nodeId",
@@ -188,7 +193,7 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
return
}
- if err := s.nodesRegistry.AssignBackupToNode(leastBusyNodeID.String(), backup.ID, isCallNotifier); err != nil {
+ if err := s.tasksRegistry.AssignTaskToNode(leastBusyNodeID.String(), backup.ID, isCallNotifier); err != nil {
s.logger.Error(
"Failed to submit backup",
"nodeId",
@@ -198,7 +203,7 @@ func (s *BackupsScheduler) StartBackup(databaseID uuid.UUID, isCallNotifier bool
"error",
err,
)
- if decrementErr := s.nodesRegistry.DecrementBackupsInProgress(leastBusyNodeID.String()); decrementErr != nil {
+ if decrementErr := s.tasksRegistry.DecrementTasksInProgress(leastBusyNodeID.String()); decrementErr != nil {
s.logger.Error(
"Failed to decrement backups in progress after submit failure",
"nodeId",
@@ -393,7 +398,7 @@ func (s *BackupsScheduler) runPendingBackups() error {
}
func (s *BackupsScheduler) calculateLeastBusyNode() (*uuid.UUID, error) {
- nodes, err := s.nodesRegistry.GetAvailableNodes()
+ nodes, err := s.tasksRegistry.GetAvailableNodes()
if err != nil {
return nil, fmt.Errorf("failed to get available nodes: %w", err)
}
@@ -402,27 +407,22 @@ func (s *BackupsScheduler) calculateLeastBusyNode() (*uuid.UUID, error) {
return nil, fmt.Errorf("no nodes available")
}
- stats, err := s.nodesRegistry.GetBackupNodesStats()
+ stats, err := s.tasksRegistry.GetNodesStats()
if err != nil {
return nil, fmt.Errorf("failed to get backup nodes stats: %w", err)
}
statsMap := make(map[uuid.UUID]int)
for _, stat := range stats {
- statsMap[stat.ID] = stat.ActiveBackups
+ statsMap[stat.ID] = stat.ActiveTasks
}
- var bestNode *BackupNode
+ var bestNode *task_registry.TaskNode
var bestScore float64 = -1
- now := time.Now().UTC()
for i := range nodes {
node := &nodes[i]
- if now.Sub(node.LastHeartbeat) > 2*time.Minute {
- continue
- }
-
activeBackups := statsMap[node.ID]
var score float64
@@ -458,6 +458,13 @@ func (s *BackupsScheduler) onBackupCompleted(nodeIDStr string, backupID uuid.UUI
return
}
+ // Verify this task is actually a backup (registry contains multiple task types)
+ _, err = s.backupRepository.FindByID(backupID)
+ if err != nil {
+ // Not a backup task, ignore it
+ return
+ }
+
relation, exists := s.backupToNodeRelations[nodeID]
if !exists {
s.logger.Warn(
@@ -498,7 +505,7 @@ func (s *BackupsScheduler) onBackupCompleted(nodeIDStr string, backupID uuid.UUI
s.backupToNodeRelations[nodeID] = relation
}
- if err := s.nodesRegistry.DecrementBackupsInProgress(nodeIDStr); err != nil {
+ if err := s.tasksRegistry.DecrementTasksInProgress(nodeIDStr); err != nil {
s.logger.Error(
"Failed to decrement backups in progress",
"nodeId",
@@ -512,18 +519,14 @@ func (s *BackupsScheduler) onBackupCompleted(nodeIDStr string, backupID uuid.UUI
}
func (s *BackupsScheduler) checkDeadNodesAndFailBackups() error {
- nodes, err := s.nodesRegistry.GetAvailableNodes()
+ nodes, err := s.tasksRegistry.GetAvailableNodes()
if err != nil {
return fmt.Errorf("failed to get available nodes: %w", err)
}
aliveNodeIDs := make(map[uuid.UUID]bool)
- now := time.Now().UTC()
-
for _, node := range nodes {
- if now.Sub(node.LastHeartbeat) <= 2*time.Minute {
- aliveNodeIDs[node.ID] = true
- }
+ aliveNodeIDs[node.ID] = true
}
for nodeID, relation := range s.backupToNodeRelations {
@@ -572,7 +575,7 @@ func (s *BackupsScheduler) checkDeadNodesAndFailBackups() error {
continue
}
- if err := s.nodesRegistry.DecrementBackupsInProgress(nodeID.String()); err != nil {
+ if err := s.tasksRegistry.DecrementTasksInProgress(nodeID.String()); err != nil {
s.logger.Error(
"Failed to decrement backups in progress for dead node",
"nodeId",
diff --git a/backend/internal/features/backups/backups/backuping/scheduler_test.go b/backend/internal/features/backups/backups/backuping/scheduler_test.go
index 9ed8631a..d31ab34e 100644
--- a/backend/internal/features/backups/backups/backuping/scheduler_test.go
+++ b/backend/internal/features/backups/backups/backuping/scheduler_test.go
@@ -7,6 +7,7 @@ import (
"databasus-backend/internal/features/intervals"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
+ task_registry "databasus-backend/internal/features/tasks/registry"
users_enums "databasus-backend/internal/features/users/enums"
users_testing "databasus-backend/internal/features/users/testing"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
@@ -42,8 +43,8 @@ func Test_RunPendingBackups_WhenLastBackupWasYesterday_CreatesNewBackup(t *testi
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -111,8 +112,8 @@ func Test_RunPendingBackups_WhenLastBackupWasRecentlyCompleted_SkipsBackup(t *te
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -179,8 +180,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesDisabled_SkipsBackup(t
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -251,8 +252,8 @@ func Test_RunPendingBackups_WhenLastBackupFailedAndRetriesEnabled_CreatesNewBack
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -324,8 +325,8 @@ func Test_RunPendingBackups_WhenFailedBackupsExceedMaxRetries_SkipsBackup(t *tes
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -396,8 +397,8 @@ func Test_RunPendingBackups_WhenBackupsDisabled_SkipsBackup(t *testing.T) {
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
}()
@@ -449,6 +450,8 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
notifier := notifiers.CreateTestNotifier(workspace.ID)
database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
+ var mockNodeID uuid.UUID
+
defer func() {
backups, _ := backupRepository.FindByDatabaseID(database.ID)
for _, backup := range backups {
@@ -457,9 +460,15 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
+
+ // Clean up mock node
+ if mockNodeID != uuid.Nil {
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: mockNodeID})
+ }
+ cache_utils.ClearAllCache()
}()
backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)
@@ -479,7 +488,7 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
assert.NoError(t, err)
// Register mock node without subscribing to backups (simulates node crash after registration)
- mockNodeID := uuid.New()
+ mockNodeID = uuid.New()
err = CreateMockNodeInRegistry(mockNodeID, 100, time.Now().UTC())
assert.NoError(t, err)
@@ -493,12 +502,12 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
assert.Equal(t, backups_core.BackupStatusInProgress, backups[0].Status)
// Verify Valkey counter was incremented when backup was assigned
- stats, err := nodesRegistry.GetBackupNodesStats()
+ stats, err := nodesRegistry.GetNodesStats()
assert.NoError(t, err)
foundStat := false
for _, stat := range stats {
if stat.ID == mockNodeID {
- assert.Equal(t, 1, stat.ActiveBackups)
+ assert.Equal(t, 1, stat.ActiveTasks)
foundStat = true
break
}
@@ -523,19 +532,117 @@ func Test_CheckDeadNodesAndFailBackups_WhenNodeDies_FailsBackupAndCleansUpRegist
assert.Contains(t, *backups[0].FailMessage, "node unavailability")
// Verify Valkey counter was decremented after backup failed
- stats, err = nodesRegistry.GetBackupNodesStats()
+ stats, err = nodesRegistry.GetNodesStats()
assert.NoError(t, err)
for _, stat := range stats {
if stat.ID == mockNodeID {
- assert.Equal(t, 0, stat.ActiveBackups)
+ assert.Equal(t, 0, stat.ActiveTasks)
}
}
- // Node info should still exist in registry (not removed by checkDeadNodesAndFailBackups)
- node, err := GetNodeFromRegistry(mockNodeID)
+ time.Sleep(200 * time.Millisecond)
+}
+
+func Test_OnBackupCompleted_WhenTaskIsNotBackup_SkipsProcessing(t *testing.T) {
+ cache_utils.ClearAllCache()
+
+ user := users_testing.CreateTestUser(users_enums.UserRoleAdmin)
+ router := CreateTestRouter()
+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
+ storage := storages.CreateTestStorage(workspace.ID)
+ notifier := notifiers.CreateTestNotifier(workspace.ID)
+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
+
+ var mockNodeID uuid.UUID
+
+ defer func() {
+ backups, _ := backupRepository.FindByDatabaseID(database.ID)
+ for _, backup := range backups {
+ backupRepository.DeleteByID(backup.ID)
+ }
+
+ databases.RemoveTestDatabase(database)
+ time.Sleep(50 * time.Millisecond)
+ storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
+ workspaces_testing.RemoveTestWorkspace(workspace, router)
+
+ // Clean up mock node
+ if mockNodeID != uuid.Nil {
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: mockNodeID})
+ }
+ cache_utils.ClearAllCache()
+ }()
+
+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)
assert.NoError(t, err)
- assert.NotNil(t, node)
- assert.Equal(t, mockNodeID, node.ID)
+
+ timeOfDay := "04:00"
+ backupConfig.BackupInterval = &intervals.Interval{
+ Interval: intervals.IntervalDaily,
+ TimeOfDay: &timeOfDay,
+ }
+ backupConfig.IsBackupsEnabled = true
+ backupConfig.StorePeriod = period.PeriodWeek
+ backupConfig.Storage = storage
+ backupConfig.StorageID = &storage.ID
+
+ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
+ assert.NoError(t, err)
+
+ // Register mock node
+ mockNodeID = uuid.New()
+ err = CreateMockNodeInRegistry(mockNodeID, 100, time.Now().UTC())
+ assert.NoError(t, err)
+
+ // Start a backup and assign it to the node
+ GetBackupsScheduler().StartBackup(database.ID, false)
+ time.Sleep(100 * time.Millisecond)
+
+ backups, err := backupRepository.FindByDatabaseID(database.ID)
+ assert.NoError(t, err)
+ assert.Len(t, backups, 1)
+ assert.Equal(t, backups_core.BackupStatusInProgress, backups[0].Status)
+
+ // Get initial state of the registry
+ initialStats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ var initialActiveTasks int
+ for _, stat := range initialStats {
+ if stat.ID == mockNodeID {
+ initialActiveTasks = stat.ActiveTasks
+ break
+ }
+ }
+ assert.Equal(t, 1, initialActiveTasks, "Should have 1 active task")
+
+ // Call onBackupCompleted with a random UUID (not a backup ID)
+ nonBackupTaskID := uuid.New()
+ GetBackupsScheduler().onBackupCompleted(mockNodeID.String(), nonBackupTaskID)
+
+ time.Sleep(100 * time.Millisecond)
+
+ // Verify: Active tasks counter should remain the same (not decremented)
+ stats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ for _, stat := range stats {
+ if stat.ID == mockNodeID {
+ assert.Equal(t, initialActiveTasks, stat.ActiveTasks,
+ "Active tasks should not change for non-backup task")
+ }
+ }
+
+ // Verify: backup should still be in progress (not modified)
+ backups, err = backupRepository.FindByDatabaseID(database.ID)
+ assert.NoError(t, err)
+ assert.Len(t, backups, 1)
+ assert.Equal(t, backups_core.BackupStatusInProgress, backups[0].Status,
+ "Backup status should not change for non-backup task completion")
+
+ // Verify: backupToNodeRelations should still contain the node
+ scheduler := GetBackupsScheduler()
+ _, exists := scheduler.backupToNodeRelations[mockNodeID]
+ assert.True(t, exists, "Node should still be in backupToNodeRelations")
time.Sleep(200 * time.Millisecond)
}
@@ -549,6 +656,14 @@ func Test_CalculateLeastBusyNode_SelectsNodeWithBestScore(t *testing.T) {
node3ID := uuid.New()
now := time.Now().UTC()
+ defer func() {
+ // Clean up all mock nodes
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: node1ID})
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: node2ID})
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: node3ID})
+ cache_utils.ClearAllCache()
+ }()
+
err := CreateMockNodeInRegistry(node1ID, 100, now)
assert.NoError(t, err)
err = CreateMockNodeInRegistry(node2ID, 100, now)
@@ -557,17 +672,17 @@ func Test_CalculateLeastBusyNode_SelectsNodeWithBestScore(t *testing.T) {
assert.NoError(t, err)
for range 5 {
- err = nodesRegistry.IncrementBackupsInProgress(node1ID.String())
+ err = nodesRegistry.IncrementTasksInProgress(node1ID.String())
assert.NoError(t, err)
}
for range 2 {
- err = nodesRegistry.IncrementBackupsInProgress(node2ID.String())
+ err = nodesRegistry.IncrementTasksInProgress(node2ID.String())
assert.NoError(t, err)
}
for range 8 {
- err = nodesRegistry.IncrementBackupsInProgress(node3ID.String())
+ err = nodesRegistry.IncrementTasksInProgress(node3ID.String())
assert.NoError(t, err)
}
@@ -584,17 +699,24 @@ func Test_CalculateLeastBusyNode_SelectsNodeWithBestScore(t *testing.T) {
node50MBsID := uuid.New()
now := time.Now().UTC()
+ defer func() {
+ // Clean up all mock nodes
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: node100MBsID})
+ nodesRegistry.UnregisterNodeFromRegistry(task_registry.TaskNode{ID: node50MBsID})
+ cache_utils.ClearAllCache()
+ }()
+
err := CreateMockNodeInRegistry(node100MBsID, 100, now)
assert.NoError(t, err)
err = CreateMockNodeInRegistry(node50MBsID, 50, now)
assert.NoError(t, err)
for range 10 {
- err = nodesRegistry.IncrementBackupsInProgress(node100MBsID.String())
+ err = nodesRegistry.IncrementTasksInProgress(node100MBsID.String())
assert.NoError(t, err)
}
- err = nodesRegistry.IncrementBackupsInProgress(node50MBsID.String())
+ err = nodesRegistry.IncrementTasksInProgress(node50MBsID.String())
assert.NoError(t, err)
leastBusyNodeID, err := GetBackupsScheduler().calculateLeastBusyNode()
@@ -622,9 +744,11 @@ func Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStat
databases.RemoveTestDatabase(database)
time.Sleep(50 * time.Millisecond)
- notifiers.RemoveTestNotifier(notifier)
storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
workspaces_testing.RemoveTestWorkspace(workspace, router)
+
+ cache_utils.ClearAllCache()
}()
backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)
@@ -707,3 +831,204 @@ func Test_FailBackupsInProgress_WhenSchedulerStarts_CancelsBackupsAndUpdatesStat
time.Sleep(200 * time.Millisecond)
}
+
+func Test_StartBackup_WhenBackupCompletes_DecrementsActiveTaskCount(t *testing.T) {
+ cache_utils.ClearAllCache()
+
+ // Start scheduler so it can handle task completions
+ schedulerCancel := StartSchedulerForTest(t)
+ defer schedulerCancel()
+
+ backuperNode := CreateTestBackuperNode()
+ cancel := StartBackuperNodeForTest(t, backuperNode)
+ defer StopBackuperNodeForTest(t, cancel, backuperNode)
+
+ user := users_testing.CreateTestUser(users_enums.UserRoleAdmin)
+ router := CreateTestRouter()
+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
+ storage := storages.CreateTestStorage(workspace.ID)
+ notifier := notifiers.CreateTestNotifier(workspace.ID)
+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
+
+ defer func() {
+ backups, _ := backupRepository.FindByDatabaseID(database.ID)
+ for _, backup := range backups {
+ backupRepository.DeleteByID(backup.ID)
+ }
+
+ databases.RemoveTestDatabase(database)
+ time.Sleep(50 * time.Millisecond)
+ storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
+ workspaces_testing.RemoveTestWorkspace(workspace, router)
+ }()
+
+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)
+ assert.NoError(t, err)
+
+ timeOfDay := "04:00"
+ backupConfig.BackupInterval = &intervals.Interval{
+ Interval: intervals.IntervalDaily,
+ TimeOfDay: &timeOfDay,
+ }
+ backupConfig.IsBackupsEnabled = true
+ backupConfig.StorePeriod = period.PeriodWeek
+ backupConfig.Storage = storage
+ backupConfig.StorageID = &storage.ID
+
+ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
+ assert.NoError(t, err)
+
+ // Get initial active task count
+ stats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ var initialActiveTasks int
+ for _, stat := range stats {
+ if stat.ID == backuperNode.nodeID {
+ initialActiveTasks = stat.ActiveTasks
+ break
+ }
+ }
+ t.Logf("Initial active tasks: %d", initialActiveTasks)
+
+ // Start backup
+ GetBackupsScheduler().StartBackup(database.ID, false)
+
+ // Wait for backup to complete
+ WaitForBackupCompletion(t, database.ID, 0, 10*time.Second)
+
+ // Verify backup was created and completed
+ backups, err := backupRepository.FindByDatabaseID(database.ID)
+ assert.NoError(t, err)
+ assert.Len(t, backups, 1)
+ assert.Equal(t, backups_core.BackupStatusCompleted, backups[0].Status)
+
+ // Wait for active task count to decrease
+ decreased := WaitForActiveTasksDecrease(
+ t,
+ backuperNode.nodeID,
+ initialActiveTasks+1,
+ 10*time.Second,
+ )
+ assert.True(t, decreased, "Active task count should have decreased after backup completion")
+
+ // Verify final active task count equals initial count
+ finalStats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ for _, stat := range finalStats {
+ if stat.ID == backuperNode.nodeID {
+ t.Logf("Final active tasks: %d", stat.ActiveTasks)
+ assert.Equal(t, initialActiveTasks, stat.ActiveTasks,
+ "Active task count should return to initial value after backup completion")
+ break
+ }
+ }
+
+ time.Sleep(200 * time.Millisecond)
+}
+
+func Test_StartBackup_WhenBackupFails_DecrementsActiveTaskCount(t *testing.T) {
+ cache_utils.ClearAllCache()
+
+ // Start scheduler so it can handle task completions
+ schedulerCancel := StartSchedulerForTest(t)
+ defer schedulerCancel()
+
+ backuperNode := CreateTestBackuperNode()
+ cancel := StartBackuperNodeForTest(t, backuperNode)
+ defer StopBackuperNodeForTest(t, cancel, backuperNode)
+
+ user := users_testing.CreateTestUser(users_enums.UserRoleAdmin)
+ router := CreateTestRouter()
+ workspace := workspaces_testing.CreateTestWorkspace("Test Workspace", user, router)
+ storage := storages.CreateTestStorage(workspace.ID)
+ notifier := notifiers.CreateTestNotifier(workspace.ID)
+ database := databases.CreateTestDatabase(workspace.ID, storage, notifier)
+
+ defer func() {
+ backups, _ := backupRepository.FindByDatabaseID(database.ID)
+ for _, backup := range backups {
+ backupRepository.DeleteByID(backup.ID)
+ }
+
+ databases.RemoveTestDatabase(database)
+ time.Sleep(50 * time.Millisecond)
+ storages.RemoveTestStorage(storage.ID)
+ notifiers.RemoveTestNotifier(notifier)
+ workspaces_testing.RemoveTestWorkspace(workspace, router)
+ }()
+
+ // Set wrong password to cause backup failure
+ // We need to bypass service layer validation which would fail on connection test
+ database.Postgresql.Password = "intentionally_wrong_password"
+ dbRepo := &databases.DatabaseRepository{}
+ _, err := dbRepo.Save(database)
+ assert.NoError(t, err)
+
+ backupConfig, err := backups_config.GetBackupConfigService().GetBackupConfigByDbId(database.ID)
+ assert.NoError(t, err)
+
+ timeOfDay := "04:00"
+ backupConfig.BackupInterval = &intervals.Interval{
+ Interval: intervals.IntervalDaily,
+ TimeOfDay: &timeOfDay,
+ }
+ backupConfig.IsBackupsEnabled = true
+ backupConfig.StorePeriod = period.PeriodWeek
+ backupConfig.Storage = storage
+ backupConfig.StorageID = &storage.ID
+
+ _, err = backups_config.GetBackupConfigService().SaveBackupConfig(backupConfig)
+ assert.NoError(t, err)
+
+ // Get initial active task count
+ stats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ var initialActiveTasks int
+ for _, stat := range stats {
+ if stat.ID == backuperNode.nodeID {
+ initialActiveTasks = stat.ActiveTasks
+ break
+ }
+ }
+ t.Logf("Initial active tasks: %d", initialActiveTasks)
+
+ // Start backup
+ GetBackupsScheduler().StartBackup(database.ID, false)
+
+ // Wait for backup to fail
+ WaitForBackupCompletion(t, database.ID, 0, 10*time.Second)
+
+ // Verify backup was created and failed
+ backups, err := backupRepository.FindByDatabaseID(database.ID)
+ assert.NoError(t, err)
+ assert.Len(t, backups, 1)
+ assert.Equal(t, backups_core.BackupStatusFailed, backups[0].Status)
+ assert.NotNil(t, backups[0].FailMessage)
+ if backups[0].FailMessage != nil {
+ t.Logf("Backup failed with message: %s", *backups[0].FailMessage)
+ }
+
+ // Wait for active task count to decrease
+ decreased := WaitForActiveTasksDecrease(
+ t,
+ backuperNode.nodeID,
+ initialActiveTasks+1,
+ 10*time.Second,
+ )
+ assert.True(t, decreased, "Active task count should have decreased after backup failure")
+
+ // Verify final active task count equals initial count
+ finalStats, err := nodesRegistry.GetNodesStats()
+ assert.NoError(t, err)
+ for _, stat := range finalStats {
+ if stat.ID == backuperNode.nodeID {
+ t.Logf("Final active tasks: %d", stat.ActiveTasks)
+ assert.Equal(t, initialActiveTasks, stat.ActiveTasks,
+ "Active task count should return to initial value after backup failure")
+ break
+ }
+ }
+
+ time.Sleep(200 * time.Millisecond)
+}
diff --git a/backend/internal/features/backups/backups/backuping/testing.go b/backend/internal/features/backups/backups/backuping/testing.go
index d0f31f2d..197502eb 100644
--- a/backend/internal/features/backups/backups/backuping/testing.go
+++ b/backend/internal/features/backups/backups/backuping/testing.go
@@ -12,6 +12,7 @@ import (
"databasus-backend/internal/features/databases"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
+ task_registry "databasus-backend/internal/features/tasks/registry"
workspaces_controllers "databasus-backend/internal/features/workspaces/controllers"
workspaces_services "databasus-backend/internal/features/workspaces/services"
workspaces_testing "databasus-backend/internal/features/workspaces/testing"
@@ -42,7 +43,7 @@ func CreateTestBackuperNode() *BackuperNode {
backups_config.GetBackupConfigService(),
storages.GetStorageService(),
notifiers.GetNotifierService(),
- backupCancelManager,
+ taskCancelManager,
nodesRegistry,
logger.GetLogger(),
usecases.GetCreateBackupUsecase(),
@@ -138,6 +139,34 @@ func StartBackuperNodeForTest(t *testing.T, backuperNode *BackuperNode) context.
return nil
}
+// StartSchedulerForTest starts the BackupsScheduler in a goroutine for testing.
+// The scheduler subscribes to task completions and manages backup lifecycle.
+// Returns a context cancel function that should be deferred to stop the scheduler.
+func StartSchedulerForTest(t *testing.T) context.CancelFunc {
+ ctx, cancel := context.WithCancel(context.Background())
+
+ done := make(chan struct{})
+
+ go func() {
+ GetBackupsScheduler().Run(ctx)
+ close(done)
+ }()
+
+ // Give scheduler time to subscribe to completions
+ time.Sleep(100 * time.Millisecond)
+ t.Log("BackupsScheduler started")
+
+ return func() {
+ cancel()
+ select {
+ case <-done:
+ t.Log("BackupsScheduler stopped gracefully")
+ case <-time.After(2 * time.Second):
+ t.Log("BackupsScheduler stop timeout")
+ }
+ }
+}
+
// StopBackuperNodeForTest stops the BackuperNode by canceling its context.
// It waits for the node to unregister from the registry.
func StopBackuperNodeForTest(t *testing.T, cancel context.CancelFunc, backuperNode *BackuperNode) {
@@ -167,7 +196,7 @@ func StopBackuperNodeForTest(t *testing.T, cancel context.CancelFunc, backuperNo
}
func CreateMockNodeInRegistry(nodeID uuid.UUID, throughputMBs int, lastHeartbeat time.Time) error {
- backupNode := BackupNode{
+ backupNode := task_registry.TaskNode{
ID: nodeID,
ThroughputMBs: throughputMBs,
LastHeartbeat: lastHeartbeat,
@@ -181,7 +210,7 @@ func UpdateNodeHeartbeatDirectly(
throughputMBs int,
lastHeartbeat time.Time,
) error {
- backupNode := BackupNode{
+ backupNode := task_registry.TaskNode{
ID: nodeID,
ThroughputMBs: throughputMBs,
LastHeartbeat: lastHeartbeat,
@@ -190,7 +219,7 @@ func UpdateNodeHeartbeatDirectly(
return nodesRegistry.HearthbeatNodeInRegistry(lastHeartbeat, backupNode)
}
-func GetNodeFromRegistry(nodeID uuid.UUID) (*BackupNode, error) {
+func GetNodeFromRegistry(nodeID uuid.UUID) (*task_registry.TaskNode, error) {
nodes, err := nodesRegistry.GetAvailableNodes()
if err != nil {
return nil, err
@@ -204,3 +233,48 @@ func GetNodeFromRegistry(nodeID uuid.UUID) (*BackupNode, error) {
return nil, fmt.Errorf("node not found")
}
+
+// WaitForActiveTasksDecrease waits for the active task count to decrease below the initial count.
+// It polls the registry every 500ms until the count decreases or the timeout is reached.
+// Returns true if the count decreased, false if timeout was reached.
+func WaitForActiveTasksDecrease(
+ t *testing.T,
+ nodeID uuid.UUID,
+ initialCount int,
+ timeout time.Duration,
+) bool {
+ deadline := time.Now().UTC().Add(timeout)
+
+ for time.Now().UTC().Before(deadline) {
+ stats, err := nodesRegistry.GetNodesStats()
+ if err != nil {
+ t.Logf("WaitForActiveTasksDecrease: error getting node stats: %v", err)
+ time.Sleep(500 * time.Millisecond)
+ continue
+ }
+
+ for _, stat := range stats {
+ if stat.ID == nodeID {
+ t.Logf(
+ "WaitForActiveTasksDecrease: current active tasks = %d (initial = %d)",
+ stat.ActiveTasks,
+ initialCount,
+ )
+ if stat.ActiveTasks < initialCount {
+ t.Logf(
+ "WaitForActiveTasksDecrease: active tasks decreased from %d to %d",
+ initialCount,
+ stat.ActiveTasks,
+ )
+ return true
+ }
+ break
+ }
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ t.Logf("WaitForActiveTasksDecrease: timeout waiting for active tasks to decrease")
+ return false
+}
diff --git a/backend/internal/features/backups/backups/cancellation/backup_cancel_manager.go b/backend/internal/features/backups/backups/cancellation/backup_cancel_manager.go
deleted file mode 100644
index ad934438..00000000
--- a/backend/internal/features/backups/backups/cancellation/backup_cancel_manager.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package backups_cancellation
-
-import (
- "context"
- cache_utils "databasus-backend/internal/util/cache"
- "log/slog"
- "sync"
-
- "github.com/google/uuid"
-)
-
-const backupCancelChannel = "backup:cancel"
-
-type BackupCancelManager struct {
- mu sync.RWMutex
- cancelFuncs map[uuid.UUID]context.CancelFunc
- pubsub *cache_utils.PubSubManager
- logger *slog.Logger
-}
-
-func (m *BackupCancelManager) StartSubscription() {
- ctx := context.Background()
-
- handler := func(message string) {
- backupID, err := uuid.Parse(message)
- if err != nil {
- m.logger.Error("Invalid backup ID in cancel message", "message", message, "error", err)
- return
- }
-
- m.mu.Lock()
- defer m.mu.Unlock()
-
- cancelFunc, exists := m.cancelFuncs[backupID]
- if exists {
- cancelFunc()
- delete(m.cancelFuncs, backupID)
- m.logger.Info("Cancelled backup via Pub/Sub", "backupID", backupID)
- }
- }
-
- err := m.pubsub.Subscribe(ctx, backupCancelChannel, handler)
- if err != nil {
- m.logger.Error("Failed to subscribe to backup cancel channel", "error", err)
- } else {
- m.logger.Info("Successfully subscribed to backup cancel channel")
- }
-}
-
-func (m *BackupCancelManager) RegisterBackup(backupID uuid.UUID, cancelFunc context.CancelFunc) {
- m.mu.Lock()
- defer m.mu.Unlock()
- m.cancelFuncs[backupID] = cancelFunc
- m.logger.Debug("Registered backup", "backupID", backupID)
-}
-
-func (m *BackupCancelManager) CancelBackup(backupID uuid.UUID) error {
- ctx := context.Background()
-
- err := m.pubsub.Publish(ctx, backupCancelChannel, backupID.String())
- if err != nil {
- m.logger.Error("Failed to publish cancel message", "backupID", backupID, "error", err)
- return err
- }
-
- m.logger.Info("Published backup cancel message", "backupID", backupID)
- return nil
-}
-
-func (m *BackupCancelManager) UnregisterBackup(backupID uuid.UUID) {
- m.mu.Lock()
- defer m.mu.Unlock()
- delete(m.cancelFuncs, backupID)
- m.logger.Debug("Unregistered backup", "backupID", backupID)
-}
diff --git a/backend/internal/features/backups/backups/controller_test.go b/backend/internal/features/backups/backups/controller_test.go
index 56825b94..369a9889 100644
--- a/backend/internal/features/backups/backups/controller_test.go
+++ b/backend/internal/features/backups/backups/controller_test.go
@@ -913,7 +913,7 @@ func Test_CancelBackup_InProgressBackup_SuccessfullyCancelled(t *testing.T) {
assert.NoError(t, err)
// Register a cancellable context for the backup
- GetBackupService().backupCancelManager.RegisterBackup(backup.ID, func() {})
+ GetBackupService().taskCancelManager.RegisterTask(backup.ID, func() {})
resp := test_utils.MakePostRequest(
t,
diff --git a/backend/internal/features/backups/backups/di.go b/backend/internal/features/backups/backups/di.go
index f27752cf..ae053d3b 100644
--- a/backend/internal/features/backups/backups/di.go
+++ b/backend/internal/features/backups/backups/di.go
@@ -3,7 +3,6 @@ package backups
import (
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
"databasus-backend/internal/features/backups/backups/usecases"
@@ -12,6 +11,7 @@ import (
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
+ task_cancellation "databasus-backend/internal/features/tasks/cancellation"
workspaces_services "databasus-backend/internal/features/workspaces/services"
"databasus-backend/internal/util/encryption"
"databasus-backend/internal/util/logger"
@@ -19,7 +19,7 @@ import (
var backupRepository = &backups_core.BackupRepository{}
-var backupCancelManager = backups_cancellation.GetBackupCancelManager()
+var taskCancelManager = task_cancellation.GetTaskCancelManager()
var backupService = &BackupService{
databaseService: databases.GetDatabaseService(),
@@ -35,7 +35,7 @@ var backupService = &BackupService{
backupRemoveListeners: []backups_core.BackupRemoveListener{},
workspaceService: workspaces_services.GetWorkspaceService(),
auditLogService: audit_logs.GetAuditLogService(),
- backupCancelManager: backupCancelManager,
+ taskCancelManager: taskCancelManager,
downloadTokenService: backups_download.GetDownloadTokenService(),
backupSchedulerService: backuping.GetBackupsScheduler(),
}
diff --git a/backend/internal/features/backups/backups/service.go b/backend/internal/features/backups/backups/service.go
index bc91de1f..527797cc 100644
--- a/backend/internal/features/backups/backups/service.go
+++ b/backend/internal/features/backups/backups/service.go
@@ -9,7 +9,6 @@ import (
audit_logs "databasus-backend/internal/features/audit_logs"
"databasus-backend/internal/features/backups/backups/backuping"
- backups_cancellation "databasus-backend/internal/features/backups/backups/cancellation"
backups_core "databasus-backend/internal/features/backups/backups/core"
backups_download "databasus-backend/internal/features/backups/backups/download"
"databasus-backend/internal/features/backups/backups/encryption"
@@ -18,6 +17,7 @@ import (
encryption_secrets "databasus-backend/internal/features/encryption/secrets"
"databasus-backend/internal/features/notifiers"
"databasus-backend/internal/features/storages"
+ task_cancellation "databasus-backend/internal/features/tasks/cancellation"
users_models "databasus-backend/internal/features/users/models"
workspaces_services "databasus-backend/internal/features/workspaces/services"
util_encryption "databasus-backend/internal/util/encryption"
@@ -43,7 +43,7 @@ type BackupService struct {
workspaceService *workspaces_services.WorkspaceService
auditLogService *audit_logs.AuditLogService
- backupCancelManager *backups_cancellation.BackupCancelManager
+ taskCancelManager *task_cancellation.TaskCancelManager
downloadTokenService *backups_download.DownloadTokenService
backupSchedulerService *backuping.BackupsScheduler
}
@@ -226,7 +226,7 @@ func (s *BackupService) CancelBackup(
return errors.New("backup is not in progress")
}
- if err := s.backupCancelManager.CancelBackup(backupID); err != nil {
+ if err := s.taskCancelManager.CancelTask(backupID); err != nil {
return err
}
diff --git a/backend/internal/features/tasks/cancellation/cancel_manager.go b/backend/internal/features/tasks/cancellation/cancel_manager.go
new file mode 100644
index 00000000..5a98eac9
--- /dev/null
+++ b/backend/internal/features/tasks/cancellation/cancel_manager.go
@@ -0,0 +1,75 @@
+package task_cancellation
+
+import (
+ "context"
+ cache_utils "databasus-backend/internal/util/cache"
+ "log/slog"
+ "sync"
+
+ "github.com/google/uuid"
+)
+
+const taskCancelChannel = "task:cancel"
+
+type TaskCancelManager struct {
+ mu sync.RWMutex
+ cancelFuncs map[uuid.UUID]context.CancelFunc
+ pubsub *cache_utils.PubSubManager
+ logger *slog.Logger
+}
+
+func (m *TaskCancelManager) StartSubscription() {
+ ctx := context.Background()
+
+ handler := func(message string) {
+ taskID, err := uuid.Parse(message)
+ if err != nil {
+ m.logger.Error("Invalid task ID in cancel message", "message", message, "error", err)
+ return
+ }
+
+ m.mu.Lock()
+ defer m.mu.Unlock()
+
+ cancelFunc, exists := m.cancelFuncs[taskID]
+ if exists {
+ cancelFunc()
+ delete(m.cancelFuncs, taskID)
+ m.logger.Info("Cancelled task via Pub/Sub", "taskID", taskID)
+ }
+ }
+
+ err := m.pubsub.Subscribe(ctx, taskCancelChannel, handler)
+ if err != nil {
+ m.logger.Error("Failed to subscribe to task cancel channel", "error", err)
+ } else {
+ m.logger.Info("Successfully subscribed to task cancel channel")
+ }
+}
+
+func (m *TaskCancelManager) RegisterTask(task uuid.UUID, cancelFunc context.CancelFunc) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.cancelFuncs[task] = cancelFunc
+ m.logger.Debug("Registered task", "taskID", task)
+}
+
+func (m *TaskCancelManager) CancelTask(taskID uuid.UUID) error {
+ ctx := context.Background()
+
+ err := m.pubsub.Publish(ctx, taskCancelChannel, taskID.String())
+ if err != nil {
+ m.logger.Error("Failed to publish cancel message", "taskID", taskID, "error", err)
+ return err
+ }
+
+ m.logger.Info("Published task cancel message", "taskID", taskID)
+ return nil
+}
+
+func (m *TaskCancelManager) UnregisterTask(taskID uuid.UUID) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ delete(m.cancelFuncs, taskID)
+ m.logger.Debug("Unregistered task", "taskID", taskID)
+}
diff --git a/backend/internal/features/backups/backups/cancellation/backup_cancel_manager_test.go b/backend/internal/features/tasks/cancellation/cancel_manager_test.go
similarity index 53%
rename from backend/internal/features/backups/backups/cancellation/backup_cancel_manager_test.go
rename to backend/internal/features/tasks/cancellation/cancel_manager_test.go
index 8db186ec..50bdb481 100644
--- a/backend/internal/features/backups/backups/cancellation/backup_cancel_manager_test.go
+++ b/backend/internal/features/tasks/cancellation/cancel_manager_test.go
@@ -1,4 +1,4 @@
-package backups_cancellation
+package task_cancellation
import (
"context"
@@ -10,41 +10,41 @@ import (
"github.com/stretchr/testify/assert"
)
-func Test_RegisterBackup_BackupRegisteredSuccessfully(t *testing.T) {
- manager := backupCancelManager
+func Test_RegisterTask_TaskRegisteredSuccessfully(t *testing.T) {
+ manager := taskCancelManager
- backupID := uuid.New()
+ taskID := uuid.New()
_, cancel := context.WithCancel(context.Background())
defer cancel()
- manager.RegisterBackup(backupID, cancel)
+ manager.RegisterTask(taskID, cancel)
manager.mu.RLock()
- _, exists := manager.cancelFuncs[backupID]
+ _, exists := manager.cancelFuncs[taskID]
manager.mu.RUnlock()
- assert.True(t, exists, "Backup should be registered")
+ assert.True(t, exists, "Task should be registered")
}
-func Test_UnregisterBackup_BackupUnregisteredSuccessfully(t *testing.T) {
- manager := backupCancelManager
+func Test_UnregisterTask_TaskUnregisteredSuccessfully(t *testing.T) {
+ manager := taskCancelManager
- backupID := uuid.New()
+ taskID := uuid.New()
_, cancel := context.WithCancel(context.Background())
defer cancel()
- manager.RegisterBackup(backupID, cancel)
- manager.UnregisterBackup(backupID)
+ manager.RegisterTask(taskID, cancel)
+ manager.UnregisterTask(taskID)
manager.mu.RLock()
- _, exists := manager.cancelFuncs[backupID]
+ _, exists := manager.cancelFuncs[taskID]
manager.mu.RUnlock()
- assert.False(t, exists, "Backup should be unregistered")
+ assert.False(t, exists, "Task should be unregistered")
}
-func Test_CancelBackup_OnSameInstance_BackupCancelledViaPubSub(t *testing.T) {
- manager := backupCancelManager
+func Test_CancelTask_OnSameInstance_TaskCancelledViaPubSub(t *testing.T) {
+ manager := taskCancelManager
- backupID := uuid.New()
+ taskID := uuid.New()
ctx, cancel := context.WithCancel(context.Background())
cancelled := false
@@ -57,11 +57,11 @@ func Test_CancelBackup_OnSameInstance_BackupCancelledViaPubSub(t *testing.T) {
cancel()
}
- manager.RegisterBackup(backupID, wrappedCancel)
+ manager.RegisterTask(taskID, wrappedCancel)
manager.StartSubscription()
time.Sleep(100 * time.Millisecond)
- err := manager.CancelBackup(backupID)
+ err := manager.CancelTask(taskID)
assert.NoError(t, err, "Cancel should not return error")
time.Sleep(500 * time.Millisecond)
@@ -74,11 +74,11 @@ func Test_CancelBackup_OnSameInstance_BackupCancelledViaPubSub(t *testing.T) {
assert.Error(t, ctx.Err(), "Context should be cancelled")
}
-func Test_CancelBackup_FromDifferentInstance_BackupCancelledOnRunningInstance(t *testing.T) {
- manager1 := backupCancelManager
- manager2 := backupCancelManager
+func Test_CancelTask_FromDifferentInstance_TaskCancelledOnRunningInstance(t *testing.T) {
+ manager1 := taskCancelManager
+ manager2 := taskCancelManager
- backupID := uuid.New()
+ taskID := uuid.New()
ctx, cancel := context.WithCancel(context.Background())
cancelled := false
@@ -91,13 +91,13 @@ func Test_CancelBackup_FromDifferentInstance_BackupCancelledOnRunningInstance(t
cancel()
}
- manager1.RegisterBackup(backupID, wrappedCancel)
+ manager1.RegisterTask(taskID, wrappedCancel)
manager1.StartSubscription()
manager2.StartSubscription()
time.Sleep(100 * time.Millisecond)
- err := manager2.CancelBackup(backupID)
+ err := manager2.CancelTask(taskID)
assert.NoError(t, err, "Cancel should not return error")
time.Sleep(500 * time.Millisecond)
@@ -110,29 +110,29 @@ func Test_CancelBackup_FromDifferentInstance_BackupCancelledOnRunningInstance(t
assert.Error(t, ctx.Err(), "Context should be cancelled")
}
-func Test_CancelBackup_WhenBackupDoesNotExist_NoErrorReturned(t *testing.T) {
- manager := backupCancelManager
+func Test_CancelTask_WhenTaskDoesNotExist_NoErrorReturned(t *testing.T) {
+ manager := taskCancelManager
manager.StartSubscription()
time.Sleep(100 * time.Millisecond)
nonExistentID := uuid.New()
- err := manager.CancelBackup(nonExistentID)
- assert.NoError(t, err, "Cancelling non-existent backup should not error")
+ err := manager.CancelTask(nonExistentID)
+ assert.NoError(t, err, "Cancelling non-existent task should not error")
}
-func Test_CancelBackup_WithMultipleBackups_AllBackupsCancelled(t *testing.T) {
- manager := backupCancelManager
+func Test_CancelTask_WithMultipleTasks_AllTasksCancelled(t *testing.T) {
+ manager := taskCancelManager
- numBackups := 5
- backupIDs := make([]uuid.UUID, numBackups)
- contexts := make([]context.Context, numBackups)
- cancels := make([]context.CancelFunc, numBackups)
- cancelledFlags := make([]bool, numBackups)
+ numTasks := 5
+ taskIDs := make([]uuid.UUID, numTasks)
+ contexts := make([]context.Context, numTasks)
+ cancels := make([]context.CancelFunc, numTasks)
+ cancelledFlags := make([]bool, numTasks)
var mu sync.Mutex
- for i := 0; i < numBackups; i++ {
- backupIDs[i] = uuid.New()
+ for i := 0; i < numTasks; i++ {
+ taskIDs[i] = uuid.New()
contexts[i], cancels[i] = context.WithCancel(context.Background())
idx := i
@@ -143,31 +143,31 @@ func Test_CancelBackup_WithMultipleBackups_AllBackupsCancelled(t *testing.T) {
cancels[idx]()
}
- manager.RegisterBackup(backupIDs[i], wrappedCancel)
+ manager.RegisterTask(taskIDs[i], wrappedCancel)
}
manager.StartSubscription()
time.Sleep(100 * time.Millisecond)
- for i := 0; i < numBackups; i++ {
- err := manager.CancelBackup(backupIDs[i])
+ for i := 0; i < numTasks; i++ {
+ err := manager.CancelTask(taskIDs[i])
assert.NoError(t, err, "Cancel should not return error")
}
time.Sleep(1 * time.Second)
mu.Lock()
- for i := 0; i < numBackups; i++ {
- assert.True(t, cancelledFlags[i], "Backup %d should be cancelled", i)
+ for i := 0; i < numTasks; i++ {
+ assert.True(t, cancelledFlags[i], "Task %d should be cancelled", i)
assert.Error(t, contexts[i].Err(), "Context %d should be cancelled", i)
}
mu.Unlock()
}
-func Test_CancelBackup_AfterUnregister_BackupNotCancelled(t *testing.T) {
- manager := backupCancelManager
+func Test_CancelTask_AfterUnregister_TaskNotCancelled(t *testing.T) {
+ manager := taskCancelManager
- backupID := uuid.New()
+ taskID := uuid.New()
_, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -181,13 +181,13 @@ func Test_CancelBackup_AfterUnregister_BackupNotCancelled(t *testing.T) {
cancel()
}
- manager.RegisterBackup(backupID, wrappedCancel)
+ manager.RegisterTask(taskID, wrappedCancel)
manager.StartSubscription()
time.Sleep(100 * time.Millisecond)
- manager.UnregisterBackup(backupID)
+ manager.UnregisterTask(taskID)
- err := manager.CancelBackup(backupID)
+ err := manager.CancelTask(taskID)
assert.NoError(t, err, "Cancel should not return error")
time.Sleep(500 * time.Millisecond)
diff --git a/backend/internal/features/backups/backups/cancellation/di.go b/backend/internal/features/tasks/cancellation/di.go
similarity index 60%
rename from backend/internal/features/backups/backups/cancellation/di.go
rename to backend/internal/features/tasks/cancellation/di.go
index 24e32f73..5d1c83af 100644
--- a/backend/internal/features/backups/backups/cancellation/di.go
+++ b/backend/internal/features/tasks/cancellation/di.go
@@ -1,4 +1,4 @@
-package backups_cancellation
+package task_cancellation
import (
"context"
@@ -9,17 +9,17 @@ import (
"github.com/google/uuid"
)
-var backupCancelManager = &BackupCancelManager{
+var taskCancelManager = &TaskCancelManager{
sync.RWMutex{},
make(map[uuid.UUID]context.CancelFunc),
cache_utils.NewPubSubManager(),
logger.GetLogger(),
}
-func GetBackupCancelManager() *BackupCancelManager {
- return backupCancelManager
+func GetTaskCancelManager() *TaskCancelManager {
+ return taskCancelManager
}
func SetupDependencies() {
- backupCancelManager.StartSubscription()
+ taskCancelManager.StartSubscription()
}
diff --git a/backend/internal/features/tasks/registry/di.go b/backend/internal/features/tasks/registry/di.go
new file mode 100644
index 00000000..660ab71a
--- /dev/null
+++ b/backend/internal/features/tasks/registry/di.go
@@ -0,0 +1,18 @@
+package task_registry
+
+import (
+ cache_utils "databasus-backend/internal/util/cache"
+ "databasus-backend/internal/util/logger"
+)
+
+var taskNodesRegistry = &TaskNodesRegistry{
+ cache_utils.GetValkeyClient(),
+ logger.GetLogger(),
+ cache_utils.DefaultCacheTimeout,
+ cache_utils.NewPubSubManager(),
+ cache_utils.NewPubSubManager(),
+}
+
+func GetTaskNodesRegistry() *TaskNodesRegistry {
+ return taskNodesRegistry
+}
diff --git a/backend/internal/features/tasks/registry/dto.go b/backend/internal/features/tasks/registry/dto.go
new file mode 100644
index 00000000..59ce7cb3
--- /dev/null
+++ b/backend/internal/features/tasks/registry/dto.go
@@ -0,0 +1,29 @@
+package task_registry
+
+import (
+ "time"
+
+ "github.com/google/uuid"
+)
+
+type TaskNode struct {
+ ID uuid.UUID `json:"id"`
+ ThroughputMBs int `json:"throughputMBs"`
+ LastHeartbeat time.Time `json:"lastHeartbeat"`
+}
+
+type TaskNodeStats struct {
+ ID uuid.UUID `json:"id"`
+ ActiveTasks int `json:"activeTasks"`
+}
+
+type TaskSubmitMessage struct {
+ NodeID string `json:"nodeId"`
+ TaskID string `json:"taskId"`
+ IsCallNotifier bool `json:"isCallNotifier"`
+}
+
+type TaskCompletionMessage struct {
+ NodeID string `json:"nodeId"`
+ TaskID string `json:"taskId"`
+}
diff --git a/backend/internal/features/tasks/registry/registry.go b/backend/internal/features/tasks/registry/registry.go
new file mode 100644
index 00000000..e80cb877
--- /dev/null
+++ b/backend/internal/features/tasks/registry/registry.go
@@ -0,0 +1,641 @@
+package task_registry
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "strings"
+ "time"
+
+ cache_utils "databasus-backend/internal/util/cache"
+
+ "github.com/google/uuid"
+ "github.com/valkey-io/valkey-go"
+)
+
+const (
+ nodeInfoKeyPrefix = "node:"
+ nodeInfoKeySuffix = ":info"
+ nodeActiveTasksPrefix = "node:"
+ nodeActiveTasksSuffix = ":active_tasks"
+ taskSubmitChannel = "task:submit"
+ taskCompletionChannel = "task:completion"
+
+ deadNodeThreshold = 2 * time.Minute
+ cleanupTickerInterval = 1 * time.Second
+)
+
+// TaskNodesRegistry helps to sync tasks scheduler (backuping or restoring)
+// and task nodes which are used for network-intensive tasks processing
+//
+// Features:
+// - Track node availability and load level
+// - Assign from scheduler to node tasks needed to be processed
+// - Notify scheduler from node about task completion
+//
+// Important things to remember:
+// - Node can contain different tasks types so when task is assigned
+// or node's tasks cleaned - should be performed DB check in DB
+// that task with this ID exists for this task type at all
+// - Nodes without heathbeat for more than 2 minutes are not included
+// in available nodes list and stats
+//
+// Cleanup dead nodes performed on 2 levels:
+// - List and stats functions do not return dead nodes
+// - Periodically dead nodes are cleaned up in cache (to not
+// accumulate too many dead nodes in cache)
+type TaskNodesRegistry struct {
+ client valkey.Client
+ logger *slog.Logger
+ timeout time.Duration
+ pubsubTasks *cache_utils.PubSubManager
+ pubsubCompletions *cache_utils.PubSubManager
+}
+
+func (r *TaskNodesRegistry) Run(ctx context.Context) {
+ if err := r.cleanupDeadNodes(); err != nil {
+ r.logger.Error("Failed to cleanup dead nodes on startup", "error", err)
+ }
+
+ ticker := time.NewTicker(cleanupTickerInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if err := r.cleanupDeadNodes(); err != nil {
+ r.logger.Error("Failed to cleanup dead nodes", "error", err)
+ }
+ }
+ }
+}
+
+func (r *TaskNodesRegistry) GetAvailableNodes() ([]TaskNode, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ var allKeys []string
+ cursor := uint64(0)
+ pattern := nodeInfoKeyPrefix + "*" + nodeInfoKeySuffix
+
+ for {
+ result := r.client.Do(
+ ctx,
+ r.client.B().Scan().Cursor(cursor).Match(pattern).Count(1_000).Build(),
+ )
+
+ if result.Error() != nil {
+ return nil, fmt.Errorf("failed to scan node keys: %w", result.Error())
+ }
+
+ scanResult, err := result.AsScanEntry()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse scan result: %w", err)
+ }
+
+ allKeys = append(allKeys, scanResult.Elements...)
+
+ cursor = scanResult.Cursor
+ if cursor == 0 {
+ break
+ }
+ }
+
+ if len(allKeys) == 0 {
+ return []TaskNode{}, nil
+ }
+
+ keyDataMap, err := r.pipelineGetKeys(allKeys)
+ if err != nil {
+ return nil, fmt.Errorf("failed to pipeline get node keys: %w", err)
+ }
+
+ threshold := time.Now().UTC().Add(-deadNodeThreshold)
+ var nodes []TaskNode
+ for key, data := range keyDataMap {
+ // Skip if the key doesn't exist (data is empty)
+ if len(data) == 0 {
+ continue
+ }
+
+ var node TaskNode
+ if err := json.Unmarshal(data, &node); err != nil {
+ r.logger.Warn("Failed to unmarshal node data", "key", key, "error", err)
+ continue
+ }
+
+ // Skip nodes with zero/uninitialized heartbeat
+ if node.LastHeartbeat.IsZero() {
+ continue
+ }
+
+ if node.LastHeartbeat.Before(threshold) {
+ continue
+ }
+
+ nodes = append(nodes, node)
+ }
+
+ return nodes, nil
+}
+
+func (r *TaskNodesRegistry) GetNodesStats() ([]TaskNodeStats, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ var allKeys []string
+ cursor := uint64(0)
+ pattern := nodeActiveTasksPrefix + "*" + nodeActiveTasksSuffix
+
+ for {
+ result := r.client.Do(
+ ctx,
+ r.client.B().Scan().Cursor(cursor).Match(pattern).Count(100).Build(),
+ )
+
+ if result.Error() != nil {
+ return nil, fmt.Errorf("failed to scan active tasks keys: %w", result.Error())
+ }
+
+ scanResult, err := result.AsScanEntry()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse scan result: %w", err)
+ }
+
+ allKeys = append(allKeys, scanResult.Elements...)
+
+ cursor = scanResult.Cursor
+ if cursor == 0 {
+ break
+ }
+ }
+
+ if len(allKeys) == 0 {
+ return []TaskNodeStats{}, nil
+ }
+
+ keyDataMap, err := r.pipelineGetKeys(allKeys)
+ if err != nil {
+ return nil, fmt.Errorf("failed to pipeline get active tasks keys: %w", err)
+ }
+
+ var nodeInfoKeys []string
+ nodeIDToStatsKey := make(map[string]string)
+ for key := range keyDataMap {
+ nodeID := r.extractNodeIDFromKey(key, nodeActiveTasksPrefix, nodeActiveTasksSuffix)
+ nodeIDStr := nodeID.String()
+ infoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, nodeIDStr, nodeInfoKeySuffix)
+ nodeInfoKeys = append(nodeInfoKeys, infoKey)
+ nodeIDToStatsKey[infoKey] = key
+ }
+
+ nodeInfoMap, err := r.pipelineGetKeys(nodeInfoKeys)
+ if err != nil {
+ return nil, fmt.Errorf("failed to pipeline get node info keys: %w", err)
+ }
+
+ threshold := time.Now().UTC().Add(-deadNodeThreshold)
+ var stats []TaskNodeStats
+ for infoKey, nodeData := range nodeInfoMap {
+ // Skip if the info key doesn't exist (nodeData is empty)
+ if len(nodeData) == 0 {
+ continue
+ }
+
+ var node TaskNode
+ if err := json.Unmarshal(nodeData, &node); err != nil {
+ r.logger.Warn("Failed to unmarshal node data", "key", infoKey, "error", err)
+ continue
+ }
+
+ // Skip nodes with zero/uninitialized heartbeat
+ if node.LastHeartbeat.IsZero() {
+ continue
+ }
+
+ if node.LastHeartbeat.Before(threshold) {
+ continue
+ }
+
+ statsKey := nodeIDToStatsKey[infoKey]
+ tasksData := keyDataMap[statsKey]
+ count, err := r.parseIntFromBytes(tasksData)
+ if err != nil {
+ r.logger.Warn("Failed to parse active tasks count", "key", statsKey, "error", err)
+ continue
+ }
+
+ stat := TaskNodeStats{
+ ID: node.ID,
+ ActiveTasks: int(count),
+ }
+ stats = append(stats, stat)
+ }
+
+ return stats, nil
+}
+
+func (r *TaskNodesRegistry) IncrementTasksInProgress(nodeID string) error {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ key := fmt.Sprintf("%s%s%s", nodeActiveTasksPrefix, nodeID, nodeActiveTasksSuffix)
+ result := r.client.Do(ctx, r.client.B().Incr().Key(key).Build())
+
+ if result.Error() != nil {
+ return fmt.Errorf(
+ "failed to increment tasks in progress for node %s: %w",
+ nodeID,
+ result.Error(),
+ )
+ }
+
+ return nil
+}
+
+func (r *TaskNodesRegistry) DecrementTasksInProgress(nodeID string) error {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ key := fmt.Sprintf("%s%s%s", nodeActiveTasksPrefix, nodeID, nodeActiveTasksSuffix)
+ result := r.client.Do(ctx, r.client.B().Decr().Key(key).Build())
+
+ if result.Error() != nil {
+ return fmt.Errorf(
+ "failed to decrement tasks in progress for node %s: %w",
+ nodeID,
+ result.Error(),
+ )
+ }
+
+ newValue, err := result.AsInt64()
+ if err != nil {
+ return fmt.Errorf("failed to parse decremented value for node %s: %w", nodeID, err)
+ }
+
+ if newValue < 0 {
+ setCtx, setCancel := context.WithTimeout(context.Background(), r.timeout)
+ r.client.Do(setCtx, r.client.B().Set().Key(key).Value("0").Build())
+ setCancel()
+ r.logger.Warn("Active tasks counter went below 0, reset to 0", "nodeID", nodeID)
+ }
+
+ return nil
+}
+
+func (r *TaskNodesRegistry) HearthbeatNodeInRegistry(now time.Time, node TaskNode) error {
+ if now.IsZero() {
+ return fmt.Errorf("cannot register node with zero heartbeat timestamp")
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ node.LastHeartbeat = now
+
+ data, err := json.Marshal(node)
+ if err != nil {
+ return fmt.Errorf("failed to marshal node: %w", err)
+ }
+
+ key := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node.ID.String(), nodeInfoKeySuffix)
+ result := r.client.Do(
+ ctx,
+ r.client.B().Set().Key(key).Value(string(data)).Build(),
+ )
+
+ if result.Error() != nil {
+ return fmt.Errorf("failed to register node %s: %w", node.ID, result.Error())
+ }
+
+ return nil
+}
+
+func (r *TaskNodesRegistry) UnregisterNodeFromRegistry(node TaskNode) error {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ infoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node.ID.String(), nodeInfoKeySuffix)
+ counterKey := fmt.Sprintf(
+ "%s%s%s",
+ nodeActiveTasksPrefix,
+ node.ID.String(),
+ nodeActiveTasksSuffix,
+ )
+
+ result := r.client.Do(
+ ctx,
+ r.client.B().Del().Key(infoKey, counterKey).Build(),
+ )
+
+ if result.Error() != nil {
+ return fmt.Errorf("failed to unregister node %s: %w", node.ID, result.Error())
+ }
+
+ r.logger.Info("Unregistered node from registry", "nodeID", node.ID)
+ return nil
+}
+
+func (r *TaskNodesRegistry) AssignTaskToNode(
+ targetNodeID string,
+ taskID uuid.UUID,
+ isCallNotifier bool,
+) error {
+ ctx := context.Background()
+
+ message := TaskSubmitMessage{
+ NodeID: targetNodeID,
+ TaskID: taskID.String(),
+ IsCallNotifier: isCallNotifier,
+ }
+
+ messageJSON, err := json.Marshal(message)
+ if err != nil {
+ return fmt.Errorf("failed to marshal task submit message: %w", err)
+ }
+
+ err = r.pubsubTasks.Publish(ctx, taskSubmitChannel, string(messageJSON))
+ if err != nil {
+ return fmt.Errorf("failed to publish task submit message: %w", err)
+ }
+
+ return nil
+}
+
+func (r *TaskNodesRegistry) SubscribeNodeForTasksAssignment(
+ nodeID string,
+ handler func(taskID uuid.UUID, isCallNotifier bool),
+) error {
+ ctx := context.Background()
+
+ wrappedHandler := func(message string) {
+ var msg TaskSubmitMessage
+ if err := json.Unmarshal([]byte(message), &msg); err != nil {
+ r.logger.Warn("Failed to unmarshal task submit message", "error", err)
+ return
+ }
+
+ if msg.NodeID != nodeID {
+ return
+ }
+
+ taskID, err := uuid.Parse(msg.TaskID)
+ if err != nil {
+ r.logger.Warn(
+ "Failed to parse task ID from message",
+ "taskId",
+ msg.TaskID,
+ "error",
+ err,
+ )
+ return
+ }
+
+ handler(taskID, msg.IsCallNotifier)
+ }
+
+ err := r.pubsubTasks.Subscribe(ctx, taskSubmitChannel, wrappedHandler)
+ if err != nil {
+ return fmt.Errorf("failed to subscribe to task submit channel: %w", err)
+ }
+
+ r.logger.Info("Subscribed to task submit channel", "nodeID", nodeID)
+ return nil
+}
+
+func (r *TaskNodesRegistry) UnsubscribeNodeForTasksAssignments() error {
+ err := r.pubsubTasks.Close()
+ if err != nil {
+ return fmt.Errorf("failed to unsubscribe from task submit channel: %w", err)
+ }
+
+ r.logger.Info("Unsubscribed from task submit channel")
+ return nil
+}
+
+func (r *TaskNodesRegistry) PublishTaskCompletion(nodeID string, taskID uuid.UUID) error {
+ ctx := context.Background()
+
+ message := TaskCompletionMessage{
+ NodeID: nodeID,
+ TaskID: taskID.String(),
+ }
+
+ messageJSON, err := json.Marshal(message)
+ if err != nil {
+ return fmt.Errorf("failed to marshal task completion message: %w", err)
+ }
+
+ err = r.pubsubCompletions.Publish(ctx, taskCompletionChannel, string(messageJSON))
+ if err != nil {
+ return fmt.Errorf("failed to publish task completion message: %w", err)
+ }
+
+ return nil
+}
+
+func (r *TaskNodesRegistry) SubscribeForTasksCompletions(
+ handler func(nodeID string, taskID uuid.UUID),
+) error {
+ ctx := context.Background()
+
+ wrappedHandler := func(message string) {
+ var msg TaskCompletionMessage
+ if err := json.Unmarshal([]byte(message), &msg); err != nil {
+ r.logger.Warn("Failed to unmarshal task completion message", "error", err)
+ return
+ }
+
+ taskID, err := uuid.Parse(msg.TaskID)
+ if err != nil {
+ r.logger.Warn(
+ "Failed to parse task ID from completion message",
+ "taskId",
+ msg.TaskID,
+ "error",
+ err,
+ )
+ return
+ }
+
+ handler(msg.NodeID, taskID)
+ }
+
+ err := r.pubsubCompletions.Subscribe(ctx, taskCompletionChannel, wrappedHandler)
+ if err != nil {
+ return fmt.Errorf("failed to subscribe to task completion channel: %w", err)
+ }
+
+ r.logger.Info("Subscribed to task completion channel")
+ return nil
+}
+
+func (r *TaskNodesRegistry) UnsubscribeForTasksCompletions() error {
+ err := r.pubsubCompletions.Close()
+ if err != nil {
+ return fmt.Errorf("failed to unsubscribe from task completion channel: %w", err)
+ }
+
+ r.logger.Info("Unsubscribed from task completion channel")
+ return nil
+}
+
+func (r *TaskNodesRegistry) extractNodeIDFromKey(key, prefix, suffix string) uuid.UUID {
+ nodeIDStr := strings.TrimPrefix(key, prefix)
+ nodeIDStr = strings.TrimSuffix(nodeIDStr, suffix)
+
+ nodeID, err := uuid.Parse(nodeIDStr)
+ if err != nil {
+ r.logger.Warn("Failed to parse node ID from key", "key", key, "error", err)
+ return uuid.Nil
+ }
+
+ return nodeID
+}
+
+func (r *TaskNodesRegistry) pipelineGetKeys(keys []string) (map[string][]byte, error) {
+ if len(keys) == 0 {
+ return make(map[string][]byte), nil
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ commands := make([]valkey.Completed, 0, len(keys))
+ for _, key := range keys {
+ commands = append(commands, r.client.B().Get().Key(key).Build())
+ }
+
+ results := r.client.DoMulti(ctx, commands...)
+
+ keyDataMap := make(map[string][]byte, len(keys))
+ for i, result := range results {
+ if result.Error() != nil {
+ r.logger.Warn("Failed to get key in pipeline", "key", keys[i], "error", result.Error())
+ continue
+ }
+
+ data, err := result.AsBytes()
+ if err != nil {
+ r.logger.Warn("Failed to parse key data in pipeline", "key", keys[i], "error", err)
+ continue
+ }
+
+ keyDataMap[keys[i]] = data
+ }
+
+ return keyDataMap, nil
+}
+
+func (r *TaskNodesRegistry) parseIntFromBytes(data []byte) (int64, error) {
+ str := string(data)
+ var count int64
+ _, err := fmt.Sscanf(str, "%d", &count)
+ if err != nil {
+ return 0, fmt.Errorf("failed to parse integer from bytes: %w", err)
+ }
+ return count, nil
+}
+
+func (r *TaskNodesRegistry) cleanupDeadNodes() error {
+ ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
+ defer cancel()
+
+ var allKeys []string
+ cursor := uint64(0)
+ pattern := nodeInfoKeyPrefix + "*" + nodeInfoKeySuffix
+
+ for {
+ result := r.client.Do(
+ ctx,
+ r.client.B().Scan().Cursor(cursor).Match(pattern).Count(1_000).Build(),
+ )
+
+ if result.Error() != nil {
+ return fmt.Errorf("failed to scan node keys: %w", result.Error())
+ }
+
+ scanResult, err := result.AsScanEntry()
+ if err != nil {
+ return fmt.Errorf("failed to parse scan result: %w", err)
+ }
+
+ allKeys = append(allKeys, scanResult.Elements...)
+
+ cursor = scanResult.Cursor
+ if cursor == 0 {
+ break
+ }
+ }
+
+ if len(allKeys) == 0 {
+ return nil
+ }
+
+ keyDataMap, err := r.pipelineGetKeys(allKeys)
+ if err != nil {
+ return fmt.Errorf("failed to pipeline get node keys: %w", err)
+ }
+
+ threshold := time.Now().UTC().Add(-deadNodeThreshold)
+ var deadNodeKeys []string
+
+ for key, data := range keyDataMap {
+
+ // Skip if the key doesn't exist (data is empty)
+ if len(data) == 0 {
+ continue
+ }
+
+ var node TaskNode
+ if err := json.Unmarshal(data, &node); err != nil {
+ r.logger.Warn("Failed to unmarshal node data during cleanup", "key", key, "error", err)
+ continue
+ }
+
+ // Skip nodes with zero/uninitialized heartbeat
+ if node.LastHeartbeat.IsZero() {
+ continue
+ }
+
+ if node.LastHeartbeat.Before(threshold) {
+ nodeID := node.ID.String()
+ infoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, nodeID, nodeInfoKeySuffix)
+ statsKey := fmt.Sprintf("%s%s%s", nodeActiveTasksPrefix, nodeID, nodeActiveTasksSuffix)
+
+ deadNodeKeys = append(deadNodeKeys, infoKey, statsKey)
+ r.logger.Info(
+ "Marking node for cleanup",
+ "nodeID", nodeID,
+ "lastHeartbeat", node.LastHeartbeat,
+ "threshold", threshold,
+ )
+ }
+ }
+
+ if len(deadNodeKeys) == 0 {
+ return nil
+ }
+
+ delCtx, delCancel := context.WithTimeout(context.Background(), r.timeout)
+ defer delCancel()
+
+ result := r.client.Do(
+ delCtx,
+ r.client.B().Del().Key(deadNodeKeys...).Build(),
+ )
+
+ if result.Error() != nil {
+ return fmt.Errorf("failed to delete dead node keys: %w", result.Error())
+ }
+
+ deletedCount, err := result.AsInt64()
+ if err != nil {
+ return fmt.Errorf("failed to parse deleted count: %w", err)
+ }
+
+ r.logger.Info("Cleaned up dead nodes", "deletedKeysCount", deletedCount)
+ return nil
+}
diff --git a/backend/internal/features/tasks/registry/registry_test.go b/backend/internal/features/tasks/registry/registry_test.go
new file mode 100644
index 00000000..026bd9c6
--- /dev/null
+++ b/backend/internal/features/tasks/registry/registry_test.go
@@ -0,0 +1,1170 @@
+package task_registry
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "testing"
+ "time"
+
+ cache_utils "databasus-backend/internal/util/cache"
+ "databasus-backend/internal/util/logger"
+
+ "github.com/google/uuid"
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_HearthbeatNodeInRegistry_RegistersNodeWithTTL(t *testing.T) {
+ cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer cleanupTestNode(registry, node)
+ defer cache_utils.ClearAllCache()
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 1)
+ assert.Equal(t, node.ID, nodes[0].ID)
+ assert.Equal(t, node.ThroughputMBs, nodes[0].ThroughputMBs)
+}
+
+func Test_UnregisterNodeFromRegistry_RemovesNodeAndCounter(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ err = registry.UnregisterNodeFromRegistry(node)
+ assert.NoError(t, err)
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Empty(t, nodes)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Empty(t, stats)
+}
+
+func Test_GetAvailableNodes_ReturnsAllRegisteredNodes(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+ defer cleanupTestNode(registry, node3)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
+ assert.NoError(t, err)
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 3)
+
+ nodeIDs := make(map[uuid.UUID]bool)
+ for _, node := range nodes {
+ nodeIDs[node.ID] = true
+ }
+ assert.True(t, nodeIDs[node1.ID])
+ assert.True(t, nodeIDs[node2.ID])
+ assert.True(t, nodeIDs[node3.ID])
+}
+
+func Test_GetAvailableNodes_WhenNoNodesExist_ReturnsEmptySlice(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.NotNil(t, nodes)
+ assert.Empty(t, nodes)
+}
+
+func Test_IncrementTasksInProgress_IncrementsCounter(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer cleanupTestNode(registry, node)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 1)
+ assert.Equal(t, node.ID, stats[0].ID)
+ assert.Equal(t, 1, stats[0].ActiveTasks)
+
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err = registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 1)
+ assert.Equal(t, 2, stats[0].ActiveTasks)
+}
+
+func Test_DecrementTasksInProgress_DecrementsCounter(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer cleanupTestNode(registry, node)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Equal(t, 3, stats[0].ActiveTasks)
+
+ err = registry.DecrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err = registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Equal(t, 2, stats[0].ActiveTasks)
+
+ err = registry.DecrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err = registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Equal(t, 1, stats[0].ActiveTasks)
+}
+
+func Test_DecrementTasksInProgress_WhenNegative_ResetsToZero(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer cleanupTestNode(registry, node)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ err = registry.DecrementTasksInProgress(node.ID.String())
+ assert.NoError(t, err)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 1)
+ assert.Equal(t, 0, stats[0].ActiveTasks)
+}
+
+func Test_GetTaskNodesStats_ReturnsStatsForAllNodes(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+ defer cleanupTestNode(registry, node3)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node2.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node2.ID.String())
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node3.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node3.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node3.ID.String())
+ assert.NoError(t, err)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 3)
+
+ statsMap := make(map[uuid.UUID]int)
+ for _, stat := range stats {
+ statsMap[stat.ID] = stat.ActiveTasks
+ }
+
+ assert.Equal(t, 1, statsMap[node1.ID])
+ assert.Equal(t, 2, statsMap[node2.ID])
+ assert.Equal(t, 3, statsMap[node3.ID])
+}
+
+func Test_GetTaskNodesStats_WhenNoStats_ReturnsEmptySlice(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.NotNil(t, stats)
+ assert.Empty(t, stats)
+}
+
+func Test_MultipleNodes_RegisteredAndQueriedCorrectly(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node1.ThroughputMBs = 50
+ node2 := createTestTaskNode()
+ node2.ThroughputMBs = 100
+ node3 := createTestTaskNode()
+ node3.ThroughputMBs = 150
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+ defer cleanupTestNode(registry, node3)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
+ assert.NoError(t, err)
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 3)
+
+ nodeMap := make(map[uuid.UUID]TaskNode)
+ for _, node := range nodes {
+ nodeMap[node.ID] = node
+ }
+
+ assert.Equal(t, 50, nodeMap[node1.ID].ThroughputMBs)
+ assert.Equal(t, 100, nodeMap[node2.ID].ThroughputMBs)
+ assert.Equal(t, 150, nodeMap[node3.ID].ThroughputMBs)
+}
+
+func Test_TaskCounters_TrackedSeparatelyPerNode(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node2.ID.String())
+ assert.NoError(t, err)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 2)
+
+ statsMap := make(map[uuid.UUID]int)
+ for _, stat := range stats {
+ statsMap[stat.ID] = stat.ActiveTasks
+ }
+
+ assert.Equal(t, 2, statsMap[node1.ID])
+ assert.Equal(t, 1, statsMap[node2.ID])
+
+ err = registry.DecrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+
+ stats, err = registry.GetNodesStats()
+ assert.NoError(t, err)
+
+ statsMap = make(map[uuid.UUID]int)
+ for _, stat := range stats {
+ statsMap[stat.ID] = stat.ActiveTasks
+ }
+
+ assert.Equal(t, 1, statsMap[node1.ID])
+ assert.Equal(t, 1, statsMap[node2.ID])
+}
+
+func Test_GetAvailableNodes_SkipsInvalidJsonData(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer cleanupTestNode(registry, node)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer cancel()
+
+ invalidKey := nodeInfoKeyPrefix + uuid.New().String() + nodeInfoKeySuffix
+ registry.client.Do(
+ ctx,
+ registry.client.B().Set().Key(invalidKey).Value("invalid json data").Build(),
+ )
+ defer func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer cleanupCancel()
+ registry.client.Do(cleanupCtx, registry.client.B().Del().Key(invalidKey).Build())
+ }()
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 1)
+ assert.Equal(t, node.ID, nodes[0].ID)
+}
+
+func Test_PipelineGetKeys_HandlesEmptyKeysList(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+
+ keyDataMap, err := registry.pipelineGetKeys([]string{})
+ assert.NoError(t, err)
+ assert.NotNil(t, keyDataMap)
+ assert.Empty(t, keyDataMap)
+}
+
+func Test_HearthbeatNodeInRegistry_UpdatesLastHeartbeat(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ originalHeartbeat := node.LastHeartbeat
+ defer cleanupTestNode(registry, node)
+
+ time.Sleep(10 * time.Millisecond)
+
+ node.LastHeartbeat = time.Now().UTC()
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node)
+ assert.NoError(t, err)
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 1)
+ assert.True(t, nodes[0].LastHeartbeat.After(originalHeartbeat))
+}
+
+func Test_HearthbeatNodeInRegistry_RejectsZeroTimestamp(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+
+ err := registry.HearthbeatNodeInRegistry(time.Time{}, node)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "zero heartbeat timestamp")
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 0)
+}
+
+func createTestRegistry() *TaskNodesRegistry {
+ return &TaskNodesRegistry{
+ cache_utils.GetValkeyClient(),
+ logger.GetLogger(),
+ cache_utils.DefaultCacheTimeout,
+ cache_utils.NewPubSubManager(),
+ cache_utils.NewPubSubManager(),
+ }
+}
+
+func createTestTaskNode() TaskNode {
+ return TaskNode{
+ ID: uuid.New(),
+ ThroughputMBs: 100,
+ LastHeartbeat: time.Now().UTC(),
+ }
+}
+
+func cleanupTestNode(registry *TaskNodesRegistry, node TaskNode) {
+ registry.UnregisterNodeFromRegistry(node)
+}
+
+func Test_AssignTaskToNode_PublishesJsonMessageToChannel(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID := uuid.New()
+
+ err := registry.AssignTaskToNode(node.ID.String(), taskID, true)
+ assert.NoError(t, err)
+}
+
+func Test_SubscribeNodeForTasksAssignment_ReceivesSubmittedTasksForMatchingNode(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID := uuid.New()
+ defer registry.UnsubscribeNodeForTasksAssignments()
+
+ receivedTaskID := make(chan uuid.UUID, 1)
+ handler := func(id uuid.UUID, isCallNotifier bool) {
+ receivedTaskID <- id
+ }
+
+ err := registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.AssignTaskToNode(node.ID.String(), taskID, true)
+ assert.NoError(t, err)
+
+ select {
+ case received := <-receivedTaskID:
+ assert.Equal(t, taskID, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Timeout waiting for task message")
+ }
+}
+
+func Test_SubscribeNodeForTasksAssignment_FiltersOutTasksForDifferentNode(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ taskID := uuid.New()
+ defer registry.UnsubscribeNodeForTasksAssignments()
+
+ receivedTaskID := make(chan uuid.UUID, 1)
+ handler := func(id uuid.UUID, isCallNotifier bool) {
+ receivedTaskID <- id
+ }
+
+ err := registry.SubscribeNodeForTasksAssignment(node1.ID.String(), handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.AssignTaskToNode(node2.ID.String(), taskID, false)
+ assert.NoError(t, err)
+
+ select {
+ case <-receivedTaskID:
+ t.Fatal("Should not receive task for different node")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func Test_SubscribeNodeForTasksAssignment_ParsesJsonAndTaskIdCorrectly(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+ defer registry.UnsubscribeNodeForTasksAssignments()
+
+ receivedTasks := make(chan uuid.UUID, 2)
+ handler := func(id uuid.UUID, isCallNotifier bool) {
+ receivedTasks <- id
+ }
+
+ err := registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.AssignTaskToNode(node.ID.String(), taskID1, true)
+ assert.NoError(t, err)
+
+ err = registry.AssignTaskToNode(node.ID.String(), taskID2, false)
+ assert.NoError(t, err)
+
+ received1 := <-receivedTasks
+ received2 := <-receivedTasks
+
+ receivedIDs := []uuid.UUID{received1, received2}
+ assert.Contains(t, receivedIDs, taskID1)
+ assert.Contains(t, receivedIDs, taskID2)
+}
+
+func Test_SubscribeNodeForTasksAssignment_HandlesInvalidJson(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer registry.UnsubscribeNodeForTasksAssignments()
+
+ receivedTaskID := make(chan uuid.UUID, 1)
+ handler := func(id uuid.UUID, isCallNotifier bool) {
+ receivedTaskID <- id
+ }
+
+ err := registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ ctx := context.Background()
+ err = registry.pubsubTasks.Publish(ctx, "backup:submit", "invalid json")
+ assert.NoError(t, err)
+
+ select {
+ case <-receivedTaskID:
+ t.Fatal("Should not receive task for invalid JSON")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func Test_UnsubscribeNodeForTasksAssignments_StopsReceivingMessages(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+
+ receivedTaskID := make(chan uuid.UUID, 2)
+ handler := func(id uuid.UUID, isCallNotifier bool) {
+ receivedTaskID <- id
+ }
+
+ err := registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.AssignTaskToNode(node.ID.String(), taskID1, true)
+ assert.NoError(t, err)
+
+ received := <-receivedTaskID
+ assert.Equal(t, taskID1, received)
+
+ err = registry.UnsubscribeNodeForTasksAssignments()
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.AssignTaskToNode(node.ID.String(), taskID2, false)
+ assert.NoError(t, err)
+
+ select {
+ case <-receivedTaskID:
+ t.Fatal("Should not receive task after unsubscribe")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func Test_SubscribeNodeForTasksAssignment_WhenAlreadySubscribed_ReturnsError(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ defer registry.UnsubscribeNodeForTasksAssignments()
+
+ handler := func(id uuid.UUID, isCallNotifier bool) {}
+
+ err := registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.NoError(t, err)
+
+ err = registry.SubscribeNodeForTasksAssignment(node.ID.String(), handler)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "already subscribed")
+}
+
+func Test_MultipleNodes_EachReceivesOnlyTheirTasks(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry1 := createTestRegistry()
+ registry2 := createTestRegistry()
+ registry3 := createTestRegistry()
+
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+ taskID3 := uuid.New()
+
+ defer registry1.UnsubscribeNodeForTasksAssignments()
+ defer registry2.UnsubscribeNodeForTasksAssignments()
+ defer registry3.UnsubscribeNodeForTasksAssignments()
+ defer cleanupTestNode(registry1, node1)
+ defer cleanupTestNode(registry1, node2)
+ defer cleanupTestNode(registry1, node3)
+
+ receivedTasks1 := make(chan uuid.UUID, 3)
+ receivedTasks2 := make(chan uuid.UUID, 3)
+ receivedTasks3 := make(chan uuid.UUID, 3)
+
+ handler1 := func(id uuid.UUID, isCallNotifier bool) { receivedTasks1 <- id }
+ handler2 := func(id uuid.UUID, isCallNotifier bool) { receivedTasks2 <- id }
+ handler3 := func(id uuid.UUID, isCallNotifier bool) { receivedTasks3 <- id }
+
+ err := registry1.SubscribeNodeForTasksAssignment(node1.ID.String(), handler1)
+ assert.NoError(t, err)
+
+ err = registry2.SubscribeNodeForTasksAssignment(node2.ID.String(), handler2)
+ assert.NoError(t, err)
+
+ err = registry3.SubscribeNodeForTasksAssignment(node3.ID.String(), handler3)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ submitRegistry := createTestRegistry()
+ err = submitRegistry.AssignTaskToNode(node1.ID.String(), taskID1, true)
+ assert.NoError(t, err)
+
+ err = submitRegistry.AssignTaskToNode(node2.ID.String(), taskID2, false)
+ assert.NoError(t, err)
+
+ err = submitRegistry.AssignTaskToNode(node3.ID.String(), taskID3, true)
+ assert.NoError(t, err)
+
+ select {
+ case received := <-receivedTasks1:
+ assert.Equal(t, taskID1, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Node 1 timeout waiting for task message")
+ }
+
+ select {
+ case received := <-receivedTasks2:
+ assert.Equal(t, taskID2, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Node 2 timeout waiting for task message")
+ }
+
+ select {
+ case received := <-receivedTasks3:
+ assert.Equal(t, taskID3, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Node 3 timeout waiting for task message")
+ }
+
+ select {
+ case <-receivedTasks1:
+ t.Fatal("Node 1 should not receive additional tasks")
+ case <-time.After(300 * time.Millisecond):
+ }
+
+ select {
+ case <-receivedTasks2:
+ t.Fatal("Node 2 should not receive additional tasks")
+ case <-time.After(300 * time.Millisecond):
+ }
+
+ select {
+ case <-receivedTasks3:
+ t.Fatal("Node 3 should not receive additional tasks")
+ case <-time.After(300 * time.Millisecond):
+ }
+}
+
+func Test_PublishTaskCompletion_PublishesMessageToChannel(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID := uuid.New()
+
+ err := registry.PublishTaskCompletion(node.ID.String(), taskID)
+ assert.NoError(t, err)
+}
+
+func Test_SubscribeForTasksCompletions_ReceivesCompletedTasks(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID := uuid.New()
+ defer registry.UnsubscribeForTasksCompletions()
+
+ receivedTaskID := make(chan uuid.UUID, 1)
+ receivedNodeID := make(chan string, 1)
+ handler := func(nodeID string, taskID uuid.UUID) {
+ receivedNodeID <- nodeID
+ receivedTaskID <- taskID
+ }
+
+ err := registry.SubscribeForTasksCompletions(handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.PublishTaskCompletion(node.ID.String(), taskID)
+ assert.NoError(t, err)
+
+ select {
+ case receivedNode := <-receivedNodeID:
+ assert.Equal(t, node.ID.String(), receivedNode)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Timeout waiting for node ID")
+ }
+
+ select {
+ case received := <-receivedTaskID:
+ assert.Equal(t, taskID, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Timeout waiting for task completion message")
+ }
+}
+
+func Test_SubscribeForTasksCompletions_ParsesJsonCorrectly(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+ defer registry.UnsubscribeForTasksCompletions()
+
+ receivedTasks := make(chan uuid.UUID, 2)
+ handler := func(nodeID string, taskID uuid.UUID) {
+ receivedTasks <- taskID
+ }
+
+ err := registry.SubscribeForTasksCompletions(handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.PublishTaskCompletion(node.ID.String(), taskID1)
+ assert.NoError(t, err)
+
+ err = registry.PublishTaskCompletion(node.ID.String(), taskID2)
+ assert.NoError(t, err)
+
+ received1 := <-receivedTasks
+ received2 := <-receivedTasks
+
+ receivedIDs := []uuid.UUID{received1, received2}
+ assert.Contains(t, receivedIDs, taskID1)
+ assert.Contains(t, receivedIDs, taskID2)
+}
+
+func Test_SubscribeForTasksCompletions_HandlesInvalidJson(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ defer registry.UnsubscribeForTasksCompletions()
+
+ receivedTaskID := make(chan uuid.UUID, 1)
+ handler := func(nodeID string, taskID uuid.UUID) {
+ receivedTaskID <- taskID
+ }
+
+ err := registry.SubscribeForTasksCompletions(handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ ctx := context.Background()
+ err = registry.pubsubCompletions.Publish(ctx, "backup:completion", "invalid json")
+ assert.NoError(t, err)
+
+ select {
+ case <-receivedTaskID:
+ t.Fatal("Should not receive task for invalid JSON")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func Test_UnsubscribeForTasksCompletions_StopsReceivingMessages(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node := createTestTaskNode()
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+
+ receivedTaskID := make(chan uuid.UUID, 2)
+ handler := func(nodeID string, taskID uuid.UUID) {
+ receivedTaskID <- taskID
+ }
+
+ err := registry.SubscribeForTasksCompletions(handler)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.PublishTaskCompletion(node.ID.String(), taskID1)
+ assert.NoError(t, err)
+
+ received := <-receivedTaskID
+ assert.Equal(t, taskID1, received)
+
+ err = registry.UnsubscribeForTasksCompletions()
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ err = registry.PublishTaskCompletion(node.ID.String(), taskID2)
+ assert.NoError(t, err)
+
+ select {
+ case <-receivedTaskID:
+ t.Fatal("Should not receive task after unsubscribe")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func Test_SubscribeForTasksCompletions_WhenAlreadySubscribed_ReturnsError(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ defer registry.UnsubscribeForTasksCompletions()
+
+ handler := func(nodeID string, taskID uuid.UUID) {}
+
+ err := registry.SubscribeForTasksCompletions(handler)
+ assert.NoError(t, err)
+
+ err = registry.SubscribeForTasksCompletions(handler)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "already subscribed")
+}
+
+func Test_MultipleSubscribers_EachReceivesCompletionMessages(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry1 := createTestRegistry()
+ registry2 := createTestRegistry()
+ registry3 := createTestRegistry()
+
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+
+ taskID1 := uuid.New()
+ taskID2 := uuid.New()
+ taskID3 := uuid.New()
+
+ defer registry1.UnsubscribeForTasksCompletions()
+ defer registry2.UnsubscribeForTasksCompletions()
+ defer registry3.UnsubscribeForTasksCompletions()
+ defer cleanupTestNode(registry1, node1)
+ defer cleanupTestNode(registry1, node2)
+ defer cleanupTestNode(registry1, node3)
+
+ receivedTasks1 := make(chan uuid.UUID, 3)
+ receivedTasks2 := make(chan uuid.UUID, 3)
+ receivedTasks3 := make(chan uuid.UUID, 3)
+
+ handler1 := func(nodeID string, taskID uuid.UUID) { receivedTasks1 <- taskID }
+ handler2 := func(nodeID string, taskID uuid.UUID) { receivedTasks2 <- taskID }
+ handler3 := func(nodeID string, taskID uuid.UUID) { receivedTasks3 <- taskID }
+
+ err := registry1.SubscribeForTasksCompletions(handler1)
+ assert.NoError(t, err)
+
+ err = registry2.SubscribeForTasksCompletions(handler2)
+ assert.NoError(t, err)
+
+ err = registry3.SubscribeForTasksCompletions(handler3)
+ assert.NoError(t, err)
+
+ time.Sleep(100 * time.Millisecond)
+
+ publishRegistry := createTestRegistry()
+ err = publishRegistry.PublishTaskCompletion(node1.ID.String(), taskID1)
+ assert.NoError(t, err)
+
+ err = publishRegistry.PublishTaskCompletion(node2.ID.String(), taskID2)
+ assert.NoError(t, err)
+
+ err = publishRegistry.PublishTaskCompletion(node3.ID.String(), taskID3)
+ assert.NoError(t, err)
+
+ receivedAll1 := []uuid.UUID{}
+ receivedAll2 := []uuid.UUID{}
+ receivedAll3 := []uuid.UUID{}
+
+ for i := 0; i < 3; i++ {
+ select {
+ case received := <-receivedTasks1:
+ receivedAll1 = append(receivedAll1, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Subscriber 1 timeout waiting for completion message")
+ }
+ }
+
+ for i := 0; i < 3; i++ {
+ select {
+ case received := <-receivedTasks2:
+ receivedAll2 = append(receivedAll2, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Subscriber 2 timeout waiting for completion message")
+ }
+ }
+
+ for i := 0; i < 3; i++ {
+ select {
+ case received := <-receivedTasks3:
+ receivedAll3 = append(receivedAll3, received)
+ case <-time.After(2 * time.Second):
+ t.Fatal("Subscriber 3 timeout waiting for completion message")
+ }
+ }
+
+ assert.Contains(t, receivedAll1, taskID1)
+ assert.Contains(t, receivedAll1, taskID2)
+ assert.Contains(t, receivedAll1, taskID3)
+
+ assert.Contains(t, receivedAll2, taskID1)
+ assert.Contains(t, receivedAll2, taskID2)
+ assert.Contains(t, receivedAll2, taskID3)
+
+ assert.Contains(t, receivedAll3, taskID1)
+ assert.Contains(t, receivedAll3, taskID2)
+ assert.Contains(t, receivedAll3, taskID3)
+}
+
+func Test_GetAvailableNodes_ExcludesStaleNodesFromCache(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+ defer cleanupTestNode(registry, node3)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer cancel()
+
+ key := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node2.ID.String(), nodeInfoKeySuffix)
+ result := registry.client.Do(ctx, registry.client.B().Get().Key(key).Build())
+ assert.NoError(t, result.Error())
+
+ data, err := result.AsBytes()
+ assert.NoError(t, err)
+
+ var node TaskNode
+ err = json.Unmarshal(data, &node)
+ assert.NoError(t, err)
+
+ node.LastHeartbeat = time.Now().UTC().Add(-3 * time.Minute)
+ modifiedData, err := json.Marshal(node)
+ assert.NoError(t, err)
+
+ setCtx, setCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer setCancel()
+ setResult := registry.client.Do(
+ setCtx,
+ registry.client.B().Set().Key(key).Value(string(modifiedData)).Build(),
+ )
+ assert.NoError(t, setResult.Error())
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 2)
+
+ nodeIDs := make(map[uuid.UUID]bool)
+ for _, n := range nodes {
+ nodeIDs[n.ID] = true
+ }
+ assert.True(t, nodeIDs[node1.ID])
+ assert.False(t, nodeIDs[node2.ID])
+ assert.True(t, nodeIDs[node3.ID])
+}
+
+func Test_GetNodesStats_ExcludesStaleNodesFromCache(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ node3 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+ defer cleanupTestNode(registry, node3)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node3)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node2.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node3.ID.String())
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer cancel()
+
+ key := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node2.ID.String(), nodeInfoKeySuffix)
+ result := registry.client.Do(ctx, registry.client.B().Get().Key(key).Build())
+ assert.NoError(t, result.Error())
+
+ data, err := result.AsBytes()
+ assert.NoError(t, err)
+
+ var node TaskNode
+ err = json.Unmarshal(data, &node)
+ assert.NoError(t, err)
+
+ node.LastHeartbeat = time.Now().UTC().Add(-3 * time.Minute)
+ modifiedData, err := json.Marshal(node)
+ assert.NoError(t, err)
+
+ setCtx, setCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer setCancel()
+ setResult := registry.client.Do(
+ setCtx,
+ registry.client.B().Set().Key(key).Value(string(modifiedData)).Build(),
+ )
+ assert.NoError(t, setResult.Error())
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 2)
+
+ statsMap := make(map[uuid.UUID]int)
+ for _, stat := range stats {
+ statsMap[stat.ID] = stat.ActiveTasks
+ }
+
+ assert.Equal(t, 1, statsMap[node1.ID])
+ _, hasNode2 := statsMap[node2.ID]
+ assert.False(t, hasNode2)
+ assert.Equal(t, 1, statsMap[node3.ID])
+}
+
+func Test_CleanupDeadNodes_RemovesNodeInfoAndCounter(t *testing.T) {
+ cache_utils.ClearAllCache()
+ defer cache_utils.ClearAllCache()
+
+ registry := createTestRegistry()
+ node1 := createTestTaskNode()
+ node2 := createTestTaskNode()
+ defer cleanupTestNode(registry, node1)
+ defer cleanupTestNode(registry, node2)
+
+ err := registry.HearthbeatNodeInRegistry(time.Now().UTC(), node1)
+ assert.NoError(t, err)
+ err = registry.HearthbeatNodeInRegistry(time.Now().UTC(), node2)
+ assert.NoError(t, err)
+
+ err = registry.IncrementTasksInProgress(node1.ID.String())
+ assert.NoError(t, err)
+ err = registry.IncrementTasksInProgress(node2.ID.String())
+ assert.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer cancel()
+
+ key := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node2.ID.String(), nodeInfoKeySuffix)
+ result := registry.client.Do(ctx, registry.client.B().Get().Key(key).Build())
+ assert.NoError(t, result.Error())
+
+ data, err := result.AsBytes()
+ assert.NoError(t, err)
+
+ var node TaskNode
+ err = json.Unmarshal(data, &node)
+ assert.NoError(t, err)
+
+ node.LastHeartbeat = time.Now().UTC().Add(-3 * time.Minute)
+ modifiedData, err := json.Marshal(node)
+ assert.NoError(t, err)
+
+ setCtx, setCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer setCancel()
+ setResult := registry.client.Do(
+ setCtx,
+ registry.client.B().Set().Key(key).Value(string(modifiedData)).Build(),
+ )
+ assert.NoError(t, setResult.Error())
+
+ err = registry.cleanupDeadNodes()
+ assert.NoError(t, err)
+
+ checkCtx, checkCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer checkCancel()
+
+ infoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node2.ID.String(), nodeInfoKeySuffix)
+ infoResult := registry.client.Do(checkCtx, registry.client.B().Get().Key(infoKey).Build())
+ assert.Error(t, infoResult.Error())
+
+ counterKey := fmt.Sprintf(
+ "%s%s%s",
+ nodeActiveTasksPrefix,
+ node2.ID.String(),
+ nodeActiveTasksSuffix,
+ )
+ counterCtx, counterCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer counterCancel()
+ counterResult := registry.client.Do(
+ counterCtx,
+ registry.client.B().Get().Key(counterKey).Build(),
+ )
+ assert.Error(t, counterResult.Error())
+
+ activeInfoKey := fmt.Sprintf("%s%s%s", nodeInfoKeyPrefix, node1.ID.String(), nodeInfoKeySuffix)
+ activeCtx, activeCancel := context.WithTimeout(context.Background(), registry.timeout)
+ defer activeCancel()
+ activeResult := registry.client.Do(
+ activeCtx,
+ registry.client.B().Get().Key(activeInfoKey).Build(),
+ )
+ assert.NoError(t, activeResult.Error())
+
+ nodes, err := registry.GetAvailableNodes()
+ assert.NoError(t, err)
+ assert.Len(t, nodes, 1)
+ assert.Equal(t, node1.ID, nodes[0].ID)
+
+ stats, err := registry.GetNodesStats()
+ assert.NoError(t, err)
+ assert.Len(t, stats, 1)
+ assert.Equal(t, node1.ID, stats[0].ID)
+}
diff --git a/backend/internal/util/cache/cache_test.go b/backend/internal/util/cache/cache_test.go
new file mode 100644
index 00000000..4075a05e
--- /dev/null
+++ b/backend/internal/util/cache/cache_test.go
@@ -0,0 +1,51 @@
+package cache_utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_ClearAllCache_AfterClear_CacheIsEmpty(t *testing.T) {
+ client := getCache()
+
+ // Arrange: Set up multiple cache entries with different prefixes
+ testKeys := []struct {
+ prefix string
+ key string
+ value string
+ }{
+ {"test:user:", "user1", "John Doe"},
+ {"test:user:", "user2", "Jane Smith"},
+ {"test:session:", "session1", "abc123"},
+ {"test:session:", "session2", "def456"},
+ {"test:data:", "item1", "value1"},
+ }
+
+ // Set all test keys
+ for _, tk := range testKeys {
+ cacheUtil := NewCacheUtil[string](client, tk.prefix)
+ cacheUtil.Set(tk.key, &tk.value)
+ }
+
+ // Verify keys were set correctly before clearing
+ for _, tk := range testKeys {
+ cacheUtil := NewCacheUtil[string](client, tk.prefix)
+ retrieved := cacheUtil.Get(tk.key)
+ assert.NotNil(t, retrieved, "Key %s should exist before clearing", tk.prefix+tk.key)
+ assert.Equal(t, tk.value, *retrieved, "Retrieved value should match set value")
+ }
+
+ // Act: Clear all cache
+ err := ClearAllCache()
+
+ // Assert: No error returned
+ assert.NoError(t, err, "ClearAllCache should not return an error")
+
+ // Assert: All keys should be deleted
+ for _, tk := range testKeys {
+ cacheUtil := NewCacheUtil[string](client, tk.prefix)
+ retrieved := cacheUtil.Get(tk.key)
+ assert.Nil(t, retrieved, "Key %s should be deleted after clearing", tk.prefix+tk.key)
+ }
+}
diff --git a/backend/internal/util/tools/mariadb.go b/backend/internal/util/tools/mariadb.go
index 8cfd0141..19ac95cf 100644
--- a/backend/internal/util/tools/mariadb.go
+++ b/backend/internal/util/tools/mariadb.go
@@ -87,17 +87,20 @@ func VerifyMariadbInstallation(
logger *slog.Logger,
envMode env_utils.EnvMode,
mariadbInstallDir string,
+ isShowLogs bool,
) {
clientVersions := []MariadbClientVersion{MariadbClientLegacy, MariadbClientModern}
for _, clientVersion := range clientVersions {
binDir := getMariadbBasePath(clientVersion, envMode, mariadbInstallDir)
- logger.Info(
- "Verifying MariaDB installation",
- "clientVersion", clientVersion,
- "path", binDir,
- )
+ if isShowLogs {
+ logger.Info(
+ "Verifying MariaDB installation",
+ "clientVersion", clientVersion,
+ "path", binDir,
+ )
+ }
if _, err := os.Stat(binDir); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -133,12 +136,14 @@ func VerifyMariadbInstallation(
}
cmdPath := GetMariadbExecutable(cmd, dummyServerVersion, envMode, mariadbInstallDir)
- logger.Info(
- "Checking for MariaDB command",
- "clientVersion", clientVersion,
- "command", cmd,
- "path", cmdPath,
- )
+ if isShowLogs {
+ logger.Info(
+ "Checking for MariaDB command",
+ "clientVersion", clientVersion,
+ "command", cmd,
+ "path", cmdPath,
+ )
+ }
if _, err := os.Stat(cmdPath); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -162,11 +167,15 @@ func VerifyMariadbInstallation(
continue
}
- logger.Info("MariaDB command found", "clientVersion", clientVersion, "command", cmd)
+ if isShowLogs {
+ logger.Info("MariaDB command found", "clientVersion", clientVersion, "command", cmd)
+ }
}
}
- logger.Info("MariaDB client tools verification completed!")
+ if isShowLogs {
+ logger.Info("MariaDB client tools verification completed!")
+ }
}
// IsMariadbBackupVersionHigherThanRestoreVersion checks if backup was made with
diff --git a/backend/internal/util/tools/mongodb.go b/backend/internal/util/tools/mongodb.go
index 24b72313..ed5eac1d 100644
--- a/backend/internal/util/tools/mongodb.go
+++ b/backend/internal/util/tools/mongodb.go
@@ -53,13 +53,16 @@ func VerifyMongodbInstallation(
logger *slog.Logger,
envMode env_utils.EnvMode,
mongodbInstallDir string,
+ isShowLogs bool,
) {
binDir := getMongodbBasePath(envMode, mongodbInstallDir)
- logger.Info(
- "Verifying MongoDB Database Tools installation",
- "path", binDir,
- )
+ if isShowLogs {
+ logger.Info(
+ "Verifying MongoDB Database Tools installation",
+ "path", binDir,
+ )
+ }
if _, err := os.Stat(binDir); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -85,11 +88,13 @@ func VerifyMongodbInstallation(
for _, cmd := range requiredCommands {
cmdPath := GetMongodbExecutable(cmd, envMode, mongodbInstallDir)
- logger.Info(
- "Checking for MongoDB command",
- "command", cmd,
- "path", cmdPath,
- )
+ if isShowLogs {
+ logger.Info(
+ "Checking for MongoDB command",
+ "command", cmd,
+ "path", cmdPath,
+ )
+ }
if _, err := os.Stat(cmdPath); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -110,10 +115,14 @@ func VerifyMongodbInstallation(
continue
}
- logger.Info("MongoDB command found", "command", cmd)
+ if isShowLogs {
+ logger.Info("MongoDB command found", "command", cmd)
+ }
}
- logger.Info("MongoDB Database Tools verification completed!")
+ if isShowLogs {
+ logger.Info("MongoDB Database Tools verification completed!")
+ }
}
// IsMongodbBackupVersionHigherThanRestoreVersion checks if backup was made with
diff --git a/backend/internal/util/tools/mysql.go b/backend/internal/util/tools/mysql.go
index 95e25e86..db601439 100644
--- a/backend/internal/util/tools/mysql.go
+++ b/backend/internal/util/tools/mysql.go
@@ -55,6 +55,7 @@ func VerifyMysqlInstallation(
logger *slog.Logger,
envMode env_utils.EnvMode,
mysqlInstallDir string,
+ isShowLogs bool,
) {
versions := []MysqlVersion{
MysqlVersion57,
@@ -71,13 +72,15 @@ func VerifyMysqlInstallation(
for _, version := range versions {
binDir := getMysqlBasePath(version, envMode, mysqlInstallDir)
- logger.Info(
- "Verifying MySQL installation",
- "version",
- string(version),
- "path",
- binDir,
- )
+ if isShowLogs {
+ logger.Info(
+ "Verifying MySQL installation",
+ "version",
+ string(version),
+ "path",
+ binDir,
+ )
+ }
if _, err := os.Stat(binDir); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -108,15 +111,17 @@ func VerifyMysqlInstallation(
mysqlInstallDir,
)
- logger.Info(
- "Checking for MySQL command",
- "command",
- cmd,
- "version",
- string(version),
- "path",
- cmdPath,
- )
+ if isShowLogs {
+ logger.Info(
+ "Checking for MySQL command",
+ "command",
+ cmd,
+ "version",
+ string(version),
+ "path",
+ cmdPath,
+ )
+ }
if _, err := os.Stat(cmdPath); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -143,25 +148,31 @@ func VerifyMysqlInstallation(
continue
}
+ if isShowLogs {
+ logger.Info(
+ "MySQL command found",
+ "command",
+ cmd,
+ "version",
+ string(version),
+ )
+ }
+ }
+
+ if isShowLogs {
logger.Info(
- "MySQL command found",
- "command",
- cmd,
+ "Installation of MySQL verified",
"version",
string(version),
+ "path",
+ binDir,
)
}
-
- logger.Info(
- "Installation of MySQL verified",
- "version",
- string(version),
- "path",
- binDir,
- )
}
- logger.Info("MySQL version-specific client tools verification completed!")
+ if isShowLogs {
+ logger.Info("MySQL version-specific client tools verification completed!")
+ }
}
// IsMysqlBackupVersionHigherThanRestoreVersion checks if backup was made with
diff --git a/backend/internal/util/tools/postgresql.go b/backend/internal/util/tools/postgresql.go
index e3bb96b4..d4bf4566 100644
--- a/backend/internal/util/tools/postgresql.go
+++ b/backend/internal/util/tools/postgresql.go
@@ -40,6 +40,7 @@ func VerifyPostgresesInstallation(
logger *slog.Logger,
envMode env_utils.EnvMode,
postgresesInstallDir string,
+ isShowLogs bool,
) {
versions := []PostgresqlVersion{
PostgresqlVersion12,
@@ -59,13 +60,15 @@ func VerifyPostgresesInstallation(
for _, version := range versions {
binDir := getPostgresqlBasePath(version, envMode, postgresesInstallDir)
- logger.Info(
- "Verifying PostgreSQL installation",
- "version",
- string(version),
- "path",
- binDir,
- )
+ if isShowLogs {
+ logger.Info(
+ "Verifying PostgreSQL installation",
+ "version",
+ string(version),
+ "path",
+ binDir,
+ )
+ }
if _, err := os.Stat(binDir); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -96,15 +99,17 @@ func VerifyPostgresesInstallation(
postgresesInstallDir,
)
- logger.Info(
- "Checking for PostgreSQL command",
- "command",
- cmd,
- "version",
- string(version),
- "path",
- cmdPath,
- )
+ if isShowLogs {
+ logger.Info(
+ "Checking for PostgreSQL command",
+ "command",
+ cmd,
+ "version",
+ string(version),
+ "path",
+ cmdPath,
+ )
+ }
if _, err := os.Stat(cmdPath); os.IsNotExist(err) {
if envMode == env_utils.EnvModeDevelopment {
@@ -131,25 +136,33 @@ func VerifyPostgresesInstallation(
os.Exit(1)
}
+ if isShowLogs {
+ logger.Info(
+ "PostgreSQL command found",
+ "command",
+ cmd,
+ "version",
+ string(version),
+ )
+ }
+ }
+
+ if isShowLogs {
logger.Info(
- "PostgreSQL command found",
- "command",
- cmd,
+ "Installation of PostgreSQL verified",
"version",
string(version),
+ "path",
+ binDir,
)
}
+ }
+ if isShowLogs {
logger.Info(
- "Installation of PostgreSQL verified",
- "version",
- string(version),
- "path",
- binDir,
+ "All PostgreSQL version-specific client tools verification completed successfully!",
)
}
-
- logger.Info("All PostgreSQL version-specific client tools verification completed successfully!")
}
// EscapePgpassField escapes special characters in a field value for .pgpass file format.