Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: harden project instance archive and purge paths (#21118)

p0ny1b86bb7560e2
diff --git a/backend/api/mcp/gen/openapi.yaml b/backend/api/mcp/gen/openapi.yaml
index 596fa39385..29846b828a 100644
--- a/backend/api/mcp/gen/openapi.yaml
+++ b/backend/api/mcp/gen/openapi.yaml
@@ -2327,6 +2327,8 @@ paths:
summary: DeleteInstance
description: |-
Deletes or soft-deletes a database instance.
+ Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ targeting the instance is pending, available, or running.
Permissions required: bb.instances.delete
operationId: bytebase.v1.InstanceService.DeleteInstance
parameters:
@@ -2555,6 +2557,8 @@ paths:
summary: UndeleteInstance
description: |-
Restores a soft-deleted database instance.
+ Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ the instance is pending, available, or running.
Permissions required: bb.instances.undelete
operationId: bytebase.v1.InstanceService.UndeleteInstance
parameters:
diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.go
index 93891ccc36..1ac4f15f9b 100644
--- a/backend/api/v1/instance_service.go
+++ b/backend/api/v1/instance_service.go
@@ -926,28 +926,22 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Reque
return connect.NewResponse(&emptypb.Empty{}), nil
}
- // Regular soft delete flow
- // Idempotent: if already deleted, return success
- if instance.Deleted {
- return connect.NewResponse(&emptypb.Empty{}), nil
- }
-
- if instance.ProjectID == nil {
- databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID})
- if err != nil {
- return nil, err
- }
+ // Regular soft delete flow. Already archived instances still pass through
+ // the lifecycle guard so legacy active task runs are reported consistently.
+ alreadyDeleted := instance.Deleted
+ var moveDatabasesToProjectID *string
+ if !alreadyDeleted && instance.ProjectID == nil {
if req.Msg.Force {
- if len(databases) > 0 {
- defaultProjectID, err := s.store.GetDefaultProjectID(ctx, common.GetWorkspaceIDFromContext(ctx))
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get default project ID"))
- }
- if err := s.store.BatchUpdateDatabases(ctx, databases, &store.BatchUpdateDatabases{Workspace: common.GetWorkspaceIDFromContext(ctx), ProjectID: &defaultProjectID}); err != nil {
- return nil, connect.NewError(connect.CodeInternal, err)
- }
+ defaultProjectID, err := s.store.GetDefaultProjectID(ctx, common.GetWorkspaceIDFromContext(ctx))
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get default project ID"))
}
+ moveDatabasesToProjectID = &defaultProjectID
} else {
+ databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID})
+ if err != nil {
+ return nil, err
+ }
var databaseNames []string
for _, database := range databases {
if !common.IsDefaultProject(common.GetWorkspaceIDFromContext(ctx), database.ProjectID) {
@@ -961,9 +955,10 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Reque
}
patch := &store.UpdateInstanceMessage{
- ResourceID: &instance.ResourceID,
- Workspace: instance.Workspace,
- Deleted: &deletePatch,
+ ResourceID: &instance.ResourceID,
+ Workspace: instance.Workspace,
+ Deleted: &deletePatch,
+ MoveDatabasesToProjectID: moveDatabasesToProjectID,
}
if instance.ProjectID == nil {
metadata := proto.CloneOf(instance.Metadata)
@@ -971,11 +966,14 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Reque
patch.Metadata = metadata
}
if _, err := s.store.UpdateInstance(ctx, patch); err != nil {
+ if common.ErrorCode(err) == common.Conflict {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)
+ }
return nil, connect.NewError(connect.CodeInternal, err)
}
// Handle sample instance deletion if applicable
- if s.sampleInstanceManager != nil {
+ if !alreadyDeleted && s.sampleInstanceManager != nil {
if err := s.sampleInstanceManager.HandleInstanceDeletion(ctx, common.GetWorkspaceIDFromContext(ctx), instance.ResourceID); err != nil {
slog.Warn("failed to handle sample instance deletion", log.BBError(err), slog.String("instance", instance.ResourceID))
}
@@ -995,6 +993,13 @@ func (s *InstanceService) UndeleteInstance(ctx context.Context, req *connect.Req
result := s.convertToV1Instance(ctx, instance)
return connect.NewResponse(result), nil
}
+ activeTaskRunCount, err := s.store.GetActiveTaskRunCountForInstance(ctx, instance.ResourceID)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ if activeTaskRunCount > 0 {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("instance %s has %d active task run(s); cancel them or wait for them to finish before restoring", instance.ResourceID, activeTaskRunCount))
+ }
if err := s.instanceCountGuard(ctx); err != nil {
return nil, err
}
@@ -1008,6 +1013,9 @@ func (s *InstanceService) UndeleteInstance(ctx context.Context, req *connect.Req
Deleted: &undeletePatch,
})
if err != nil {
+ if common.ErrorCode(err) == common.Conflict {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)
+ }
return nil, connect.NewError(connect.CodeInternal, err)
}
diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.go
index 31971c99a0..7488cb1d27 100644
--- a/backend/api/v1/rollout_service.go
+++ b/backend/api/v1/rollout_service.go
@@ -891,6 +891,9 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request
}
if err := s.store.CreatePendingTaskRuns(ctx, user.Email, taskRunCreates...); err != nil {
+ if common.ErrorCode(err) == common.Conflict {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)
+ }
return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))
}
diff --git a/backend/generated-go/v1/instance_service_grpc.pb.go b/backend/generated-go/v1/instance_service_grpc.pb.go
index 81a8428c5d..bffb3fd91e 100644
--- a/backend/generated-go/v1/instance_service_grpc.pb.go
+++ b/backend/generated-go/v1/instance_service_grpc.pb.go
@@ -54,9 +54,13 @@ type InstanceServiceClient interface {
// Permissions required: bb.instances.update
UpdateInstance(ctx context.Context, in *UpdateInstanceRequest, opts ...grpc.CallOption) (*Instance, error)
// Deletes or soft-deletes a database instance.
+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ // targeting the instance is pending, available, or running.
// Permissions required: bb.instances.delete
DeleteInstance(ctx context.Context, in *DeleteInstanceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
// Restores a soft-deleted database instance.
+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ // the instance is pending, available, or running.
// Permissions required: bb.instances.undelete
UndeleteInstance(ctx context.Context, in *UndeleteInstanceRequest, opts ...grpc.CallOption) (*Instance, error)
// Syncs database schemas and metadata from an instance.
@@ -239,9 +243,13 @@ type InstanceServiceServer interface {
// Permissions required: bb.instances.update
UpdateInstance(context.Context, *UpdateInstanceRequest) (*Instance, error)
// Deletes or soft-deletes a database instance.
+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ // targeting the instance is pending, available, or running.
// Permissions required: bb.instances.delete
DeleteInstance(context.Context, *DeleteInstanceRequest) (*emptypb.Empty, error)
// Restores a soft-deleted database instance.
+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ // the instance is pending, available, or running.
// Permissions required: bb.instances.undelete
UndeleteInstance(context.Context, *UndeleteInstanceRequest) (*Instance, error)
// Syncs database schemas and metadata from an instance.
diff --git a/backend/generated-go/v1/v1connect/instance_service.connect.go b/backend/generated-go/v1/v1connect/instance_service.connect.go
index 6a090ec344..e886ed83d8 100644
--- a/backend/generated-go/v1/v1connect/instance_service.connect.go
+++ b/backend/generated-go/v1/v1connect/instance_service.connect.go
@@ -90,9 +90,13 @@ type InstanceServiceClient interface {
// Permissions required: bb.instances.update
UpdateInstance(context.Context, *connect.Request[v1.UpdateInstanceRequest]) (*connect.Response[v1.Instance], error)
// Deletes or soft-deletes a database instance.
+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ // targeting the instance is pending, available, or running.
// Permissions required: bb.instances.delete
DeleteInstance(context.Context, *connect.Request[v1.DeleteInstanceRequest]) (*connect.Response[emptypb.Empty], error)
// Restores a soft-deleted database instance.
+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ // the instance is pending, available, or running.
// Permissions required: bb.instances.undelete
UndeleteInstance(context.Context, *connect.Request[v1.UndeleteInstanceRequest]) (*connect.Response[v1.Instance], error)
// Syncs database schemas and metadata from an instance.
@@ -307,9 +311,13 @@ type InstanceServiceHandler interface {
// Permissions required: bb.instances.update
UpdateInstance(context.Context, *connect.Request[v1.UpdateInstanceRequest]) (*connect.Response[v1.Instance], error)
// Deletes or soft-deletes a database instance.
+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ // targeting the instance is pending, available, or running.
// Permissions required: bb.instances.delete
DeleteInstance(context.Context, *connect.Request[v1.DeleteInstanceRequest]) (*connect.Response[emptypb.Empty], error)
// Restores a soft-deleted database instance.
+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ // the instance is pending, available, or running.
// Permissions required: bb.instances.undelete
UndeleteInstance(context.Context, *connect.Request[v1.UndeleteInstanceRequest]) (*connect.Response[v1.Instance], error)
// Syncs database schemas and metadata from an instance.
diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.go
index 5860ff359d..deeb30841a 100644
--- a/backend/runner/taskrun/pending_scheduler.go
+++ b/backend/runner/taskrun/pending_scheduler.go
@@ -128,13 +128,26 @@ func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.T
return nil
}
- // Check 3: Database mutual exclusion (for sequential tasks)
+ // Check 3: Instance archival. A pending task run must not be promoted to
+ // AVAILABLE while its target instance is archived.
+ instance, err := s.store.GetInstanceByResourceID(ctx, task.InstanceID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get instance")
+ }
+ if instance == nil {
+ return errors.Errorf("instance %v not found", task.InstanceID)
+ }
+ if instance.Deleted {
+ return nil
+ }
+
+ // Check 4: Database mutual exclusion (for sequential tasks)
canProceed, _ := sc.checkDatabaseMutualExclusion(task)
if !canProceed {
return nil
}
- // Check 4: Parallel task limit per rollout
+ // Check 5: Parallel task limit per rollout
maxParallel, err := s.getMaxParallelForTask(ctx, task)
if err != nil {
return errors.Wrapf(err, "failed to get max parallel limit")
diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.go
index d3bfdedbb4..d92d6b81d4 100644
--- a/backend/runner/taskrun/running_scheduler.go
+++ b/backend/runner/taskrun/running_scheduler.go
@@ -238,9 +238,23 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int64, task *
}
// validateTaskFreshness checks for state drift between task creation and execution time.
-// Returns an error if the target database has been deleted, its project has changed,
-// or its environment has changed since the task was created.
+// Returns an error if the target instance has been archived or deleted, the target
+// database has been deleted, its project has changed, or its environment has
+// changed since the task was created.
func (s *Scheduler) validateTaskFreshness(ctx context.Context, task *store.TaskMessage) error {
+ // Every task targets an instance; a task run must not newly start while its
+ // target instance is archived. Already-started task runs are not affected.
+ instance, err := s.store.GetInstanceByResourceID(ctx, task.InstanceID)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get instance for drift validation")
+ }
+ if instance == nil {
+ return errors.Errorf("target instance %q has been deleted", task.InstanceID)
+ }
+ if instance.Deleted {
+ return errors.Errorf("target instance %q has been archived", task.InstanceID)
+ }
+
// DATABASE_CREATE tasks have DatabaseName = nil — the database doesn't exist yet.
if task.Type == storepb.Task_DATABASE_CREATE {
return nil
diff --git a/backend/runner/taskrun/running_scheduler_test.go b/backend/runner/taskrun/running_scheduler_test.go
index ac107d6fef..b62aa2c001 100644
--- a/backend/runner/taskrun/running_scheduler_test.go
+++ b/backend/runner/taskrun/running_scheduler_test.go
@@ -6,6 +6,8 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/bytebase/bytebase/backend/component/bus"
+ "github.com/bytebase/bytebase/backend/component/config"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/store"
)
@@ -143,14 +145,113 @@ func TestCheckTaskDrift(t *testing.T) {
}
}
-func TestValidateTaskFreshness_DatabaseCreateSkipsValidation(t *testing.T) {
- // Construct a Scheduler with nil store. If validateTaskFreshness tries to
- // call the store for DATABASE_CREATE tasks, it will panic — proving the
- // early return works.
- s := &Scheduler{}
- task := &store.TaskMessage{
- Type: storepb.Task_DATABASE_CREATE,
- }
- err := s.validateTaskFreshness(context.Background(), task)
+func TestValidateTaskFreshnessChecksInstanceArchival(t *testing.T) {
+ ctx := context.Background()
+ s := setupRolloutCreatorStore(ctx, t)
+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{
+ ProjectID: "project-a",
+ Name: "freshness validation plan",
+ Config: &storepb.PlanConfig{},
+ require.NoError(t, err)
+ instanceID := "instance-a"
+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: instanceID,
+ Workspace: "default",
+ Metadata: &storepb.Instance{
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ require.NoError(t, err)
+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{
+ InstanceID: instanceID,
+ Type: storepb.Task_DATABASE_CREATE,
+ Payload: &storepb.Task{},
+ }})
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ require.NoError(t, tx.Commit())
+
+ scheduler := &Scheduler{store: s}
+ // DATABASE_CREATE has no database target yet, but its target instance must
+ // still be live for the task run to be allowed to start.
+ require.NoError(t, scheduler.validateTaskFreshness(ctx, tasks[0]), "a live instance should pass freshness validation")
+
+ // Seed a legacy state to keep freshness validation defensive for archived
+ // DATABASE_CREATE tasks that have no database target yet.
+ _, err = s.GetDB().ExecContext(ctx, `
+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1
+ `, instanceID)
+ require.NoError(t, err)
+ err = scheduler.validateTaskFreshness(ctx, tasks[0])
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "archived", "an archived instance must block a DATABASE_CREATE task from starting")
+}
+
+func TestScheduleRunningTaskRunsSkipsArchivedInstance(t *testing.T) {
+ ctx := context.Background()
+ s := setupRolloutCreatorStore(ctx, t)
+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{
+ ProjectID: "project-a",
+ Name: "archived instance claim plan",
+ Config: &storepb.PlanConfig{},
+ require.NoError(t, err)
+ instanceID := "instance-a"
+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: instanceID,
+ Workspace: "default",
+ Metadata: &storepb.Instance{
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ require.NoError(t, err)
+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{
+ InstanceID: instanceID,
+ Type: storepb.Task_DATABASE_CREATE,
+ Payload: &storepb.Task{},
+ }})
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ require.NoError(t, tx.Commit())
+ require.NoError(t, s.CreatePendingTaskRuns(ctx, "", &store.TaskRunMessage{
+ ProjectID: plan.ProjectID,
+ TaskUID: tasks[0].ID,
+ }))
+ taskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})
+ require.NoError(t, err)
+ require.Len(t, taskRuns, 1)
+ _, err = s.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{
+ ID: taskRuns[0].ID,
+ ProjectID: plan.ProjectID,
+ Status: storepb.TaskRun_AVAILABLE,
+ AllowedStatuses: []storepb.TaskRun_Status{storepb.TaskRun_PENDING},
+ })
+ require.NoError(t, err)
+
+ // Seed a legacy state that predates the archive precondition. New archive
+ // requests reject AVAILABLE task runs, but claiming must remain defensive
+ // for existing inconsistent rows.
+ _, err = s.GetDB().ExecContext(ctx, `
+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1
+ `, instanceID)
+ require.NoError(t, err)
+
+ b, err := bus.New()
+ require.NoError(t, err)
+ scheduler := &Scheduler{store: s, bus: b, profile: &config.Profile{ReplicaID: "replica-a"}}
+ require.NoError(t, scheduler.scheduleRunningTaskRuns(ctx))
+
+ taskRuns, err = s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})
require.NoError(t, err)
+ require.Len(t, taskRuns, 1)
+ require.Equal(t, storepb.TaskRun_AVAILABLE, taskRuns[0].Status, "an archived instance's available task run must not be claimed")
}
diff --git a/backend/runner/taskrun/scheduler_test.go b/backend/runner/taskrun/scheduler_test.go
index 5d0f14decf..0ef1592dab 100644
--- a/backend/runner/taskrun/scheduler_test.go
+++ b/backend/runner/taskrun/scheduler_test.go
@@ -91,3 +91,57 @@ func TestSchedulePendingTaskRunsSkipsArchivedProject(t *testing.T) {
require.Len(t, taskRuns, 1)
require.Equal(t, storepb.TaskRun_AVAILABLE, taskRuns[0].Status)
}
+
+func TestSchedulePendingTaskRunsSkipsArchivedInstance(t *testing.T) {
+ ctx := context.Background()
+ s := setupRolloutCreatorStore(ctx, t)
+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{
+ ProjectID: "project-a",
+ Name: "archived instance plan",
+ Config: &storepb.PlanConfig{},
+ require.NoError(t, err)
+ instanceID := "instance-a"
+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{
+ ResourceID: instanceID,
+ Workspace: "default",
+ Metadata: &storepb.Instance{
+ Engine: storepb.Engine_POSTGRES,
+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},
+ },
+ })
+ require.NoError(t, err)
+
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ require.NoError(t, err)
+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{
+ InstanceID: instanceID,
+ Type: storepb.Task_TASK_TYPE_UNSPECIFIED,
+ Payload: &storepb.Task{},
+ }})
+ require.NoError(t, err)
+ require.Len(t, tasks, 1)
+ require.NoError(t, tx.Commit())
+ require.NoError(t, s.CreatePendingTaskRuns(ctx, "", &store.TaskRunMessage{
+ ProjectID: plan.ProjectID,
+ TaskUID: tasks[0].ID,
+ }))
+
+ // Seed a legacy state that predates the archive precondition. New archive
+ // requests reject PENDING task runs, but the scheduler must still avoid
+ // promoting existing inconsistent rows.
+ _, err = s.GetDB().ExecContext(ctx, `
+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1
+ `, instanceID)
+ require.NoError(t, err)
+
+ b, err := bus.New()
+ require.NoError(t, err)
+ scheduler := &Scheduler{store: s, bus: b}
+ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))
+
+ taskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})
+ require.NoError(t, err)
+ require.Len(t, taskRuns, 1)
+ require.Equal(t, storepb.TaskRun_PENDING, taskRuns[0].Status, "an archived instance's pending task run must not be promoted")
+}
diff --git a/backend/store/README.md b/backend/store/README.md
index a4b3129793..306660a169 100644
--- a/backend/store/README.md
+++ b/backend/store/README.md
@@ -26,12 +26,14 @@ that branch. The active-project check in `nextProjectID` covers this case only f
writers that call it; it is not a repository-wide purge fence because other
writers bypass `nextProjectID`.
-Database creation, database-sync, and Query History writers, together with direct
+Database creation, database-sync, batch database updates, task-run creation,
+and Query History writers, together with direct instance archive and direct
project/instance purge, additionally take the matching transaction-scoped purge
-fence before any row lock. This closes the absent database-descendant gap;
-writers then retain the normal child-to-parent row-lock order. Database sync may
-continue for an archived project while its row exists, but never through a
-soft-deleted instance.
+fence before any row lock. This closes absent-descendant gaps; writers then
+retain the normal child-to-parent row-lock order. Database sync may continue for
+an archived project while its row exists, but never through a soft-deleted
+instance. Direct instance archive and restore fail while any targeting task run
+is pending, available, or running.
Every new or modified writer of purge-managed data must define its project
lifecycle policy: require an active project for new resources, or require only an
diff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.go
index 56b269e8c7..929d08fedd 100644
--- a/backend/store/advisory_lock.go
+++ b/backend/store/advisory_lock.go
@@ -29,8 +29,8 @@ const (
// AdvisoryLockKeyProjectPurge serializes project-owned writes with project
// purge, including writes to an otherwise absent database descendant.
AdvisoryLockKeyProjectPurge AdvisoryLockKey = 1006
- // AdvisoryLockKeyInstancePurge serializes database writes with instance
- // purge, which cannot lock an absent database descendant.
+ // AdvisoryLockKeyInstancePurge serializes instance lifecycle operations with
+ // descendant creation and purge when the descendant row may not exist yet.
AdvisoryLockKeyInstancePurge AdvisoryLockKey = 1007
)
diff --git a/backend/store/batch_update_database_purge_test.go b/backend/store/batch_update_database_purge_test.go
new file mode 100644
index 0000000000..2b35f4fd4c
--- /dev/null
+++ b/backend/store/batch_update_database_purge_test.go
@@ -0,0 +1,243 @@
+package store_test
+
+import (
+ "database/sql"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+func batchMoveDatabaseIntoProjectA(fixture *projectDeletionLockOrderFixture) error {
+ projectID := "project-a"
+ return fixture.store.BatchUpdateDatabases(fixture.ctx, []*store.DatabaseMessage{{
+ InstanceID: "instance-a",
+ DatabaseName: "db-a",
+ }}, &store.BatchUpdateDatabases{
+ Workspace: "default",
+ ProjectID: &projectID,
+ })
+}
+
+func requireProjectPurgeTerminalState(t *testing.T, fixture *projectDeletionLockOrderFixture) {
+ t.Helper()
+ var projectCount, orphanCount, databaseCount int
+ var project string
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT COUNT(*) FROM project WHERE resource_id = 'project-a'").Scan(&projectCount))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT COUNT(*) FROM db WHERE project = 'project-a'").Scan(&orphanCount))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT COUNT(*) FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&databaseCount))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&project))
+ require.Zero(t, projectCount, "project purge must remove the archived project")
+ require.Zero(t, orphanCount, "no database row may reference the purged project")
+ require.Equal(t, 1, databaseCount, "the workspace-instance database must survive")
+ require.Equal(t, "default", project, "the workspace-instance database must be reassigned to the default project")
+}
+
+func TestBatchUpdateDatabasesAndDeleteProjectSerialize(t *testing.T) {
+ // The batch moves a workspace-instance database into the archived project
+ // while that project is being purged. Without purge fencing the terminal
+ // project deletion fails on db.project's foreign key.
+ const seedSQL = `
+ INSERT INTO instance (resource_id, workspace, project)
+ VALUES ('instance-a', 'default', NULL);
+ INSERT INTO db (instance, name, project) VALUES ('instance-a', 'db-a', 'default');
+ `
+ purgeProjectA := func(fixture *projectDeletionLockOrderFixture) error {
+ return fixture.store.DeleteProject(fixture.ctx, "default", "project-a")
+ }
+
+ t.Run("purge first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)
+ const barrierID = 9941
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER DELETE ON project FOR EACH ROW")
+
+ purgeResult := make(chan error, 1)
+ go func() { purgeResult <- purgeProjectA(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ batchResult := make(chan error, 1)
+ go func() { batchResult <- batchMoveDatabaseIntoProjectA(fixture) }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+
+ requireMaintenanceResult(t, purgeResult)
+ err := <-batchResult
+ require.Error(t, err)
+ require.NotContains(t, strings.ToLower(err.Error()), "foreign key",
+ "the batch must reject the purged destination cleanly, not with an FK failure")
+ require.ErrorContains(t, err, "project-a not found",
+ "the batch must fail on the purged destination project")
+ requireProjectPurgeTerminalState(t, fixture)
+ })
+
+ t.Run("batch first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)
+ const barrierID = 9942
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ _, err := fixture.db.ExecContext(fixture.ctx, `
+ CREATE FUNCTION block_db_project_update_9942() RETURNS trigger AS $$
+ BEGIN
+ PERFORM pg_advisory_xact_lock(9942);
+ RETURN NEW;
+ END;
+ $$ LANGUAGE plpgsql;
+ CREATE TRIGGER block_db_project_update_9942
+ BEFORE UPDATE OF project ON db FOR EACH ROW
+ WHEN (OLD.project IS DISTINCT FROM NEW.project)
+ EXECUTE FUNCTION block_db_project_update_9942();
+ `)
+ require.NoError(t, err)
+
+ batchResult := make(chan error, 1)
+ go func() { batchResult <- batchMoveDatabaseIntoProjectA(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ purgeResult := make(chan error, 1)
+ go func() { purgeResult <- purgeProjectA(fixture) }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+
+ requireMaintenanceResult(t, batchResult)
+ requireMaintenanceResult(t, purgeResult)
+ requireProjectPurgeTerminalState(t, fixture)
+ })
+}
+
+func TestBatchUpdateDatabasesRejectsArchivedInstance(t *testing.T) {
+ const seedSQL = `
+ INSERT INTO instance (resource_id, workspace, project, deleted)
+ VALUES ('instance-a', 'default', NULL, TRUE);
+ INSERT INTO db (instance, name, project) VALUES ('instance-a', 'db-a', 'default');
+ `
+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)
+ err := batchMoveDatabaseIntoProjectA(fixture)
+ require.Error(t, err)
+ require.ErrorContains(t, err, "instance instance-a is archived")
+
+ var projectID string
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&projectID))
+ require.Equal(t, "default", projectID)
+}
+
+func TestBatchUpdateDatabasesClearsEnvironmentOnArchivedInstance(t *testing.T) {
+ environmentID, unset := "env-a", ""
+ fixture := newProjectDeletionLockOrderFixture(t, `
+ INSERT INTO instance (resource_id, workspace, project, deleted)
+ VALUES ('instance-a', 'default', NULL, TRUE);
+ INSERT INTO db (instance, name, project, environment)
+ VALUES ('instance-a', 'db-a', 'default', 'env-a');
+ `)
+
+ err := fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{
+ Workspace: "default",
+ FindByEnvironmentID: &environmentID,
+ EnvironmentID: &unset,
+ })
+ require.NoError(t, err, "environment cleanup must remain possible after an instance is archived")
+
+ var environment sql.NullString
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT environment FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&environment))
+ require.False(t, environment.Valid, "the archived instance must not retain a deleted environment reference")
+}
+
+func TestBatchUpdateDatabasesByEnvironment(t *testing.T) {
+ envA, unset := "env-a", ""
+ fixture := newProjectDeletionLockOrderFixture(t, `
+ INSERT INTO workspace (resource_id) VALUES ('other');
+ INSERT INTO instance (resource_id, workspace, project) VALUES
+ ('instance-a', 'default', NULL),
+ ('instance-b', 'other', NULL);
+ INSERT INTO db (instance, name, project, environment) VALUES
+ ('instance-a', 'db-a', 'default', 'env-a'),
+ ('instance-a', 'db-b', 'default', 'env-b'),
+ ('instance-b', 'db-c', 'default', 'env-a');
+ `)
+
+ err := fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{
+ Workspace: "default",
+ FindByEnvironmentID: &envA,
+ EnvironmentID: &unset,
+ })
+ require.NoError(t, err)
+
+ requireDatabaseEnvironment := func(instanceID, databaseName string, want string) {
+ t.Helper()
+ var environment sql.NullString
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT environment FROM db WHERE instance = $1 AND name = $2", instanceID, databaseName).Scan(&environment))
+ if want == "" {
+ require.False(t, environment.Valid, "database %s/%s environment should be unset", instanceID, databaseName)
+ } else {
+ require.True(t, environment.Valid)
+ require.Equal(t, want, environment.String)
+ }
+ }
+ requireDatabaseEnvironment("instance-a", "db-a", "")
+ requireDatabaseEnvironment("instance-a", "db-b", "env-b")
+ requireDatabaseEnvironment("instance-b", "db-c", "env-a")
+}
+
+func TestBatchUpdateDatabasesByEnvironmentRevalidatesDynamicTargets(t *testing.T) {
+ environmentID, unset := "env-a", ""
+ fixture := newProjectDeletionLockOrderFixture(t, `
+ INSERT INTO instance (resource_id, workspace, project)
+ VALUES ('instance-a', 'default', NULL);
+ INSERT INTO db (instance, name, project, environment) VALUES
+ ('instance-a', 'db-a', 'default', 'env-a'),
+ ('instance-a', 'db-b', 'default', 'env-b');
+ `)
+
+ // Block the batch on its default-project purge fence. It reaches that fence
+ // only after the outside-transaction target read has completed.
+ conn, err := fixture.db.Conn(fixture.ctx)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, conn.Close()) })
+ _, err = conn.ExecContext(fixture.ctx, "SELECT pg_advisory_lock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _, err := conn.ExecContext(fixture.ctx, "SELECT pg_advisory_unlock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")
+ require.NoError(t, err)
+ })
+ var barrierPID int
+ require.NoError(t, conn.QueryRowContext(fixture.ctx, "SELECT pg_backend_pid()").Scan(&barrierPID))
+
+ result := make(chan error, 1)
+ go func() {
+ result <- fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{
+ Workspace: "default",
+ FindByEnvironmentID: &environmentID,
+ EnvironmentID: &unset,
+ })
+ }()
+ waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, barrierPID)
+
+ // db-b was not in the pre-read target set, but becomes a match before the
+ // fenced transaction scans. The batch must either include it or explicitly
+ // request a retry; it must not report success and silently leave it behind.
+ _, err = fixture.db.ExecContext(fixture.ctx,
+ "UPDATE db SET environment = 'env-a' WHERE instance = 'instance-a' AND name = 'db-b'")
+ require.NoError(t, err)
+ _, err = conn.ExecContext(fixture.ctx, "SELECT pg_advisory_unlock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")
+ require.NoError(t, err)
+
+ err = <-result
+ if err != nil {
+ require.NotContains(t, strings.ToLower(err.Error()), "foreign key")
+ require.Contains(t, strings.ToLower(err.Error()), "retry")
+ return
+ }
+
+ var environment sql.NullString
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT environment FROM db WHERE instance = 'instance-a' AND name = 'db-b'").Scan(&environment))
+ require.False(t, environment.Valid, "a matching database added during the fence wait must not be skipped")
+}
diff --git a/backend/store/database.go b/backend/store/database.go
index f3c4ed1417..d0e23e93f7 100644
--- a/backend/store/database.go
+++ b/backend/store/database.go
@@ -388,6 +388,12 @@ func (s *Store) UpdateDatabase(ctx context.Context, patch *UpdateDatabaseMessage
}
// BatchUpdateDatabases updates databases in batch.
+//
+// The batch runs in one transaction after acquiring every affected source and
+// destination project purge fence and instance purge fence in deterministic
+// sorted order. Without the fences, a concurrent project purge could sweep the
+// destination project's rows and make the terminal project deletion fail on
+// the db.project foreign key.
func (s *Store) BatchUpdateDatabases(ctx context.Context, databases []*DatabaseMessage, update *BatchUpdateDatabases) error {
set := qb.Q()
if update.ProjectID != nil {
@@ -424,74 +430,317 @@ func (s *Store) BatchUpdateDatabases(ctx context.Context, databases []*DatabaseM
where.And("db.instance IN (SELECT resource_id FROM instance WHERE workspace = ?)", update.Workspace)
}
- validOwnership := qb.Q().Space("TRUE")
+ // Discover every affected database before acquiring purge fences so the
+ // source/destination project and instance fence sets are known up front.
+ // The transaction below re-reads these targets under the fences.
+ targets, err := s.listDatabaseBatchTargets(ctx, where)
+ if err != nil {
+ return err
+ }
+ fenceInstances := make([]string, 0, len(targets))
+ fenceProjects := make([]string, 0, len(targets))
+ for _, target := range targets {
+ fenceInstances = append(fenceInstances, target.instanceID)
+ fenceProjects = append(fenceProjects, target.projectID)
+ if target.instanceProject != "" {
+ fenceProjects = append(fenceProjects, target.instanceProject)
+ }
+ }
if update.ProjectID != nil {
- validOwnership = qb.Q().Space(`
- NOT EXISTS (
- SELECT 1 FROM targets
- WHERE instance_project IS NOT NULL
- AND (project <> instance_project OR instance_project <> ?)
- )
- `, *update.ProjectID)
+ fenceProjects = append(fenceProjects, *update.ProjectID)
}
+ slices.Sort(fenceInstances)
+ fenceInstances = slices.Compact(fenceInstances)
+ slices.Sort(fenceProjects)
+ fenceProjects = slices.Compact(fenceProjects)
+ var updated []databaseBatchTarget
+ err = s.withDatabaseBatchPurgeFence(ctx, fenceInstances, fenceProjects, func(tx *sql.Tx) error {
+ // Dynamic environment matching can gain rows between the pre-read and
+ // the purge fences. Lock the complete current match set so a newly
+ // matching row is either included or causes a retry below; never leave
+ // it silently unchanged.
+ locked, err := lockDatabaseBatchTargets(ctx, tx, where, targets, update.FindByEnvironmentID != nil)
+ if err != nil {
+ return err
+ }
+ for _, target := range locked {
+ if !slices.Contains(fenceInstances, target.instanceID) || !slices.Contains(fenceProjects, target.projectID) || (target.instanceProject != "" && !slices.Contains(fenceProjects, target.instanceProject)) {
+ return common.Errorf(common.Conflict, "batch database targets changed; retry")
+ }
+ }
+ instances, err := lockDatabaseBatchInstances(ctx, tx, locked)
+ if err != nil {
+ return err
+ }
+ if update.ProjectID != nil {
+ for instanceID, instance := range instances {
+ if instance.deleted {
+ return common.Errorf(common.Conflict, "instance %s is archived", instanceID)
+ }
+ }
+ }
+ // Revalidate lifecycle under the transaction: every target must still
+ // be covered by the pre-read purge fences. A purge that committed
+ // before the fences were acquired shows up here as a retry instead of
+ // an FK failure.
+ for _, target := range locked {
+ if !slices.Contains(fenceProjects, target.projectID) {
+ return common.Errorf(common.Conflict, "database ownership changed to project %s for %s; retry", target.projectID, common.FormatDatabase(target.instanceID, target.databaseName))
+ }
+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" && !slices.Contains(fenceProjects, instanceProject) {
+ return common.Errorf(common.Conflict, "database ownership changed to project instance %s for %s; retry", instanceProject, common.FormatDatabase(target.instanceID, target.databaseName))
+ }
+ }
+ // Lock every affected project, including the destination, after the
+ // database rows and instances. A purged destination fails cleanly with
+ // NotFound instead of an FK violation.
+ projectIDs := make([]string, 0, len(locked)+1)
+ for _, target := range locked {
+ projectIDs = append(projectIDs, target.projectID)
+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" {
+ projectIDs = append(projectIDs, instanceProject)
+ }
+ }
+ if update.ProjectID != nil {
+ projectIDs = append(projectIDs, *update.ProjectID)
+ }
+ slices.Sort(projectIDs)
+ projectIDs = slices.Compact(projectIDs)
+ for _, projectID := range projectIDs {
+ var foundProjectID string
+ if err := tx.QueryRowContext(ctx, "SELECT resource_id FROM project WHERE resource_id = $1 FOR UPDATE", projectID).Scan(&foundProjectID); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return common.Errorf(common.NotFound, "project %s not found", projectID)
+ }
+ return errors.Wrapf(err, "failed to lock project %s", projectID)
+ }
+ }
+ // Atomic validation: any invalid target rejects the whole batch before
+ // any row is updated.
+ if update.ProjectID != nil {
+ for _, target := range locked {
+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" {
+ if target.projectID != instanceProject || instanceProject != *update.ProjectID {
+ return common.Errorf(common.Invalid, "cannot move a project instance database to another project")
+ }
+ }
+ }
+ }
+ if len(locked) == 0 {
+ return nil
+ }
+ updateInstances, updateNames := make([]string, 0, len(locked)), make([]string, 0, len(locked))
+ for _, target := range locked {
+ updateInstances = append(updateInstances, target.instanceID)
+ updateNames = append(updateNames, target.databaseName)
+ }
+ q := qb.Q().Space(`
+ UPDATE db SET ?
+ FROM unnest(?::TEXT[], ?::TEXT[]) AS u(instance, name)
+ WHERE db.instance = u.instance AND db.name = u.name
+ `, set, updateInstances, updateNames)
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return errors.Wrap(err, "failed to build batch database update sql")
+ }
+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {
+ return errors.Wrap(err, "failed to update databases")
+ }
+ updated = locked
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ for _, target := range updated {
+ s.removeDatabaseCache(ctx, target.instanceID, target.databaseName)
+ }
+ return nil
+}
+
+type databaseBatchTarget struct {
+ instanceID string
+ databaseName string
+ projectID string
+ instanceProject string
+}
+
+// listDatabaseBatchTargets finds the databases matched by the batch update
+// where clause. It runs outside the purge fences; the result only determines
+// which fences to acquire and which rows to lock inside the transaction.
+func (s *Store) listDatabaseBatchTargets(ctx context.Context, where *qb.Query) ([]databaseBatchTarget, error) {
q := qb.Q().Space(`
- WITH targets AS MATERIALIZED (
- SELECT db.instance, db.name, db.project, instance.project AS instance_project
- FROM db
- JOIN instance ON instance.resource_id = db.instance
- WHERE ?
- ),
- validation AS (
- SELECT ? AS valid
- ),
- updated AS (
- UPDATE db
- SET ?
- FROM targets, validation
- WHERE validation.valid
- AND db.instance = targets.instance
- AND db.name = targets.name
- RETURNING db.instance, db.name
- )
- SELECT validation.valid, updated.instance, updated.name
- FROM validation
- LEFT JOIN updated ON TRUE
- `, where, validOwnership, set)
+ SELECT db.instance, db.name, db.project, instance.project
+ FROM db
+ JOIN instance ON instance.resource_id = db.instance
+ WHERE ?
+ ORDER BY db.instance, db.name
+ `, where)
query, args, err := q.ToSQL()
if err != nil {
- return errors.Wrap(err, "failed to build sql")
+ return nil, errors.Wrap(err, "failed to build batch database target query")
}
rows, err := s.GetDB().QueryContext(ctx, query, args...)
if err != nil {
- return errors.Wrap(err, "failed to update databases")
+ return nil, errors.Wrap(err, "failed to find batch database targets")
+ }
+ defer rows.Close()
+ var targets []databaseBatchTarget
+ for rows.Next() {
+ var target databaseBatchTarget
+ var instanceProject sql.NullString
+ if err := rows.Scan(&target.instanceID, &target.databaseName, &target.projectID, &instanceProject); err != nil {
+ return nil, errors.Wrap(err, "failed to scan batch database target")
+ }
+ if instanceProject.Valid {
+ target.instanceProject = instanceProject.String
+ }
+ targets = append(targets, target)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read batch database targets")
+ }
+ return targets, nil
+}
+
+// withDatabaseBatchPurgeFence serializes a batch database write with direct
+// project and instance purge. Advisory fences are acquired in deterministic
+// sorted order (all project fences, then all instance fences) before any row
+// lock.
+func (s *Store) withDatabaseBatchPurgeFence(
+ ctx context.Context,
+ instances []string,
+ projects []string,
+ write func(*sql.Tx) error,
+) error {
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ if err != nil {
+ return errors.Wrap(err, "failed to begin batch database write transaction")
+ }
+ defer tx.Rollback()
+ for _, projectID := range projects {
+ if err := acquireProjectPurgeLock(ctx, tx, projectID); err != nil {
+ return errors.Wrapf(err, "failed to lock project purge fence for %s", projectID)
+ }
+ }
+ for _, instanceID := range instances {
+ if err := acquireInstancePurgeLock(ctx, tx, instanceID); err != nil {
+ return errors.Wrapf(err, "failed to lock instance purge fence for %s", instanceID)
+ }
+ }
+ if err := write(tx); err != nil {
+ return err
+ }
+ return errors.Wrap(tx.Commit(), "failed to commit batch database write transaction")
+}
+
+// lockDatabaseBatchTargets locks the batch targets in full primary-key order.
+// Dynamic environment queries re-scan the original where clause so rows that
+// start matching after the pre-read are detected. Static targets re-apply the
+// clause to the pre-read identities, so rows that no longer match are skipped.
+// Only db rows are locked.
+func lockDatabaseBatchTargets(ctx context.Context, tx *sql.Tx, where *qb.Query, targets []databaseBatchTarget, dynamic bool) ([]databaseBatchTarget, error) {
+ if len(targets) == 0 && !dynamic {
+ return nil, nil
+ }
+ q := qb.Q().Space(`
+ SELECT db.instance, db.name, db.project, instance.project
+ FROM db
+ JOIN instance ON instance.resource_id = db.instance
+ WHERE ?
+ `, where)
+ if !dynamic {
+ instances, names := make([]string, 0, len(targets)), make([]string, 0, len(targets))
+ for _, target := range targets {
+ instances, names = append(instances, target.instanceID), append(names, target.databaseName)
+ }
+ q.And(`(db.instance, db.name) IN (SELECT * FROM unnest(?::TEXT[], ?::TEXT[]))`, instances, names)
+ }
+ q.Space("ORDER BY db.instance, db.name FOR UPDATE OF db")
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to build batch database lock query")
+ }
+ rows, err := tx.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to lock batch database targets")
}
defer rows.Close()
+ var locked []databaseBatchTarget
+ for rows.Next() {
+ var target databaseBatchTarget
+ var instanceProject sql.NullString
+ if err := rows.Scan(&target.instanceID, &target.databaseName, &target.projectID, &instanceProject); err != nil {
+ return nil, errors.Wrap(err, "failed to scan locked batch database target")
+ }
+ if instanceProject.Valid {
+ target.instanceProject = instanceProject.String
+ }
+ locked = append(locked, target)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read locked batch database targets")
+ }
+ return locked, nil
+}
- var updated []*DatabaseMessage
+type databaseBatchInstance struct {
+ projectID string
+ deleted bool
+}
+
+// lockDatabaseBatchInstances locks the instances of the locked targets in
+// sorted order and returns their current lifecycle and owning project.
+func lockDatabaseBatchInstances(ctx context.Context, tx *sql.Tx, targets []databaseBatchTarget) (map[string]databaseBatchInstance, error) {
+ instanceIDs := make([]string, 0, len(targets))
+ for _, target := range targets {
+ instanceIDs = append(instanceIDs, target.instanceID)
+ }
+ slices.Sort(instanceIDs)
+ instanceIDs = slices.Compact(instanceIDs)
+ instances := make(map[string]databaseBatchInstance, len(instanceIDs))
+ if len(instanceIDs) == 0 {
+ return instances, nil
+ }
+ q := qb.Q().Space(`
+ SELECT resource_id, project, deleted
+ FROM instance
+ WHERE resource_id IN (SELECT * FROM unnest(?::TEXT[]))
+ ORDER BY resource_id
+ FOR NO KEY UPDATE
+ `, instanceIDs)
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to build batch database instance lock query")
+ }
+ rows, err := tx.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to lock batch database instances")
+ }
+ defer rows.Close()
for rows.Next() {
- var valid bool
- var instanceID, databaseName sql.NullString
- if err := rows.Scan(&valid, &instanceID, &databaseName); err != nil {
- return errors.Wrap(err, "failed to scan updated database")
- }
- if !valid {
- return common.Errorf(common.Invalid, "cannot move a project instance database to another project")
- }
- if instanceID.Valid && databaseName.Valid {
- updated = append(updated, &DatabaseMessage{
- InstanceID: instanceID.String,
- DatabaseName: databaseName.String,
- })
+ var instanceID string
+ var project sql.NullString
+ var deleted bool
+ if err := rows.Scan(&instanceID, &project, &deleted); err != nil {
+ return nil, errors.Wrap(err, "failed to scan locked batch database instance")
+ }
+ instance := databaseBatchInstance{deleted: deleted}
+ if project.Valid {
+ instance.projectID = project.String
}
+ instances[instanceID] = instance
}
if err := rows.Err(); err != nil {
- return errors.Wrap(err, "failed to update databases")
+ return nil, errors.Wrap(err, "failed to read locked batch database instances")
}
- for _, database := range updated {
- s.removeDatabaseCache(ctx, database.InstanceID, database.DatabaseName)
+ for _, instanceID := range instanceIDs {
+ if _, ok := instances[instanceID]; !ok {
+ return nil, common.Errorf(common.NotFound, "instance %s not found", instanceID)
+ }
}
- return nil
+ return instances, nil
}
type databaseOwnership struct {
diff --git a/backend/store/instance.go b/backend/store/instance.go
index 2037deaf21..844002a4d8 100644
--- a/backend/store/instance.go
+++ b/backend/store/instance.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
+ "slices"
"strings"
"github.com/google/cel-go/cel"
@@ -40,6 +41,10 @@ type UpdateInstanceMessage struct {
Deleted *bool
EnvironmentID *string
Metadata *storepb.Instance
+ // MoveDatabasesToProjectID atomically transfers all databases while changing
+ // the instance lifecycle. It is only valid when archiving a resource-scoped
+ // workspace instance.
+ MoveDatabasesToProjectID *string
}
// FindInstanceMessage is the message for finding instances.
@@ -280,6 +285,9 @@ func (s *Store) CreateInstance(ctx context.Context, instanceCreate *InstanceMess
// UpdateInstance updates an instance.
func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessage) (*InstanceMessage, error) {
+ if patch.MoveDatabasesToProjectID != nil && (patch.ResourceID == nil || patch.Deleted == nil || !*patch.Deleted) {
+ return nil, errors.New("moving instance databases is only supported while archiving a resource-scoped instance")
+ }
set := qb.Q()
if v := patch.EnvironmentID; v != nil {
@@ -328,6 +336,9 @@ func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessage
Space("WHERE ?", where)
if v := patch.ResourceID; v != nil {
+ if patch.Deleted != nil {
+ return s.updateInstanceLifecycle(ctx, patch, q)
+ }
q.Space("RETURNING workspace, project")
query, args, err := q.ToSQL()
if err != nil {
@@ -359,6 +370,220 @@ func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessage
return nil, nil
}
+func (s *Store) updateInstanceLifecycle(ctx context.Context, patch *UpdateInstanceMessage, q *qb.Query) (*InstanceMessage, error) {
+ resourceID := *patch.ResourceID
+ var projectFences []string
+ if destinationProjectID := patch.MoveDatabasesToProjectID; destinationProjectID != nil {
+ var err error
+ projectFences, err = s.listInstanceDatabaseProjects(ctx, resourceID)
+ if err != nil {
+ return nil, err
+ }
+ projectFences = append(projectFences, *destinationProjectID)
+ slices.Sort(projectFences)
+ projectFences = slices.Compact(projectFences)
+ }
+
+ tx, err := s.GetDB().BeginTx(ctx, nil)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to begin instance lifecycle transaction")
+ }
+ defer tx.Rollback()
+ for _, projectID := range projectFences {
+ if err := acquireProjectPurgeLock(ctx, tx, projectID); err != nil {
+ return nil, errors.Wrapf(err, "failed to lock project purge fence for %s", projectID)
+ }
+ }
+ if err := acquireInstancePurgeLock(ctx, tx, resourceID); err != nil {
+ return nil, errors.Wrapf(err, "failed to lock instance lifecycle fence for %s", resourceID)
+ }
+
+ activeTaskRunCount, err := lockActiveTaskRunsForInstance(ctx, tx, resourceID)
+ if err != nil {
+ return nil, err
+ }
+ if activeTaskRunCount > 0 {
+ operation := "archiving"
+ if !*patch.Deleted {
+ operation = "restoring"
+ }
+ return nil, common.Errorf(common.Conflict, "instance %s has %d active task run(s); cancel them or wait for them to finish before %s", resourceID, activeTaskRunCount, operation)
+ }
+
+ var movedDatabases []instanceDatabaseTarget
+ if destinationProjectID := patch.MoveDatabasesToProjectID; destinationProjectID != nil {
+ movedDatabases, err = lockInstanceDatabases(ctx, tx, resourceID)
+ if err != nil {
+ return nil, err
+ }
+ for _, database := range movedDatabases {
+ if !slices.Contains(projectFences, database.projectID) {
+ return nil, common.Errorf(common.Conflict, "database ownership changed to project %s for %s; retry", database.projectID, common.FormatDatabase(resourceID, database.name))
+ }
+ }
+
+ var instanceProject sql.NullString
+ if err := tx.QueryRowContext(ctx, `
+ SELECT project FROM instance WHERE resource_id = $1 FOR NO KEY UPDATE
+ `, resourceID).Scan(&instanceProject); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, common.Errorf(common.NotFound, "instance %s not found", resourceID)
+ }
+ return nil, errors.Wrapf(err, "failed to lock instance %s", resourceID)
+ }
+ if instanceProject.Valid {
+ return nil, common.Errorf(common.Invalid, "cannot transfer databases while archiving project instance %s", resourceID)
+ }
+
+ projectIDs := append([]string(nil), *destinationProjectID)
+ for _, database := range movedDatabases {
+ projectIDs = append(projectIDs, database.projectID)
+ }
+ slices.Sort(projectIDs)
+ projectIDs = slices.Compact(projectIDs)
+ for _, projectID := range projectIDs {
+ var foundProjectID string
+ if err := tx.QueryRowContext(ctx, "SELECT resource_id FROM project WHERE resource_id = $1 FOR UPDATE", projectID).Scan(&foundProjectID); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, common.Errorf(common.NotFound, "project %s not found", projectID)
+ }
+ return nil, errors.Wrapf(err, "failed to lock project %s", projectID)
+ }
+ }
+ if _, err := tx.ExecContext(ctx, "UPDATE db SET project = $1 WHERE instance = $2", *destinationProjectID, resourceID); err != nil {
+ return nil, errors.Wrapf(err, "failed to transfer databases for instance %s", resourceID)
+ }
+ }
+
+ q.Space("RETURNING workspace, project")
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to build instance lifecycle query")
+ }
+ var workspace string
+ var project sql.NullString
+ if err := tx.QueryRowContext(ctx, query, args...).Scan(&workspace, &project); err != nil {
+ return nil, err
+ }
+ if err := tx.Commit(); err != nil {
+ return nil, errors.Wrap(err, "failed to commit instance lifecycle transaction")
+ }
+
+ s.instanceCache.Remove(getInstanceCacheKey(resourceID))
+ for _, database := range movedDatabases {
+ s.removeDatabaseCache(ctx, resourceID, database.name)
+ }
+ find := &FindInstanceMessage{Workspace: workspace, ResourceID: patch.ResourceID}
+ if project.Valid {
+ find.ProjectID = &project.String
+ } else {
+ find.WorkspaceOnly = true
+ }
+ return s.GetInstance(ctx, find)
+}
+
+func (s *Store) listInstanceDatabaseProjects(ctx context.Context, resourceID string) ([]string, error) {
+ rows, err := s.GetDB().QueryContext(ctx, `
+ SELECT DISTINCT project FROM db WHERE instance = $1 ORDER BY project
+ `, resourceID)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to find database projects for instance %s", resourceID)
+ }
+ defer rows.Close()
+ var projects []string
+ for rows.Next() {
+ var projectID string
+ if err := rows.Scan(&projectID); err != nil {
+ return nil, errors.Wrap(err, "failed to scan instance database project")
+ }
+ projects = append(projects, projectID)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read instance database projects")
+ }
+ return projects, nil
+}
+
+type instanceDatabaseTarget struct {
+ name string
+ projectID string
+}
+
+func lockInstanceDatabases(ctx context.Context, tx *sql.Tx, resourceID string) ([]instanceDatabaseTarget, error) {
+ rows, err := tx.QueryContext(ctx, `
+ SELECT name, project
+ FROM db
+ WHERE instance = $1
+ ORDER BY instance, name
+ FOR UPDATE
+ `, resourceID)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to lock databases for instance %s", resourceID)
+ }
+ defer rows.Close()
+ var databases []instanceDatabaseTarget
+ for rows.Next() {
+ var database instanceDatabaseTarget
+ if err := rows.Scan(&database.name, &database.projectID); err != nil {
+ return nil, errors.Wrap(err, "failed to scan instance database")
+ }
+ databases = append(databases, database)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read instance databases")
+ }
+ return databases, nil
+}
+
+// GetActiveTaskRunCountForInstance returns the number of task runs that would
+// block a direct instance archive or restore.
+func (s *Store) GetActiveTaskRunCountForInstance(ctx context.Context, resourceID string) (int, error) {
+ var count int
+ if err := s.GetDB().QueryRowContext(ctx, `
+ SELECT COUNT(*)
+ FROM task_run
+ JOIN task
+ ON task.project = task_run.project
+ AND task.id = task_run.task_id
+ WHERE task.instance = $1
+ AND task_run.status IN ('PENDING', 'AVAILABLE', 'RUNNING')
+ `, resourceID).Scan(&count); err != nil {
+ return 0, errors.Wrapf(err, "failed to count active task runs for instance %s", resourceID)
+ }
+ return count, nil
+}
+
+func lockActiveTaskRunsForInstance(ctx context.Context, tx *sql.Tx, resourceID string) (int, error) {
+ rows, err := tx.QueryContext(ctx, `
+ SELECT task_run.project, task_run.id
+ FROM task_run
+ JOIN task
+ ON task.project = task_run.project
+ AND task.id = task_run.task_id
+ WHERE task.instance = $1
+ AND task_run.status IN ('PENDING', 'AVAILABLE', 'RUNNING')
+ ORDER BY task_run.project, task_run.id
+ FOR UPDATE OF task_run
+ `, resourceID)
+ if err != nil {
+ return 0, errors.Wrapf(err, "failed to lock active task runs for instance %s", resourceID)
+ }
+ defer rows.Close()
+ activeTaskRunCount := 0
+ for rows.Next() {
+ var projectID string
+ var taskRunID int64
+ if err := rows.Scan(&projectID, &taskRunID); err != nil {
+ return 0, errors.Wrap(err, "failed to scan active task run")
+ }
+ activeTaskRunCount++
+ }
+ if err := rows.Err(); err != nil {
+ return 0, errors.Wrap(err, "failed to read active task runs")
+ }
+ return activeTaskRunCount, nil
+}
+
// GetActivatedInstanceCount gets the number of activated instances.
func (s *Store) GetActivatedInstanceCount(ctx context.Context, workspaceID string) (int, error) {
q := qb.Q().Space("SELECT COUNT(1) FROM instance WHERE workspace = ?", workspaceID).And("(metadata ?? 'activation') AND (metadata->>'activation')::boolean = TRUE AND deleted = FALSE")
diff --git a/backend/store/instance_archive_task_run_test.go b/backend/store/instance_archive_task_run_test.go
new file mode 100644
index 0000000000..d2409e4212
--- /dev/null
+++ b/backend/store/instance_archive_task_run_test.go
@@ -0,0 +1,236 @@
+package store_test
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+const instanceArchiveTaskRunSeed = `
+ UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a';
+ INSERT INTO instance (resource_id, workspace, project)
+ VALUES ('instance-a', 'default', 'project-a');
+ INSERT INTO plan (id, creator, project, name, description)
+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_CREATE');
+`
+
+func archiveInstanceA(fixture *projectDeletionLockOrderFixture) error {
+ deleted := true
+ _, err := fixture.store.UpdateInstance(fixture.ctx, &store.UpdateInstanceMessage{
+ ResourceID: new("instance-a"),
+ Workspace: "default",
+ Deleted: &deleted,
+ })
+ return err
+}
+
+func createInstanceATaskRun(fixture *projectDeletionLockOrderFixture) error {
+ return fixture.store.CreatePendingTaskRuns(fixture.ctx, "[email protected]", &store.TaskRunMessage{
+ ProjectID: "project-a",
+ TaskUID: 101,
+ })
+}
+
+func forceArchiveWorkspaceInstanceA(fixture *projectDeletionLockOrderFixture) error {
+ deleted := true
+ destinationProjectID := "default"
+ _, err := fixture.store.UpdateInstance(fixture.ctx, &store.UpdateInstanceMessage{
+ ResourceID: new("instance-a"),
+ Workspace: "default",
+ Deleted: &deleted,
+ MoveDatabasesToProjectID: &destinationProjectID,
+ })
+ return err
+}
+
+func createInstanceADatabase(fixture *projectDeletionLockOrderFixture) error {
+ _, err := fixture.store.CreateDatabaseDefault(fixture.ctx, &store.DatabaseMessage{
+ InstanceID: "instance-a",
+ DatabaseName: "db-a",
+ ProjectID: "project-a",
+ })
+ return err
+}
+
+func TestArchiveInstanceAndCreateTaskRunSerialize(t *testing.T) {
+ t.Run("archive first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, instanceArchiveTaskRunSeed)
+ const barrierID = 9945
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE OF deleted ON instance FOR EACH ROW")
+
+ archiveResult := make(chan error, 1)
+ go func() { archiveResult <- archiveInstanceA(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ createResult := make(chan error, 1)
+ go func() { createResult <- createInstanceATaskRun(fixture) }()
+ barrierReleased := false
+ defer func() {
+ if !barrierReleased {
+ barrier.release(t)
+ <-archiveResult
+ <-createResult
+ }
+ }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+ barrierReleased = true
+
+ requireMaintenanceResult(t, archiveResult)
+ createErr := <-createResult
+ require.Error(t, createErr)
+ require.Equal(t, common.Conflict, common.ErrorCode(createErr))
+
+ var deleted bool
+ var activeTaskRunCount int
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",
+ ).Scan(&deleted))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx, `
+ SELECT COUNT(*) FROM task_run
+ WHERE project = 'project-a' AND status IN ('PENDING', 'AVAILABLE', 'RUNNING')
+ `).Scan(&activeTaskRunCount))
+ require.True(t, deleted)
+ require.Zero(t, activeTaskRunCount)
+ })
+
+ t.Run("creation first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, instanceArchiveTaskRunSeed)
+ const barrierID = 9946
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER INSERT ON task_run FOR EACH ROW")
+
+ createResult := make(chan error, 1)
+ go func() { createResult <- createInstanceATaskRun(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ archiveResult := make(chan error, 1)
+ go func() { archiveResult <- archiveInstanceA(fixture) }()
+ barrierReleased := false
+ defer func() {
+ if !barrierReleased {
+ barrier.release(t)
+ <-createResult
+ <-archiveResult
+ }
+ }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+ barrierReleased = true
+
+ requireMaintenanceResult(t, createResult)
+ archiveErr := <-archiveResult
+ require.Error(t, archiveErr)
+ require.Equal(t, common.Conflict, common.ErrorCode(archiveErr))
+
+ var deleted bool
+ var activeTaskRunCount int
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",
+ ).Scan(&deleted))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx, `
+ SELECT COUNT(*) FROM task_run
+ WHERE project = 'project-a' AND status IN ('PENDING', 'AVAILABLE', 'RUNNING')
+ `).Scan(&activeTaskRunCount))
+ require.False(t, deleted)
+ require.Equal(t, 1, activeTaskRunCount)
+ })
+}
+
+func TestForceArchiveInstanceAndCreateFirstDatabaseSerialize(t *testing.T) {
+ const seedSQL = `
+ UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a';
+ INSERT INTO instance (resource_id, workspace, project)
+ VALUES ('instance-a', 'default', NULL);
+ `
+
+ t.Run("archive first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)
+ const barrierID = 9947
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE OF deleted ON instance FOR EACH ROW")
+
+ archiveResult := make(chan error, 1)
+ go func() { archiveResult <- forceArchiveWorkspaceInstanceA(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ createResult := make(chan error, 1)
+ go func() { createResult <- createInstanceADatabase(fixture) }()
+ barrierReleased := false
+ defer func() {
+ if !barrierReleased {
+ barrier.release(t)
+ <-archiveResult
+ <-createResult
+ }
+ }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+ barrierReleased = true
+
+ requireMaintenanceResult(t, archiveResult)
+ createErr := <-createResult
+ require.Error(t, createErr)
+ require.NotContains(t, strings.ToLower(createErr.Error()), "foreign key")
+
+ var deleted bool
+ var databaseCount int
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",
+ ).Scan(&deleted))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT COUNT(*) FROM db WHERE instance = 'instance-a' AND name = 'db-a'",
+ ).Scan(&databaseCount))
+ require.True(t, deleted)
+ require.Zero(t, databaseCount)
+ })
+
+ t.Run("creation first", func(t *testing.T) {
+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)
+ const barrierID = 9948
+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)
+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER INSERT ON db FOR EACH ROW")
+
+ createResult := make(chan error, 1)
+ go func() { createResult <- createInstanceADatabase(fixture) }()
+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+
+ archiveResult := make(chan error, 1)
+ go func() { archiveResult <- forceArchiveWorkspaceInstanceA(fixture) }()
+ barrierReleased := false
+ defer func() {
+ if !barrierReleased {
+ barrier.release(t)
+ <-createResult
+ <-archiveResult
+ }
+ }()
+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)
+ barrier.release(t)
+ barrierReleased = true
+
+ requireMaintenanceResult(t, createResult)
+ archiveErr := <-archiveResult
+ require.Error(t, archiveErr)
+ require.Equal(t, common.Conflict, common.ErrorCode(archiveErr))
+ require.NotContains(t, strings.ToLower(archiveErr.Error()), "foreign key")
+
+ var deleted bool
+ var projectID string
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",
+ ).Scan(&deleted))
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'",
+ ).Scan(&projectID))
+ require.False(t, deleted)
+ require.Equal(t, "project-a", projectID)
+ })
+}
diff --git a/backend/store/project.go b/backend/store/project.go
index 4a3f8b99d4..2338191bf3 100644
--- a/backend/store/project.go
+++ b/backend/store/project.go
@@ -331,6 +331,120 @@ func (s *Store) removeProjectCache(resourceID string) {
s.projectCache.Remove(resourceID)
}
+// projectDescendantCacheKeys snapshots the cache keys of every instance,
+// database, and schema row that a project purge is about to remove or
+// reassign, so their cache entries can be invalidated after the purge commits.
+type projectDescendantCacheKeys struct {
+ instanceIDs []string
+ databases []projectDatabaseCacheKey
+ schemas []projectSchemaCacheKey
+}
+
+type projectDatabaseCacheKey struct {
+ workspace string
+ instanceID string
+ databaseName string
+}
+
+type projectSchemaCacheKey struct {
+ instanceID string
+ databaseName string
+}
+
+// captureProjectDescendantCacheKeys reads the cache keys of every descendant
+// row that DeleteProject will delete (project instances and their databases
+// and schemas) or reassign (workspace-instance databases owned by the
+// project). It must run inside the purge transaction before any of those rows
+// are removed.
+func captureProjectDescendantCacheKeys(ctx context.Context, tx *stdsql.Tx, projectID string) (*projectDescendantCacheKeys, error) {
+ keys := &projectDescendantCacheKeys{}
+
+ rows, err := tx.QueryContext(ctx, `
+ SELECT resource_id
+ FROM instance
+ WHERE project = $1
+ ORDER BY resource_id
+ `, projectID)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to capture project instance cache keys for project %s", projectID)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var instanceID string
+ if err := rows.Scan(&instanceID); err != nil {
+ return nil, errors.Wrap(err, "failed to scan project instance cache key")
+ }
+ keys.instanceIDs = append(keys.instanceIDs, instanceID)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read project instance cache keys")
+ }
+
+ rows, err = tx.QueryContext(ctx, `
+ SELECT db.instance, db.name, instance.workspace
+ FROM db
+ JOIN instance ON instance.resource_id = db.instance
+ WHERE instance.project = $1 OR db.project = $1
+ ORDER BY db.instance, db.name
+ `, projectID)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to capture project database cache keys for project %s", projectID)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var key projectDatabaseCacheKey
+ if err := rows.Scan(&key.instanceID, &key.databaseName, &key.workspace); err != nil {
+ return nil, errors.Wrap(err, "failed to scan project database cache key")
+ }
+ keys.databases = append(keys.databases, key)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read project database cache keys")
+ }
+
+ rows, err = tx.QueryContext(ctx, `
+ SELECT db_schema.instance, db_schema.db_name
+ FROM db_schema
+ JOIN instance ON instance.resource_id = db_schema.instance
+ WHERE instance.project = $1
+ ORDER BY db_schema.instance, db_schema.db_name
+ `, projectID)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to capture project schema cache keys for project %s", projectID)
+ }
+ defer rows.Close()
+ for rows.Next() {
+ var key projectSchemaCacheKey
+ if err := rows.Scan(&key.instanceID, &key.databaseName); err != nil {
+ return nil, errors.Wrap(err, "failed to scan project schema cache key")
+ }
+ keys.schemas = append(keys.schemas, key)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read project schema cache keys")
+ }
+
+ return keys, nil
+}
+
+// removeProjectDescendantCaches invalidates the captured descendant cache
+// entries and the project entry itself. It must only run after the purge
+// transaction commits; a failed purge must not publish invalidation.
+func (s *Store) removeProjectDescendantCaches(keys *projectDescendantCacheKeys, projectID string) {
+ for _, instanceID := range keys.instanceIDs {
+ s.instanceCache.Remove(getInstanceCacheKey(instanceID))
+ }
+ for _, key := range keys.databases {
+ // Remove both the unscoped (runner) and workspace-scoped (API) entries.
+ s.databaseCache.Remove(getDatabaseCacheKey("", key.instanceID, key.databaseName))
+ s.databaseCache.Remove(getDatabaseCacheKey(key.workspace, key.instanceID, key.databaseName))
+ }
+ for _, key := range keys.schemas {
+ s.dbSchemaCache.Remove(getDBSchemaCacheKey(key.instanceID, key.databaseName))
+ }
+ s.projectCache.Remove(projectID)
+}
+
// DeleteProject permanently purges a soft-deleted project and all related resources.
// This operation is irreversible and should only be used for:
// - Administrative cleanup of old soft-deleted projects
@@ -351,6 +465,14 @@ func (s *Store) DeleteProject(ctx context.Context, workspace string, resourceID
return errors.Wrapf(err, "failed to lock project purge fence for %s", resourceID)
}
+ // Capture descendant cache keys before any descendant rows are removed so
+ // the post-commit invalidation is precise and does not require re-reading
+ // rows that no longer exist.
+ cacheKeys, err := captureProjectDescendantCacheKeys(ctx, tx, resourceID)
+ if err != nil {
+ return errors.Wrap(err, "failed to capture project descendant cache keys")
+ }
+
// Delete query history before locking database-scoped rows to preserve the
// canonical sibling-branch order.
q := qb.Q().Space("DELETE FROM query_history WHERE project = ?", resourceID)
@@ -723,8 +845,9 @@ func (s *Store) DeleteProject(ctx context.Context, workspace string, resourceID
return errors.Wrap(err, "failed to commit transaction")
}
- // Clear the project from cache
- s.projectCache.Remove(resourceID)
+ // Publish invalidation only after the purge commits: every captured
+ // descendant entry is removed while unrelated cache entries survive.
+ s.removeProjectDescendantCaches(cacheKeys, resourceID)
return nil
}
diff --git a/backend/store/project_purge_cache_test.go b/backend/store/project_purge_cache_test.go
new file mode 100644
index 0000000000..e73b7a72f3
--- /dev/null
+++ b/backend/store/project_purge_cache_test.go
@@ -0,0 +1,239 @@
+package store_test
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ "github.com/bytebase/bytebase/backend/migrator"
+ "github.com/bytebase/bytebase/backend/store"
+)
+
+// newProjectPurgeCacheFixture boots a real PostgreSQL with caches ENABLED and
+// seeds a workspace, the default project, a deleted purgeable project, and an
+// active project for ID-reuse scenarios.
+func newProjectPurgeCacheFixture(t *testing.T) (context.Context, *sql.DB, *store.Store) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ t.Cleanup(cancel)
+ container := testcontainer.GetTestPgContainer(ctx, t)
+ t.Cleanup(func() { container.Close(context.Background()) })
+ db := container.GetDB()
+ require.NoError(t, migrator.MigrateSchema(ctx, db))
+ _, err := db.ExecContext(ctx, `
+ INSERT INTO workspace (resource_id) VALUES ('default');
+ INSERT INTO project (resource_id, workspace, name, deleted) VALUES
+ ('default', 'default', 'Default', FALSE),
+ ('project-a', 'default', 'Project A', TRUE),
+ ('project-b', 'default', 'Project B', FALSE);
+ `)
+ require.NoError(t, err)
+ pgURL := fmt.Sprintf(
+ "host=%s port=%s user=postgres password=root-password database=postgres",
+ container.GetHost(), container.GetPort(),
+ )
+ s, err := store.New(ctx, pgURL, true)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, s.Close()) })
+ return ctx, db, s
+}
+
+// seedProjectPurgeFixture creates a project instance and a workspace instance
+// owned by project-a, a database on each, and schema rows for both databases.
+func seedProjectPurgeFixture(ctx context.Context, t *testing.T, db *sql.DB) {
+ t.Helper()
+ _, err := db.ExecContext(ctx, `
+ INSERT INTO instance (resource_id, workspace, project) VALUES
+ ('project-instance', 'default', 'project-a'),
+ ('workspace-instance', 'default', NULL);
+ INSERT INTO db (instance, name, project) VALUES
+ ('project-instance', 'project-db', 'project-a'),
+ ('workspace-instance', 'workspace-db', 'project-a');
+ INSERT INTO db_schema (instance, db_name) VALUES
+ ('project-instance', 'project-db'),
+ ('workspace-instance', 'workspace-db');
+ `)
+ require.NoError(t, err)
+}
+
+// warmProjectPurgeCaches populates the instance, scoped and unscoped database,
+// and schema cache entries for every seeded descendant.
+func warmProjectPurgeCaches(ctx context.Context, t *testing.T, s *store.Store) {
+ t.Helper()
+ projectInstanceID := "project-instance"
+ projectDBName := "project-db"
+ workspaceInstanceID := "workspace-instance"
+ workspaceDBName := "workspace-db"
+ for _, find := range []*store.FindInstanceMessage{
+ {Workspace: "default", ResourceID: &projectInstanceID},
+ {Workspace: "default", ResourceID: &workspaceInstanceID},
+ } {
+ instance, err := s.GetInstance(ctx, find)
+ require.NoError(t, err)
+ require.NotNil(t, instance)
+ }
+ for _, find := range []*store.FindDatabaseMessage{
+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ {Workspace: "default", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},
+ {Workspace: "", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},
+ } {
+ database, err := s.GetDatabase(ctx, find)
+ require.NoError(t, err)
+ require.NotNil(t, database)
+ }
+ for _, schema := range []struct {
+ instanceID string
+ databaseName string
+ }{
+ {instanceID: projectInstanceID, databaseName: projectDBName},
+ {instanceID: workspaceInstanceID, databaseName: workspaceDBName},
+ } {
+ metadata, err := s.GetDBSchemaSnapshot(ctx, "default", schema.instanceID, schema.databaseName)
+ require.NoError(t, err)
+ require.NotNil(t, metadata)
+ }
+}
+
+// TestDeleteProjectPurgeInvalidatesDescendantCaches warms every descendant
+// cache entry, purges the project, and requires the purged descendants to be
+// immediately unavailable through the cached getters while surviving rows
+// still resolve correctly.
+func TestDeleteProjectPurgeInvalidatesDescendantCaches(t *testing.T) {
+ ctx, db, s := newProjectPurgeCacheFixture(t)
+ seedProjectPurgeFixture(ctx, t, db)
+ warmProjectPurgeCaches(ctx, t, s)
+
+ projectInstanceID := "project-instance"
+ projectDBName := "project-db"
+ workspaceInstanceID := "workspace-instance"
+ workspaceDBName := "workspace-db"
+
+ require.NoError(t, s.DeleteProject(ctx, "default", "project-a"))
+
+ // Observable seam: purged descendants are immediately unavailable through
+ // the cached getters.
+ instance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})
+ require.NoError(t, err)
+ require.Nil(t, instance)
+ for _, find := range []*store.FindDatabaseMessage{
+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ } {
+ database, err := s.GetDatabase(ctx, find)
+ require.NoError(t, err)
+ require.Nil(t, database)
+ }
+ schema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)
+ require.NoError(t, err)
+ require.Nil(t, schema)
+
+ // Surviving rows still resolve: the workspace instance was not flushed and
+ // its schema snapshot is still readable.
+ workspaceInstance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &workspaceInstanceID})
+ require.NoError(t, err)
+ require.NotNil(t, workspaceInstance)
+ workspaceSchema, err := s.GetDBSchemaSnapshot(ctx, "default", workspaceInstanceID, workspaceDBName)
+ require.NoError(t, err)
+ require.NotNil(t, workspaceSchema)
+
+ // The workspace-instance database moved to the default project is re-read
+ // from the database instead of being served from a stale cached entry.
+ for _, find := range []*store.FindDatabaseMessage{
+ {Workspace: "default", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},
+ {Workspace: "", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},
+ } {
+ movedDB, err := s.GetDatabase(ctx, find)
+ require.NoError(t, err)
+ require.NotNil(t, movedDB)
+ require.Equal(t, "default", movedDB.ProjectID)
+ }
+}
+
+// TestDeleteProjectPurgeSupportsDescendantIDReuse proves that reusing a
+// purged instance resource ID and database name cannot expose stale cached
+// data. Reused rows are inserted directly so a stale cache entry would still
+// be observable through the getters.
+func TestDeleteProjectPurgeSupportsDescendantIDReuse(t *testing.T) {
+ ctx, db, s := newProjectPurgeCacheFixture(t)
+ seedProjectPurgeFixture(ctx, t, db)
+ warmProjectPurgeCaches(ctx, t, s)
+
+ require.NoError(t, s.DeleteProject(ctx, "default", "project-a"))
+
+ projectInstanceID := "project-instance"
+ projectDBName := "project-db"
+ _, err := db.ExecContext(ctx, `
+ INSERT INTO instance (resource_id, workspace, project) VALUES
+ ('project-instance', 'default', 'project-b');
+ INSERT INTO db (instance, name, project) VALUES
+ ('project-instance', 'project-db', 'project-b');
+ `)
+ require.NoError(t, err)
+
+ reusedInstance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})
+ require.NoError(t, err)
+ require.NotNil(t, reusedInstance)
+ require.Equal(t, "project-b", *reusedInstance.ProjectID)
+ for _, find := range []*store.FindDatabaseMessage{
+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},
+ } {
+ reusedDB, err := s.GetDatabase(ctx, find)
+ require.NoError(t, err)
+ require.NotNil(t, reusedDB)
+ require.Equal(t, "project-b", reusedDB.ProjectID)
+ }
+ // The reused database has no schema row; a stale schema cache entry for the
+ // purged database must not leak.
+ reusedSchema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)
+ require.NoError(t, err)
+ require.Nil(t, reusedSchema)
+}
+
+// TestDeleteProjectFailedTransactionKeepsDescendantCaches pins that a purge
+// transaction that deletes descendants but fails before commit publishes no
+// invalidation. After the failed purge the underlying rows are removed
+// directly, so any surviving getter result can only come from the still-warm
+// cache entries.
+func TestDeleteProjectFailedTransactionKeepsDescendantCaches(t *testing.T) {
+ ctx, db, s := newProjectPurgeCacheFixture(t)
+ seedProjectPurgeFixture(ctx, t, db)
+ warmProjectPurgeCaches(ctx, t, s)
+
+ projectInstanceID := "project-instance"
+ projectDBName := "project-db"
+ // project-a is not marked deleted, so DeleteProject removes all descendant
+ // rows inside the transaction and then fails on the deleted guard, rolling
+ // the whole purge back.
+ _, err := db.ExecContext(ctx, `UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a'`)
+ require.NoError(t, err)
+ err = s.DeleteProject(ctx, "default", "project-a")
+ require.Error(t, err)
+
+ // Remove the descendant rows behind the store's back so that only a
+ // surviving cache entry can satisfy the getters.
+ _, err = db.ExecContext(ctx, `
+ DELETE FROM db_schema WHERE instance = 'project-instance' AND db_name = 'project-db';
+ DELETE FROM db WHERE instance = 'project-instance' AND name = 'project-db';
+ DELETE FROM instance WHERE resource_id = 'project-instance';
+ `)
+ require.NoError(t, err)
+
+ instance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})
+ require.NoError(t, err)
+ require.NotNil(t, instance)
+ require.Equal(t, "project-a", *instance.ProjectID)
+ database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName})
+ require.NoError(t, err)
+ require.NotNil(t, database)
+ require.Equal(t, "project-a", database.ProjectID)
+ schema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)
+ require.NoError(t, err)
+ require.NotNil(t, schema)
+}
diff --git a/backend/store/task_run.go b/backend/store/task_run.go
index 83baa7c24b..dbab7f0fb3 100644
--- a/backend/store/task_run.go
+++ b/backend/store/task_run.go
@@ -3,6 +3,7 @@ package store
import (
"context"
"database/sql"
+ "slices"
"time"
"github.com/pkg/errors"
@@ -241,7 +242,9 @@ func (s *Store) ClaimAvailableTaskRuns(ctx context.Context, replicaID string) ([
SELECT task_run.project, task_run.id
FROM task_run
JOIN project ON project.resource_id = task_run.project
- WHERE task_run.status = ? AND project.deleted = FALSE
+ JOIN task ON task.project = task_run.project AND task.id = task_run.task_id
+ JOIN instance ON instance.resource_id = task.instance
+ WHERE task_run.status = ? AND project.deleted = FALSE AND instance.deleted = FALSE
FOR UPDATE OF task_run SKIP LOCKED
)
RETURNING id, task_id, project
@@ -342,15 +345,24 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creat
payloadStr = s
}
}
+ instanceIDs, err := s.listTaskRunCreationInstances(ctx, projects, taskUIDs)
+ if err != nil {
+ return err
+ }
tx, err := s.GetDB().BeginTx(ctx, nil)
if err != nil {
return errors.Wrapf(err, "failed to begin tx")
}
defer tx.Rollback()
+ for _, instanceID := range instanceIDs {
+ if err := acquireInstancePurgeLock(ctx, tx, instanceID); err != nil {
+ return errors.Wrapf(err, "failed to lock instance lifecycle fence for %s", instanceID)
+ }
+ }
lockQ := qb.Q().Space(`
- SELECT task.project, task.id
+ SELECT task.project, task.id, task.instance
FROM (
SELECT
unnest(CAST(? AS TEXT[])) AS project,
@@ -364,6 +376,7 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creat
return errors.Wrapf(err, "failed to build task lock sql")
}
lockedTaskCount := 0
+ lockedInstanceIDs := make([]string, 0, len(instanceIDs))
if err := func() error {
rows, err := tx.QueryContext(ctx, lockQuery, lockArgs...)
if err != nil {
@@ -373,9 +386,14 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creat
for rows.Next() {
var lockedProjectID string
var lockedTaskUID int64
- if err := rows.Scan(&lockedProjectID, &lockedTaskUID); err != nil {
+ var lockedInstanceID string
+ if err := rows.Scan(&lockedProjectID, &lockedTaskUID, &lockedInstanceID); err != nil {
return errors.Wrapf(err, "failed to scan locked task")
}
+ if !slices.Contains(instanceIDs, lockedInstanceID) {
+ return common.Errorf(common.Conflict, "task %s/%d changed to instance %s; retry", lockedProjectID, lockedTaskUID, lockedInstanceID)
+ }
+ lockedInstanceIDs = append(lockedInstanceIDs, lockedInstanceID)
lockedTaskCount++
}
if err := rows.Err(); err != nil {
@@ -388,8 +406,24 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creat
if lockedTaskCount == 0 {
return nil
}
+ slices.Sort(lockedInstanceIDs)
+ lockedInstanceIDs = slices.Compact(lockedInstanceIDs)
+ for _, instanceID := range lockedInstanceIDs {
+ var deleted bool
+ if err := tx.QueryRowContext(ctx, `
+ SELECT deleted FROM instance WHERE resource_id = $1 FOR UPDATE
+ `, instanceID).Scan(&deleted); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return common.Errorf(common.NotFound, "instance %s not found", instanceID)
+ }
+ return errors.Wrapf(err, "failed to lock instance %s", instanceID)
+ }
+ if deleted {
+ return common.Errorf(common.Conflict, "instance %s is archived", instanceID)
+ }
+ }
- // Keep the child-to-parent lock order used by project deletion: task, then project.
+ // Keep the child-to-parent lock order used by project deletion: task, instance, then project.
baseID, err := nextProjectID(ctx, tx, "task_run", projectID)
if err != nil {
return err
@@ -465,6 +499,40 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creat
return nil
}
+func (s *Store) listTaskRunCreationInstances(ctx context.Context, projects []string, taskUIDs []int64) ([]string, error) {
+ q := qb.Q().Space(`
+ SELECT DISTINCT task.instance
+ FROM (
+ SELECT
+ unnest(CAST(? AS TEXT[])) AS project,
+ unnest(CAST(? AS BIGINT[])) AS task_id
+ ) requested_tasks
+ JOIN task ON task.project = requested_tasks.project AND task.id = requested_tasks.task_id
+ ORDER BY task.instance
+ `, projects, taskUIDs)
+ query, args, err := q.ToSQL()
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to build task run instance query")
+ }
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to list task run instances")
+ }
+ defer rows.Close()
+ var instanceIDs []string
+ for rows.Next() {
+ var instanceID string
+ if err := rows.Scan(&instanceID); err != nil {
+ return nil, errors.Wrap(err, "failed to scan task run instance")
+ }
+ instanceIDs = append(instanceIDs, instanceID)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, errors.Wrap(err, "failed to read task run instances")
+ }
+ return instanceIDs, nil
+}
+
// patchTaskRunStatusImpl updates a taskRun status. Returns the new state of the taskRun after update.
func (*Store) patchTaskRunStatusImpl(ctx context.Context, txn *sql.Tx, patch *TaskRunStatusPatch) (*TaskRunMessage, error) {
set := qb.Q()
diff --git a/backend/store/task_run_lock_order_test.go b/backend/store/task_run_lock_order_test.go
index 25a3208f03..1ddac9f5d7 100644
--- a/backend/store/task_run_lock_order_test.go
+++ b/backend/store/task_run_lock_order_test.go
@@ -3,12 +3,15 @@ package store_test
import (
"context"
"database/sql"
+ "fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ "github.com/bytebase/bytebase/backend/migrator"
"github.com/bytebase/bytebase/backend/store"
)
@@ -40,6 +43,87 @@ func TestClaimAvailableTaskRunsSkipsDeletedProjects(t *testing.T) {
require.False(t, replicaID.Valid)
}
+func TestClaimAvailableTaskRunsSkipsArchivedInstances(t *testing.T) {
+ fixture := newTaskRunArchivalLockOrderFixture(t, `
+ INSERT INTO instance (resource_id, workspace, deleted) VALUES ('instance-a', 'default', TRUE);
+ INSERT INTO plan (id, creator, project, name, description)
+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_MIGRATE');
+ INSERT INTO task_run (id, project, task_id, attempt, status)
+ VALUES (101, 'project-a', 101, 0, 'AVAILABLE');
+ `)
+
+ claimed, err := fixture.store.ClaimAvailableTaskRuns(fixture.ctx, "replica-a")
+ require.Empty(t, claimed)
+ require.NoError(t, err)
+
+ var status string
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT status FROM task_run WHERE project = 'project-a' AND id = 101",
+ ).Scan(&status))
+ require.Equal(t, "AVAILABLE", status)
+
+ var replicaID sql.NullString
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT replica_id FROM task_run WHERE project = 'project-a' AND id = 101",
+ ).Scan(&replicaID))
+ require.False(t, replicaID.Valid)
+}
+
+func TestClaimAvailableTaskRunsClaimsLiveInstanceSpecialTaskTypes(t *testing.T) {
+ fixture := newTaskRunArchivalLockOrderFixture(t, `
+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');
+ INSERT INTO plan (id, creator, project, name, description)
+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_MIGRATE');
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (102, 'project-a', 101, 'instance-a', 'DATABASE_CREATE');
+ INSERT INTO task_run (id, project, task_id, attempt, status)
+ VALUES (101, 'project-a', 101, 0, 'AVAILABLE');
+ INSERT INTO task_run (id, project, task_id, attempt, status)
+ VALUES (102, 'project-a', 102, 0, 'AVAILABLE');
+ `)
+
+ claimed, err := fixture.store.ClaimAvailableTaskRuns(fixture.ctx, "replica-a")
+ require.NoError(t, err)
+ require.Len(t, claimed, 2, "live-instance task runs of any task type should be claimable")
+
+ var status string
+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,
+ "SELECT status FROM task_run WHERE project = 'project-a' AND id = 101",
+ ).Scan(&status))
+ require.Equal(t, "RUNNING", status)
+}
+
+func newTaskRunArchivalLockOrderFixture(t *testing.T, seedSQL string) *projectDeletionLockOrderFixture {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ t.Cleanup(cancel)
+ container := testcontainer.GetTestPgContainer(ctx, t)
+ t.Cleanup(func() { container.Close(context.Background()) })
+ db := container.GetDB()
+ require.NoError(t, migrator.MigrateSchema(ctx, db))
+
+ _, err := db.ExecContext(ctx, `
+ INSERT INTO workspace (resource_id) VALUES ('default');
+ INSERT INTO project (resource_id, workspace, name) VALUES ('default', 'default', 'Default');
+ INSERT INTO project (resource_id, workspace, name) VALUES ('project-a', 'default', 'Project A');
+ `+seedSQL)
+ require.NoError(t, err)
+
+ pgURL := fmt.Sprintf(
+ "host=%s port=%s user=postgres password=root-password database=postgres",
+ container.GetHost(), container.GetPort(),
+ )
+ s, err := store.New(ctx, pgURL, false)
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+ return &projectDeletionLockOrderFixture{ctx: ctx, db: db, store: s}
+}
+
func TestCreatePendingTaskRunsDoesNotDeadlockWithProjectDeletion(t *testing.T) {
fixture := newProjectDeletionLockOrderFixture(t, `
INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');
diff --git a/backend/tests/project_instance_archive_task_run_test.go b/backend/tests/project_instance_archive_task_run_test.go
new file mode 100644
index 0000000000..136540431d
--- /dev/null
+++ b/backend/tests/project_instance_archive_task_run_test.go
@@ -0,0 +1,200 @@
+package tests
+
+import (
+ "context"
+ "database/sql"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+//nolint:tparallel // Subtests mutate one task run through successive lifecycle states.
+func TestArchiveInstanceRejectsActiveTaskRuns(t *testing.T) {
+ t.Parallel()
+ a := require.New(t)
+ ctx := context.Background()
+ ctl := &controller{}
+ ctx, err := ctl.StartServerWithExternalPg(ctx)
+ a.NoError(err)
+ defer ctl.Close(ctx)
+
+ pg, err := provisionPgInstance(ctx, t)
+ a.NoError(err)
+ instance := createProjectInstanceTestInstance(
+ ctx,
+ t,
+ ctl,
+ &ctl.project.Name,
+ "archive-active-task-runs",
+ "archive active task runs",
+ pg,
+ )
+ projectID, err := common.GetProjectID(ctl.project.Name)
+ a.NoError(err)
+ _, instanceID, err := common.GetInstanceResourceName(instance.Name)
+ a.NoError(err)
+
+ metadataDB, err := sql.Open("pgx", ctl.profile.PgURL)
+ a.NoError(err)
+ defer metadataDB.Close()
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO plan (id, creator, project, name, description)
+ VALUES (101, '[email protected]', $1, 'Archive precondition plan', '')
+ `, projectID)
+ a.NoError(err)
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (101, $1, 101, $2, 'DATABASE_CREATE')
+ `, projectID, instanceID)
+ a.NoError(err)
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO task_run (id, project, task_id, attempt, status)
+ VALUES (101, $1, 101, 0, 'PENDING')
+ `, projectID)
+ a.NoError(err)
+
+ for _, status := range []string{"PENDING", "AVAILABLE", "RUNNING"} {
+ t.Run(status, func(t *testing.T) {
+ _, err := metadataDB.ExecContext(ctx, `
+ UPDATE task_run SET status = $1 WHERE project = $2 AND id = 101
+ `, status, projectID)
+ require.NoError(t, err)
+
+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{
+ Name: instance.Name,
+ }))
+ require.Error(t, err)
+ require.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err))
+
+ got, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))
+ require.NoError(t, err)
+ require.Equal(t, v1pb.State_ACTIVE, got.Msg.State)
+ })
+ }
+
+ _, err = metadataDB.ExecContext(ctx, `
+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101
+ `, projectID)
+ a.NoError(err)
+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{
+ Name: instance.Name,
+ }))
+ a.NoError(err)
+ archived, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))
+ a.NoError(err)
+ a.Equal(v1pb.State_DELETED, archived.Msg.State)
+
+ // Legacy data may contain an active task run on an already archived
+ // instance. Restoration must not make that work runnable without an
+ // explicit cancel-and-retry decision from the user.
+ _, err = metadataDB.ExecContext(ctx, `
+ UPDATE task_run SET status = 'PENDING' WHERE project = $1 AND id = 101
+ `, projectID)
+ a.NoError(err)
+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{
+ Name: instance.Name,
+ }))
+ a.Error(err)
+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))
+ _, err = ctl.instanceServiceClient.UndeleteInstance(ctx, connect.NewRequest(&v1pb.UndeleteInstanceRequest{
+ Name: instance.Name,
+ }))
+ a.Error(err)
+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))
+ stillArchived, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))
+ a.NoError(err)
+ a.Equal(v1pb.State_DELETED, stillArchived.Msg.State)
+
+ _, err = metadataDB.ExecContext(ctx, `
+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101
+ `, projectID)
+ a.NoError(err)
+ restored, err := ctl.instanceServiceClient.UndeleteInstance(ctx, connect.NewRequest(&v1pb.UndeleteInstanceRequest{
+ Name: instance.Name,
+ }))
+ a.NoError(err)
+ a.Equal(v1pb.State_ACTIVE, restored.Msg.State)
+}
+
+func TestForceArchiveInstanceRejectsActiveTaskRunsWithoutTransferringDatabases(t *testing.T) {
+ t.Parallel()
+ a := require.New(t)
+ ctx := context.Background()
+ ctl := &controller{}
+ ctx, err := ctl.StartServerWithExternalPg(ctx)
+ a.NoError(err)
+ defer ctl.Close(ctx)
+
+ pg, err := provisionPgInstance(ctx, t)
+ a.NoError(err)
+ instance := createProjectInstanceTestInstance(
+ ctx,
+ t,
+ ctl,
+ nil,
+ "force-archive-active-task-runs",
+ "force archive active task runs",
+ pg,
+ )
+ projectID, err := common.GetProjectID(ctl.project.Name)
+ a.NoError(err)
+ _, instanceID, err := common.GetInstanceResourceName(instance.Name)
+ a.NoError(err)
+
+ metadataDB, err := sql.Open("pgx", ctl.profile.PgURL)
+ a.NoError(err)
+ defer metadataDB.Close()
+ var workspaceID string
+ a.NoError(metadataDB.QueryRowContext(ctx, `
+ SELECT workspace FROM instance WHERE resource_id = $1
+ `, instanceID).Scan(&workspaceID))
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO db (instance, name, project) VALUES ($1, 'db-a', $2)
+ `, instanceID, projectID)
+ a.NoError(err)
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO plan (id, creator, project, name, description)
+ VALUES (101, '[email protected]', $1, 'Force archive precondition plan', '')
+ `, projectID)
+ a.NoError(err)
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO task (id, project, plan_id, instance, type)
+ VALUES (101, $1, 101, $2, 'DATABASE_CREATE')
+ `, projectID, instanceID)
+ a.NoError(err)
+ _, err = metadataDB.ExecContext(ctx, `
+ INSERT INTO task_run (id, project, task_id, attempt, status)
+ VALUES (101, $1, 101, 0, 'RUNNING')
+ `, projectID)
+ a.NoError(err)
+
+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{
+ Name: instance.Name,
+ Force: true,
+ }))
+ a.Error(err)
+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))
+ var databaseProjectID string
+ a.NoError(metadataDB.QueryRowContext(ctx, `
+ SELECT project FROM db WHERE instance = $1 AND name = 'db-a'
+ `, instanceID).Scan(&databaseProjectID))
+ a.Equal(projectID, databaseProjectID, "a rejected force archive must not transfer databases")
+
+ _, err = metadataDB.ExecContext(ctx, `
+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101
+ `, projectID)
+ a.NoError(err)
+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{
+ Name: instance.Name,
+ Force: true,
+ }))
+ a.NoError(err)
+ a.NoError(metadataDB.QueryRowContext(ctx, `
+ SELECT project FROM db WHERE instance = $1 AND name = 'db-a'
+ `, instanceID).Scan(&databaseProjectID))
+ a.True(common.IsDefaultProject(workspaceID, databaseProjectID))
+}
diff --git a/docs/adr/0002-support-workspace-and-project-instances.md b/docs/adr/0002-support-workspace-and-project-instances.md
index 71d027af9e..4d0bc43e66 100644
--- a/docs/adr/0002-support-workspace-and-project-instances.md
+++ b/docs/adr/0002-support-workspace-and-project-instances.md
@@ -189,17 +189,20 @@ writers so that no writer can commit after the owning project is removed.
Directly archiving a project instance leaves its databases assigned to the
owning project, but they are unavailable to data and workflow operations until
-the instance is restored. Instance restore and purge operations remain
-available. Archival does not stop or delete the physical databases and does not
-fence already-started descendant writers; their committed results remain
-unavailable until restoration. Purging serializes against those writers, then
-deletes the databases, their metadata and history, and instance-targeting tasks
-and task runs. Project-level issues, plans, and releases remain, retaining
-canonical target names that may later identify a replacement resource if the
-purged instance ID is reused; audit logs also remain under the workspace
-retention policy. This matches the workspace-instance purge boundary, except
-that the workspace-instance `force` behavior that transfers databases to the
-default project does not apply. `DeleteInstance` rejects `force=true` with
+the instance is restored. Archive and restore return `FAILED_PRECONDITION` while
+any targeting task run is `PENDING`, `AVAILABLE`, or `RUNNING`; operators must
+cancel that work or wait for it to finish before retrying the lifecycle request.
+Applying the same guard to restoration prevents legacy queued work from
+silently resuming. Archival does not stop or delete the physical databases.
+Purging serializes against descendant writers, then deletes the databases,
+their metadata and history, and instance-targeting tasks and task runs.
+Project-level issues, plans, and releases remain, retaining canonical target
+names that may later identify a replacement resource if the purged instance ID
+is reused; audit logs also remain under the workspace retention policy. This
+matches the workspace-instance purge boundary, except that the
+workspace-instance `force` behavior that transfers databases to the default
+project does not apply. For workspace instances, that transfer and archival
+commit atomically after the active-task-run check. `DeleteInstance` rejects `force=true` with
`INVALID_ARGUMENT` for a project instance; ordinary archival needs no force
because its databases remain owned by the project.
diff --git a/frontend/src/types/proto-es/v1/instance_service_pb.d.ts b/frontend/src/types/proto-es/v1/instance_service_pb.d.ts
index 96533a80f9..0785d7d6b8 100644
--- a/frontend/src/types/proto-es/v1/instance_service_pb.d.ts
+++ b/frontend/src/types/proto-es/v1/instance_service_pb.d.ts
@@ -1822,6 +1822,8 @@ export declare const InstanceService: GenService<{
},
/**
* Deletes or soft-deletes a database instance.
+ * Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ * targeting the instance is pending, available, or running.
* Permissions required: bb.instances.delete
*
* @generated from rpc bytebase.v1.InstanceService.DeleteInstance
@@ -1833,6 +1835,8 @@ export declare const InstanceService: GenService<{
},
/**
* Restores a soft-deleted database instance.
+ * Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ * the instance is pending, available, or running.
* Permissions required: bb.instances.undelete
*
* @generated from rpc bytebase.v1.InstanceService.UndeleteInstance
diff --git a/proto/gen/grpc-doc/openapi.yaml b/proto/gen/grpc-doc/openapi.yaml
index c07531efa6..f097cd19b7 100644
--- a/proto/gen/grpc-doc/openapi.yaml
+++ b/proto/gen/grpc-doc/openapi.yaml
@@ -1872,6 +1872,8 @@ paths:
summary: DeleteInstance
description: |-
Deletes or soft-deletes a database instance.
+ Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ targeting the instance is pending, available, or running.
Permissions required: bb.instances.delete
operationId: InstanceService_DeleteInstance
parameters:
@@ -5123,6 +5125,8 @@ paths:
summary: UndeleteInstance
description: |-
Restores a soft-deleted database instance.
+ Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ the instance is pending, available, or running.
Permissions required: bb.instances.undelete
operationId: InstanceService_UndeleteInstance
parameters:
@@ -6936,6 +6940,8 @@ paths:
summary: DeleteInstance
description: |-
Deletes or soft-deletes a database instance.
+ Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ targeting the instance is pending, available, or running.
Permissions required: bb.instances.delete
operationId: InstanceService_DeleteInstance2
parameters:
@@ -10196,6 +10202,8 @@ paths:
summary: UndeleteInstance
description: |-
Restores a soft-deleted database instance.
+ Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ the instance is pending, available, or running.
Permissions required: bb.instances.undelete
operationId: InstanceService_UndeleteInstance2
parameters:
diff --git a/proto/gen/grpc-doc/v1/README.md b/proto/gen/grpc-doc/v1/README.md
index 6abb05f76e..3a3e1a3ccd 100644
--- a/proto/gen/grpc-doc/v1/README.md
+++ b/proto/gen/grpc-doc/v1/README.md
@@ -7219,8 +7219,8 @@ InstanceService manages database instances and their connections.
| ListInstances | [ListInstancesRequest](#bytebase-v1-ListInstancesRequest) | [ListInstancesResponse](#bytebase-v1-ListInstancesResponse) | Lists database instances, optionally within a project. Permissions required: bb.instances.list |
| CreateInstance | [CreateInstanceRequest](#bytebase-v1-CreateInstanceRequest) | [Instance](#bytebase-v1-Instance) | Creates a new database instance. Permissions required: bb.instances.create |
| UpdateInstance | [UpdateInstanceRequest](#bytebase-v1-UpdateInstanceRequest) | [Instance](#bytebase-v1-Instance) | Updates a database instance. Permissions required: bb.instances.update |
-| DeleteInstance | [DeleteInstanceRequest](#bytebase-v1-DeleteInstanceRequest) | [.google.protobuf.Empty](#google-protobuf-Empty) | Deletes or soft-deletes a database instance. Permissions required: bb.instances.delete |
-| UndeleteInstance | [UndeleteInstanceRequest](#bytebase-v1-UndeleteInstanceRequest) | [Instance](#bytebase-v1-Instance) | Restores a soft-deleted database instance. Permissions required: bb.instances.undelete |
+| DeleteInstance | [DeleteInstanceRequest](#bytebase-v1-DeleteInstanceRequest) | [.google.protobuf.Empty](#google-protobuf-Empty) | Deletes or soft-deletes a database instance. Soft-delete requests fail with FAILED_PRECONDITION while any task run targeting the instance is pending, available, or running. Permissions required: bb.instances.delete |
+| UndeleteInstance | [UndeleteInstanceRequest](#bytebase-v1-UndeleteInstanceRequest) | [Instance](#bytebase-v1-Instance) | Restores a soft-deleted database instance. Restore requests fail with FAILED_PRECONDITION while any task run targeting the instance is pending, available, or running. Permissions required: bb.instances.undelete |
| SyncInstance | [SyncInstanceRequest](#bytebase-v1-SyncInstanceRequest) | [SyncInstanceResponse](#bytebase-v1-SyncInstanceResponse) | Syncs database schemas and metadata from an instance. Permissions required: bb.instances.sync |
| ListInstanceDatabase | [ListInstanceDatabaseRequest](#bytebase-v1-ListInstanceDatabaseRequest) | [ListInstanceDatabaseResponse](#bytebase-v1-ListInstanceDatabaseResponse) | Lists all databases within an instance without creating them. Permissions required: bb.instances.get |
| BatchSyncInstances | [BatchSyncInstancesRequest](#bytebase-v1-BatchSyncInstancesRequest) | [BatchSyncInstancesResponse](#bytebase-v1-BatchSyncInstancesResponse) | Syncs multiple instances in a single request. Permissions required: bb.instances.sync |
diff --git a/proto/gen/grpc-doc/v1/index.html b/proto/gen/grpc-doc/v1/index.html
index 88021dda46..87e3159283 100644
--- a/proto/gen/grpc-doc/v1/index.html
+++ b/proto/gen/grpc-doc/v1/index.html
@@ -20325,6 +20325,8 @@ Permissions required: bb.instances.update</p></td>
<td><a href="#bytebase.v1.DeleteInstanceRequest">DeleteInstanceRequest</a></td>
<td><a href="#google.protobuf.Empty">.google.protobuf.Empty</a></td>
<td><p>Deletes or soft-deletes a database instance.
+Soft-delete requests fail with FAILED_PRECONDITION while any task run
+targeting the instance is pending, available, or running.
Permissions required: bb.instances.delete</p></td>
</tr>
@@ -20333,6 +20335,8 @@ Permissions required: bb.instances.delete</p></td>
<td><a href="#bytebase.v1.UndeleteInstanceRequest">UndeleteInstanceRequest</a></td>
<td><a href="#bytebase.v1.Instance">Instance</a></td>
<td><p>Restores a soft-deleted database instance.
+Restore requests fail with FAILED_PRECONDITION while any task run targeting
+the instance is pending, available, or running.
Permissions required: bb.instances.undelete</p></td>
</tr>
diff --git a/proto/v1/v1/instance_service.proto b/proto/v1/v1/instance_service.proto
index 3d13235951..d9df5f7428 100644
--- a/proto/v1/v1/instance_service.proto
+++ b/proto/v1/v1/instance_service.proto
@@ -78,6 +78,8 @@ service InstanceService {
}
// Deletes or soft-deletes a database instance.
+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run
+ // targeting the instance is pending, available, or running.
// Permissions required: bb.instances.delete
rpc DeleteInstance(DeleteInstanceRequest) returns (google.protobuf.Empty) {
option (google.api.http) = {
@@ -91,6 +93,8 @@ service InstanceService {
}
// Restores a soft-deleted database instance.
+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting
+ // the instance is pending, available, or running.
// Permissions required: bb.instances.undelete
rpc UndeleteInstance(UndeleteInstanceRequest) returns (Instance) {
option (google.api.http) = {