Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: harden project instance archive and purge paths (#21118)
p0ny1b86bb7560e2
diff --git a/backend/api/mcp/gen/openapi.yaml b/backend/api/mcp/gen/openapi.yamlindex 596fa39385..29846b828a 100644--- a/backend/api/mcp/gen/openapi.yaml+++ b/backend/api/mcp/gen/openapi.yaml@@ -2327,6 +2327,8 @@ paths:summary: DeleteInstancedescription: |-Deletes or soft-deletes a database instance.+ Soft-delete requests fail with FAILED_PRECONDITION while any task run+ targeting the instance is pending, available, or running.Permissions required: bb.instances.deleteoperationId: bytebase.v1.InstanceService.DeleteInstanceparameters:@@ -2555,6 +2557,8 @@ paths:summary: UndeleteInstancedescription: |-Restores a soft-deleted database instance.+ Restore requests fail with FAILED_PRECONDITION while any task run targeting+ the instance is pending, available, or running.Permissions required: bb.instances.undeleteoperationId: bytebase.v1.InstanceService.UndeleteInstanceparameters:diff --git a/backend/api/v1/instance_service.go b/backend/api/v1/instance_service.goindex 93891ccc36..1ac4f15f9b 100644--- a/backend/api/v1/instance_service.go+++ b/backend/api/v1/instance_service.go@@ -926,28 +926,22 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Requereturn connect.NewResponse(&emptypb.Empty{}), nil}- // Regular soft delete flow- // Idempotent: if already deleted, return success- if instance.Deleted {- return connect.NewResponse(&emptypb.Empty{}), nil- }-- if instance.ProjectID == nil {- databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID})- if err != nil {- return nil, err- }+ // Regular soft delete flow. Already archived instances still pass through+ // the lifecycle guard so legacy active task runs are reported consistently.+ alreadyDeleted := instance.Deleted+ var moveDatabasesToProjectID *string+ if !alreadyDeleted && instance.ProjectID == nil {if req.Msg.Force {- if len(databases) > 0 {- defaultProjectID, err := s.store.GetDefaultProjectID(ctx, common.GetWorkspaceIDFromContext(ctx))- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get default project ID"))- }- if err := s.store.BatchUpdateDatabases(ctx, databases, &store.BatchUpdateDatabases{Workspace: common.GetWorkspaceIDFromContext(ctx), ProjectID: &defaultProjectID}); err != nil {- return nil, connect.NewError(connect.CodeInternal, err)- }+ defaultProjectID, err := s.store.GetDefaultProjectID(ctx, common.GetWorkspaceIDFromContext(ctx))+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to get default project ID"))}+ moveDatabasesToProjectID = &defaultProjectID} else {+ databases, err := s.store.ListDatabases(ctx, &store.FindDatabaseMessage{Workspace: common.GetWorkspaceIDFromContext(ctx), InstanceID: &instance.ResourceID})+ if err != nil {+ return nil, err+ }var databaseNames []stringfor _, database := range databases {if !common.IsDefaultProject(common.GetWorkspaceIDFromContext(ctx), database.ProjectID) {@@ -961,9 +955,10 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Reque}patch := &store.UpdateInstanceMessage{- ResourceID: &instance.ResourceID,- Workspace: instance.Workspace,- Deleted: &deletePatch,+ ResourceID: &instance.ResourceID,+ Workspace: instance.Workspace,+ Deleted: &deletePatch,+ MoveDatabasesToProjectID: moveDatabasesToProjectID,}if instance.ProjectID == nil {metadata := proto.CloneOf(instance.Metadata)@@ -971,11 +966,14 @@ func (s *InstanceService) DeleteInstance(ctx context.Context, req *connect.Requepatch.Metadata = metadata}if _, err := s.store.UpdateInstance(ctx, patch); err != nil {+ if common.ErrorCode(err) == common.Conflict {+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)+ }return nil, connect.NewError(connect.CodeInternal, err)}// Handle sample instance deletion if applicable- if s.sampleInstanceManager != nil {+ if !alreadyDeleted && s.sampleInstanceManager != nil {if err := s.sampleInstanceManager.HandleInstanceDeletion(ctx, common.GetWorkspaceIDFromContext(ctx), instance.ResourceID); err != nil {slog.Warn("failed to handle sample instance deletion", log.BBError(err), slog.String("instance", instance.ResourceID))}@@ -995,6 +993,13 @@ func (s *InstanceService) UndeleteInstance(ctx context.Context, req *connect.Reqresult := s.convertToV1Instance(ctx, instance)return connect.NewResponse(result), nil}+ activeTaskRunCount, err := s.store.GetActiveTaskRunCountForInstance(ctx, instance.ResourceID)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, err)+ }+ if activeTaskRunCount > 0 {+ return nil, connect.NewError(connect.CodeFailedPrecondition, errors.Errorf("instance %s has %d active task run(s); cancel them or wait for them to finish before restoring", instance.ResourceID, activeTaskRunCount))+ }if err := s.instanceCountGuard(ctx); err != nil {return nil, err}@@ -1008,6 +1013,9 @@ func (s *InstanceService) UndeleteInstance(ctx context.Context, req *connect.ReqDeleted: &undeletePatch,})if err != nil {+ if common.ErrorCode(err) == common.Conflict {+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)+ }return nil, connect.NewError(connect.CodeInternal, err)}diff --git a/backend/api/v1/rollout_service.go b/backend/api/v1/rollout_service.goindex 31971c99a0..7488cb1d27 100644--- a/backend/api/v1/rollout_service.go+++ b/backend/api/v1/rollout_service.go@@ -891,6 +891,9 @@ func (s *RolloutService) BatchRunTasks(ctx context.Context, req *connect.Request}if err := s.store.CreatePendingTaskRuns(ctx, user.Email, taskRunCreates...); err != nil {+ if common.ErrorCode(err) == common.Conflict {+ return nil, connect.NewError(connect.CodeFailedPrecondition, err)+ }return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create pending task runs, error %v", err))}diff --git a/backend/generated-go/v1/instance_service_grpc.pb.go b/backend/generated-go/v1/instance_service_grpc.pb.goindex 81a8428c5d..bffb3fd91e 100644--- a/backend/generated-go/v1/instance_service_grpc.pb.go+++ b/backend/generated-go/v1/instance_service_grpc.pb.go@@ -54,9 +54,13 @@ type InstanceServiceClient interface {// Permissions required: bb.instances.updateUpdateInstance(ctx context.Context, in *UpdateInstanceRequest, opts ...grpc.CallOption) (*Instance, error)// Deletes or soft-deletes a database instance.+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run+ // targeting the instance is pending, available, or running.// Permissions required: bb.instances.deleteDeleteInstance(ctx context.Context, in *DeleteInstanceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)// Restores a soft-deleted database instance.+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting+ // the instance is pending, available, or running.// Permissions required: bb.instances.undeleteUndeleteInstance(ctx context.Context, in *UndeleteInstanceRequest, opts ...grpc.CallOption) (*Instance, error)// Syncs database schemas and metadata from an instance.@@ -239,9 +243,13 @@ type InstanceServiceServer interface {// Permissions required: bb.instances.updateUpdateInstance(context.Context, *UpdateInstanceRequest) (*Instance, error)// Deletes or soft-deletes a database instance.+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run+ // targeting the instance is pending, available, or running.// Permissions required: bb.instances.deleteDeleteInstance(context.Context, *DeleteInstanceRequest) (*emptypb.Empty, error)// Restores a soft-deleted database instance.+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting+ // the instance is pending, available, or running.// Permissions required: bb.instances.undeleteUndeleteInstance(context.Context, *UndeleteInstanceRequest) (*Instance, error)// Syncs database schemas and metadata from an instance.diff --git a/backend/generated-go/v1/v1connect/instance_service.connect.go b/backend/generated-go/v1/v1connect/instance_service.connect.goindex 6a090ec344..e886ed83d8 100644--- a/backend/generated-go/v1/v1connect/instance_service.connect.go+++ b/backend/generated-go/v1/v1connect/instance_service.connect.go@@ -90,9 +90,13 @@ type InstanceServiceClient interface {// Permissions required: bb.instances.updateUpdateInstance(context.Context, *connect.Request[v1.UpdateInstanceRequest]) (*connect.Response[v1.Instance], error)// Deletes or soft-deletes a database instance.+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run+ // targeting the instance is pending, available, or running.// Permissions required: bb.instances.deleteDeleteInstance(context.Context, *connect.Request[v1.DeleteInstanceRequest]) (*connect.Response[emptypb.Empty], error)// Restores a soft-deleted database instance.+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting+ // the instance is pending, available, or running.// Permissions required: bb.instances.undeleteUndeleteInstance(context.Context, *connect.Request[v1.UndeleteInstanceRequest]) (*connect.Response[v1.Instance], error)// Syncs database schemas and metadata from an instance.@@ -307,9 +311,13 @@ type InstanceServiceHandler interface {// Permissions required: bb.instances.updateUpdateInstance(context.Context, *connect.Request[v1.UpdateInstanceRequest]) (*connect.Response[v1.Instance], error)// Deletes or soft-deletes a database instance.+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run+ // targeting the instance is pending, available, or running.// Permissions required: bb.instances.deleteDeleteInstance(context.Context, *connect.Request[v1.DeleteInstanceRequest]) (*connect.Response[emptypb.Empty], error)// Restores a soft-deleted database instance.+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting+ // the instance is pending, available, or running.// Permissions required: bb.instances.undeleteUndeleteInstance(context.Context, *connect.Request[v1.UndeleteInstanceRequest]) (*connect.Response[v1.Instance], error)// Syncs database schemas and metadata from an instance.diff --git a/backend/runner/taskrun/pending_scheduler.go b/backend/runner/taskrun/pending_scheduler.goindex 5860ff359d..deeb30841a 100644--- a/backend/runner/taskrun/pending_scheduler.go+++ b/backend/runner/taskrun/pending_scheduler.go@@ -128,13 +128,26 @@ func (s *Scheduler) schedulePendingTaskRun(ctx context.Context, taskRun *store.Treturn nil}- // Check 3: Database mutual exclusion (for sequential tasks)+ // Check 3: Instance archival. A pending task run must not be promoted to+ // AVAILABLE while its target instance is archived.+ instance, err := s.store.GetInstanceByResourceID(ctx, task.InstanceID)+ if err != nil {+ return errors.Wrapf(err, "failed to get instance")+ }+ if instance == nil {+ return errors.Errorf("instance %v not found", task.InstanceID)+ }+ if instance.Deleted {+ return nil+ }++ // Check 4: Database mutual exclusion (for sequential tasks)canProceed, _ := sc.checkDatabaseMutualExclusion(task)if !canProceed {return nil}- // Check 4: Parallel task limit per rollout+ // Check 5: Parallel task limit per rolloutmaxParallel, err := s.getMaxParallelForTask(ctx, task)if err != nil {return errors.Wrapf(err, "failed to get max parallel limit")diff --git a/backend/runner/taskrun/running_scheduler.go b/backend/runner/taskrun/running_scheduler.goindex d3bfdedbb4..d92d6b81d4 100644--- a/backend/runner/taskrun/running_scheduler.go+++ b/backend/runner/taskrun/running_scheduler.go@@ -238,9 +238,23 @@ func (s *Scheduler) runTaskRunOnce(ctx context.Context, taskRunUID int64, task *}// validateTaskFreshness checks for state drift between task creation and execution time.-// Returns an error if the target database has been deleted, its project has changed,-// or its environment has changed since the task was created.+// Returns an error if the target instance has been archived or deleted, the target+// database has been deleted, its project has changed, or its environment has+// changed since the task was created.func (s *Scheduler) validateTaskFreshness(ctx context.Context, task *store.TaskMessage) error {+ // Every task targets an instance; a task run must not newly start while its+ // target instance is archived. Already-started task runs are not affected.+ instance, err := s.store.GetInstanceByResourceID(ctx, task.InstanceID)+ if err != nil {+ return errors.Wrapf(err, "failed to get instance for drift validation")+ }+ if instance == nil {+ return errors.Errorf("target instance %q has been deleted", task.InstanceID)+ }+ if instance.Deleted {+ return errors.Errorf("target instance %q has been archived", task.InstanceID)+ }+// DATABASE_CREATE tasks have DatabaseName = nil — the database doesn't exist yet.if task.Type == storepb.Task_DATABASE_CREATE {return nildiff --git a/backend/runner/taskrun/running_scheduler_test.go b/backend/runner/taskrun/running_scheduler_test.goindex ac107d6fef..b62aa2c001 100644--- a/backend/runner/taskrun/running_scheduler_test.go+++ b/backend/runner/taskrun/running_scheduler_test.go@@ -6,6 +6,8 @@ import ("github.com/stretchr/testify/require"+ "github.com/bytebase/bytebase/backend/component/bus"+ "github.com/bytebase/bytebase/backend/component/config"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/store")@@ -143,14 +145,113 @@ func TestCheckTaskDrift(t *testing.T) {}}-func TestValidateTaskFreshness_DatabaseCreateSkipsValidation(t *testing.T) {- // Construct a Scheduler with nil store. If validateTaskFreshness tries to- // call the store for DATABASE_CREATE tasks, it will panic — proving the- // early return works.- s := &Scheduler{}- task := &store.TaskMessage{- Type: storepb.Task_DATABASE_CREATE,- }- err := s.validateTaskFreshness(context.Background(), task)+func TestValidateTaskFreshnessChecksInstanceArchival(t *testing.T) {+ ctx := context.Background()+ s := setupRolloutCreatorStore(ctx, t)+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: "project-a",+ Name: "freshness validation plan",+ Config: &storepb.PlanConfig{},+ }, "[email protected]")+ require.NoError(t, err)+ instanceID := "instance-a"+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: instanceID,+ Workspace: "default",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)++ tx, err := s.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{+ InstanceID: instanceID,+ Type: storepb.Task_DATABASE_CREATE,+ Payload: &storepb.Task{},+ }})+ require.NoError(t, err)+ require.Len(t, tasks, 1)+ require.NoError(t, tx.Commit())++ scheduler := &Scheduler{store: s}+ // DATABASE_CREATE has no database target yet, but its target instance must+ // still be live for the task run to be allowed to start.+ require.NoError(t, scheduler.validateTaskFreshness(ctx, tasks[0]), "a live instance should pass freshness validation")++ // Seed a legacy state to keep freshness validation defensive for archived+ // DATABASE_CREATE tasks that have no database target yet.+ _, err = s.GetDB().ExecContext(ctx, `+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1+ `, instanceID)+ require.NoError(t, err)+ err = scheduler.validateTaskFreshness(ctx, tasks[0])+ require.Error(t, err)+ require.Contains(t, err.Error(), "archived", "an archived instance must block a DATABASE_CREATE task from starting")+}++func TestScheduleRunningTaskRunsSkipsArchivedInstance(t *testing.T) {+ ctx := context.Background()+ s := setupRolloutCreatorStore(ctx, t)+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: "project-a",+ Name: "archived instance claim plan",+ Config: &storepb.PlanConfig{},+ }, "[email protected]")+ require.NoError(t, err)+ instanceID := "instance-a"+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: instanceID,+ Workspace: "default",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)++ tx, err := s.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{+ InstanceID: instanceID,+ Type: storepb.Task_DATABASE_CREATE,+ Payload: &storepb.Task{},+ }})+ require.NoError(t, err)+ require.Len(t, tasks, 1)+ require.NoError(t, tx.Commit())+ require.NoError(t, s.CreatePendingTaskRuns(ctx, "", &store.TaskRunMessage{+ ProjectID: plan.ProjectID,+ TaskUID: tasks[0].ID,+ }))+ taskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})+ require.NoError(t, err)+ require.Len(t, taskRuns, 1)+ _, err = s.UpdateTaskRunStatus(ctx, &store.TaskRunStatusPatch{+ ID: taskRuns[0].ID,+ ProjectID: plan.ProjectID,+ Status: storepb.TaskRun_AVAILABLE,+ AllowedStatuses: []storepb.TaskRun_Status{storepb.TaskRun_PENDING},+ })+ require.NoError(t, err)++ // Seed a legacy state that predates the archive precondition. New archive+ // requests reject AVAILABLE task runs, but claiming must remain defensive+ // for existing inconsistent rows.+ _, err = s.GetDB().ExecContext(ctx, `+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1+ `, instanceID)+ require.NoError(t, err)++ b, err := bus.New()+ require.NoError(t, err)+ scheduler := &Scheduler{store: s, bus: b, profile: &config.Profile{ReplicaID: "replica-a"}}+ require.NoError(t, scheduler.scheduleRunningTaskRuns(ctx))++ taskRuns, err = s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})require.NoError(t, err)+ require.Len(t, taskRuns, 1)+ require.Equal(t, storepb.TaskRun_AVAILABLE, taskRuns[0].Status, "an archived instance's available task run must not be claimed")}diff --git a/backend/runner/taskrun/scheduler_test.go b/backend/runner/taskrun/scheduler_test.goindex 5d0f14decf..0ef1592dab 100644--- a/backend/runner/taskrun/scheduler_test.go+++ b/backend/runner/taskrun/scheduler_test.go@@ -91,3 +91,57 @@ func TestSchedulePendingTaskRunsSkipsArchivedProject(t *testing.T) {require.Len(t, taskRuns, 1)require.Equal(t, storepb.TaskRun_AVAILABLE, taskRuns[0].Status)}++func TestSchedulePendingTaskRunsSkipsArchivedInstance(t *testing.T) {+ ctx := context.Background()+ s := setupRolloutCreatorStore(ctx, t)+ plan, err := s.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: "project-a",+ Name: "archived instance plan",+ Config: &storepb.PlanConfig{},+ }, "[email protected]")+ require.NoError(t, err)+ instanceID := "instance-a"+ _, err = s.CreateInstance(ctx, &store.InstanceMessage{+ ResourceID: instanceID,+ Workspace: "default",+ Metadata: &storepb.Instance{+ Engine: storepb.Engine_POSTGRES,+ DataSources: []*storepb.DataSource{{Id: "admin", Type: storepb.DataSourceType_ADMIN}},+ },+ })+ require.NoError(t, err)++ tx, err := s.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ tasks, err := s.CreateMissingTasksTx(ctx, tx, plan.ProjectID, plan.UID, []*store.TaskMessage{{+ InstanceID: instanceID,+ Type: storepb.Task_TASK_TYPE_UNSPECIFIED,+ Payload: &storepb.Task{},+ }})+ require.NoError(t, err)+ require.Len(t, tasks, 1)+ require.NoError(t, tx.Commit())+ require.NoError(t, s.CreatePendingTaskRuns(ctx, "", &store.TaskRunMessage{+ ProjectID: plan.ProjectID,+ TaskUID: tasks[0].ID,+ }))++ // Seed a legacy state that predates the archive precondition. New archive+ // requests reject PENDING task runs, but the scheduler must still avoid+ // promoting existing inconsistent rows.+ _, err = s.GetDB().ExecContext(ctx, `+ UPDATE instance SET deleted = TRUE WHERE resource_id = $1+ `, instanceID)+ require.NoError(t, err)++ b, err := bus.New()+ require.NoError(t, err)+ scheduler := &Scheduler{store: s, bus: b}+ require.NoError(t, scheduler.schedulePendingTaskRuns(ctx))++ taskRuns, err := s.ListTaskRuns(ctx, &store.FindTaskRunMessage{ProjectID: plan.ProjectID})+ require.NoError(t, err)+ require.Len(t, taskRuns, 1)+ require.Equal(t, storepb.TaskRun_PENDING, taskRuns[0].Status, "an archived instance's pending task run must not be promoted")+}diff --git a/backend/store/README.md b/backend/store/README.mdindex a4b3129793..306660a169 100644--- a/backend/store/README.md+++ b/backend/store/README.md@@ -26,12 +26,14 @@ that branch. The active-project check in `nextProjectID` covers this case only fwriters that call it; it is not a repository-wide purge fence because otherwriters bypass `nextProjectID`.-Database creation, database-sync, and Query History writers, together with direct+Database creation, database-sync, batch database updates, task-run creation,+and Query History writers, together with direct instance archive and directproject/instance purge, additionally take the matching transaction-scoped purge-fence before any row lock. This closes the absent database-descendant gap;-writers then retain the normal child-to-parent row-lock order. Database sync may-continue for an archived project while its row exists, but never through a-soft-deleted instance.+fence before any row lock. This closes absent-descendant gaps; writers then+retain the normal child-to-parent row-lock order. Database sync may continue for+an archived project while its row exists, but never through a soft-deleted+instance. Direct instance archive and restore fail while any targeting task run+is pending, available, or running.Every new or modified writer of purge-managed data must define its projectlifecycle policy: require an active project for new resources, or require only andiff --git a/backend/store/advisory_lock.go b/backend/store/advisory_lock.goindex 56b269e8c7..929d08fedd 100644--- a/backend/store/advisory_lock.go+++ b/backend/store/advisory_lock.go@@ -29,8 +29,8 @@ const (// AdvisoryLockKeyProjectPurge serializes project-owned writes with project// purge, including writes to an otherwise absent database descendant.AdvisoryLockKeyProjectPurge AdvisoryLockKey = 1006- // AdvisoryLockKeyInstancePurge serializes database writes with instance- // purge, which cannot lock an absent database descendant.+ // AdvisoryLockKeyInstancePurge serializes instance lifecycle operations with+ // descendant creation and purge when the descendant row may not exist yet.AdvisoryLockKeyInstancePurge AdvisoryLockKey = 1007)diff --git a/backend/store/batch_update_database_purge_test.go b/backend/store/batch_update_database_purge_test.gonew file mode 100644index 0000000000..2b35f4fd4c--- /dev/null+++ b/backend/store/batch_update_database_purge_test.go@@ -0,0 +1,243 @@+package store_test++import (+ "database/sql"+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/store"+)++func batchMoveDatabaseIntoProjectA(fixture *projectDeletionLockOrderFixture) error {+ projectID := "project-a"+ return fixture.store.BatchUpdateDatabases(fixture.ctx, []*store.DatabaseMessage{{+ InstanceID: "instance-a",+ DatabaseName: "db-a",+ }}, &store.BatchUpdateDatabases{+ Workspace: "default",+ ProjectID: &projectID,+ })+}++func requireProjectPurgeTerminalState(t *testing.T, fixture *projectDeletionLockOrderFixture) {+ t.Helper()+ var projectCount, orphanCount, databaseCount int+ var project string+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT COUNT(*) FROM project WHERE resource_id = 'project-a'").Scan(&projectCount))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT COUNT(*) FROM db WHERE project = 'project-a'").Scan(&orphanCount))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT COUNT(*) FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&databaseCount))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&project))+ require.Zero(t, projectCount, "project purge must remove the archived project")+ require.Zero(t, orphanCount, "no database row may reference the purged project")+ require.Equal(t, 1, databaseCount, "the workspace-instance database must survive")+ require.Equal(t, "default", project, "the workspace-instance database must be reassigned to the default project")+}++func TestBatchUpdateDatabasesAndDeleteProjectSerialize(t *testing.T) {+ // The batch moves a workspace-instance database into the archived project+ // while that project is being purged. Without purge fencing the terminal+ // project deletion fails on db.project's foreign key.+ const seedSQL = `+ INSERT INTO instance (resource_id, workspace, project)+ VALUES ('instance-a', 'default', NULL);+ INSERT INTO db (instance, name, project) VALUES ('instance-a', 'db-a', 'default');+ `+ purgeProjectA := func(fixture *projectDeletionLockOrderFixture) error {+ return fixture.store.DeleteProject(fixture.ctx, "default", "project-a")+ }++ t.Run("purge first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)+ const barrierID = 9941+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER DELETE ON project FOR EACH ROW")++ purgeResult := make(chan error, 1)+ go func() { purgeResult <- purgeProjectA(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ batchResult := make(chan error, 1)+ go func() { batchResult <- batchMoveDatabaseIntoProjectA(fixture) }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)++ requireMaintenanceResult(t, purgeResult)+ err := <-batchResult+ require.Error(t, err)+ require.NotContains(t, strings.ToLower(err.Error()), "foreign key",+ "the batch must reject the purged destination cleanly, not with an FK failure")+ require.ErrorContains(t, err, "project-a not found",+ "the batch must fail on the purged destination project")+ requireProjectPurgeTerminalState(t, fixture)+ })++ t.Run("batch first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)+ const barrierID = 9942+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ _, err := fixture.db.ExecContext(fixture.ctx, `+ CREATE FUNCTION block_db_project_update_9942() RETURNS trigger AS $$+ BEGIN+ PERFORM pg_advisory_xact_lock(9942);+ RETURN NEW;+ END;+ $$ LANGUAGE plpgsql;+ CREATE TRIGGER block_db_project_update_9942+ BEFORE UPDATE OF project ON db FOR EACH ROW+ WHEN (OLD.project IS DISTINCT FROM NEW.project)+ EXECUTE FUNCTION block_db_project_update_9942();+ `)+ require.NoError(t, err)++ batchResult := make(chan error, 1)+ go func() { batchResult <- batchMoveDatabaseIntoProjectA(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ purgeResult := make(chan error, 1)+ go func() { purgeResult <- purgeProjectA(fixture) }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)++ requireMaintenanceResult(t, batchResult)+ requireMaintenanceResult(t, purgeResult)+ requireProjectPurgeTerminalState(t, fixture)+ })+}++func TestBatchUpdateDatabasesRejectsArchivedInstance(t *testing.T) {+ const seedSQL = `+ INSERT INTO instance (resource_id, workspace, project, deleted)+ VALUES ('instance-a', 'default', NULL, TRUE);+ INSERT INTO db (instance, name, project) VALUES ('instance-a', 'db-a', 'default');+ `+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)+ err := batchMoveDatabaseIntoProjectA(fixture)+ require.Error(t, err)+ require.ErrorContains(t, err, "instance instance-a is archived")++ var projectID string+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&projectID))+ require.Equal(t, "default", projectID)+}++func TestBatchUpdateDatabasesClearsEnvironmentOnArchivedInstance(t *testing.T) {+ environmentID, unset := "env-a", ""+ fixture := newProjectDeletionLockOrderFixture(t, `+ INSERT INTO instance (resource_id, workspace, project, deleted)+ VALUES ('instance-a', 'default', NULL, TRUE);+ INSERT INTO db (instance, name, project, environment)+ VALUES ('instance-a', 'db-a', 'default', 'env-a');+ `)++ err := fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{+ Workspace: "default",+ FindByEnvironmentID: &environmentID,+ EnvironmentID: &unset,+ })+ require.NoError(t, err, "environment cleanup must remain possible after an instance is archived")++ var environment sql.NullString+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT environment FROM db WHERE instance = 'instance-a' AND name = 'db-a'").Scan(&environment))+ require.False(t, environment.Valid, "the archived instance must not retain a deleted environment reference")+}++func TestBatchUpdateDatabasesByEnvironment(t *testing.T) {+ envA, unset := "env-a", ""+ fixture := newProjectDeletionLockOrderFixture(t, `+ INSERT INTO workspace (resource_id) VALUES ('other');+ INSERT INTO instance (resource_id, workspace, project) VALUES+ ('instance-a', 'default', NULL),+ ('instance-b', 'other', NULL);+ INSERT INTO db (instance, name, project, environment) VALUES+ ('instance-a', 'db-a', 'default', 'env-a'),+ ('instance-a', 'db-b', 'default', 'env-b'),+ ('instance-b', 'db-c', 'default', 'env-a');+ `)++ err := fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{+ Workspace: "default",+ FindByEnvironmentID: &envA,+ EnvironmentID: &unset,+ })+ require.NoError(t, err)++ requireDatabaseEnvironment := func(instanceID, databaseName string, want string) {+ t.Helper()+ var environment sql.NullString+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT environment FROM db WHERE instance = $1 AND name = $2", instanceID, databaseName).Scan(&environment))+ if want == "" {+ require.False(t, environment.Valid, "database %s/%s environment should be unset", instanceID, databaseName)+ } else {+ require.True(t, environment.Valid)+ require.Equal(t, want, environment.String)+ }+ }+ requireDatabaseEnvironment("instance-a", "db-a", "")+ requireDatabaseEnvironment("instance-a", "db-b", "env-b")+ requireDatabaseEnvironment("instance-b", "db-c", "env-a")+}++func TestBatchUpdateDatabasesByEnvironmentRevalidatesDynamicTargets(t *testing.T) {+ environmentID, unset := "env-a", ""+ fixture := newProjectDeletionLockOrderFixture(t, `+ INSERT INTO instance (resource_id, workspace, project)+ VALUES ('instance-a', 'default', NULL);+ INSERT INTO db (instance, name, project, environment) VALUES+ ('instance-a', 'db-a', 'default', 'env-a'),+ ('instance-a', 'db-b', 'default', 'env-b');+ `)++ // Block the batch on its default-project purge fence. It reaches that fence+ // only after the outside-transaction target read has completed.+ conn, err := fixture.db.Conn(fixture.ctx)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, conn.Close()) })+ _, err = conn.ExecContext(fixture.ctx, "SELECT pg_advisory_lock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")+ require.NoError(t, err)+ t.Cleanup(func() {+ _, err := conn.ExecContext(fixture.ctx, "SELECT pg_advisory_unlock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")+ require.NoError(t, err)+ })+ var barrierPID int+ require.NoError(t, conn.QueryRowContext(fixture.ctx, "SELECT pg_backend_pid()").Scan(&barrierPID))++ result := make(chan error, 1)+ go func() {+ result <- fixture.store.BatchUpdateDatabases(fixture.ctx, nil, &store.BatchUpdateDatabases{+ Workspace: "default",+ FindByEnvironmentID: &environmentID,+ EnvironmentID: &unset,+ })+ }()+ waitForBackendBlockedByPID(fixture.ctx, t, fixture.db, barrierPID)++ // db-b was not in the pre-read target set, but becomes a match before the+ // fenced transaction scans. The batch must either include it or explicitly+ // request a retry; it must not report success and silently leave it behind.+ _, err = fixture.db.ExecContext(fixture.ctx,+ "UPDATE db SET environment = 'env-a' WHERE instance = 'instance-a' AND name = 'db-b'")+ require.NoError(t, err)+ _, err = conn.ExecContext(fixture.ctx, "SELECT pg_advisory_unlock($1, hashtext($2))", int64(store.AdvisoryLockKeyProjectPurge), "default")+ require.NoError(t, err)++ err = <-result+ if err != nil {+ require.NotContains(t, strings.ToLower(err.Error()), "foreign key")+ require.Contains(t, strings.ToLower(err.Error()), "retry")+ return+ }++ var environment sql.NullString+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT environment FROM db WHERE instance = 'instance-a' AND name = 'db-b'").Scan(&environment))+ require.False(t, environment.Valid, "a matching database added during the fence wait must not be skipped")+}diff --git a/backend/store/database.go b/backend/store/database.goindex f3c4ed1417..d0e23e93f7 100644--- a/backend/store/database.go+++ b/backend/store/database.go@@ -388,6 +388,12 @@ func (s *Store) UpdateDatabase(ctx context.Context, patch *UpdateDatabaseMessage}// BatchUpdateDatabases updates databases in batch.+//+// The batch runs in one transaction after acquiring every affected source and+// destination project purge fence and instance purge fence in deterministic+// sorted order. Without the fences, a concurrent project purge could sweep the+// destination project's rows and make the terminal project deletion fail on+// the db.project foreign key.func (s *Store) BatchUpdateDatabases(ctx context.Context, databases []*DatabaseMessage, update *BatchUpdateDatabases) error {set := qb.Q()if update.ProjectID != nil {@@ -424,74 +430,317 @@ func (s *Store) BatchUpdateDatabases(ctx context.Context, databases []*DatabaseMwhere.And("db.instance IN (SELECT resource_id FROM instance WHERE workspace = ?)", update.Workspace)}- validOwnership := qb.Q().Space("TRUE")+ // Discover every affected database before acquiring purge fences so the+ // source/destination project and instance fence sets are known up front.+ // The transaction below re-reads these targets under the fences.+ targets, err := s.listDatabaseBatchTargets(ctx, where)+ if err != nil {+ return err+ }+ fenceInstances := make([]string, 0, len(targets))+ fenceProjects := make([]string, 0, len(targets))+ for _, target := range targets {+ fenceInstances = append(fenceInstances, target.instanceID)+ fenceProjects = append(fenceProjects, target.projectID)+ if target.instanceProject != "" {+ fenceProjects = append(fenceProjects, target.instanceProject)+ }+ }if update.ProjectID != nil {- validOwnership = qb.Q().Space(`- NOT EXISTS (- SELECT 1 FROM targets- WHERE instance_project IS NOT NULL- AND (project <> instance_project OR instance_project <> ?)- )- `, *update.ProjectID)+ fenceProjects = append(fenceProjects, *update.ProjectID)}+ slices.Sort(fenceInstances)+ fenceInstances = slices.Compact(fenceInstances)+ slices.Sort(fenceProjects)+ fenceProjects = slices.Compact(fenceProjects)+ var updated []databaseBatchTarget+ err = s.withDatabaseBatchPurgeFence(ctx, fenceInstances, fenceProjects, func(tx *sql.Tx) error {+ // Dynamic environment matching can gain rows between the pre-read and+ // the purge fences. Lock the complete current match set so a newly+ // matching row is either included or causes a retry below; never leave+ // it silently unchanged.+ locked, err := lockDatabaseBatchTargets(ctx, tx, where, targets, update.FindByEnvironmentID != nil)+ if err != nil {+ return err+ }+ for _, target := range locked {+ if !slices.Contains(fenceInstances, target.instanceID) || !slices.Contains(fenceProjects, target.projectID) || (target.instanceProject != "" && !slices.Contains(fenceProjects, target.instanceProject)) {+ return common.Errorf(common.Conflict, "batch database targets changed; retry")+ }+ }+ instances, err := lockDatabaseBatchInstances(ctx, tx, locked)+ if err != nil {+ return err+ }+ if update.ProjectID != nil {+ for instanceID, instance := range instances {+ if instance.deleted {+ return common.Errorf(common.Conflict, "instance %s is archived", instanceID)+ }+ }+ }+ // Revalidate lifecycle under the transaction: every target must still+ // be covered by the pre-read purge fences. A purge that committed+ // before the fences were acquired shows up here as a retry instead of+ // an FK failure.+ for _, target := range locked {+ if !slices.Contains(fenceProjects, target.projectID) {+ return common.Errorf(common.Conflict, "database ownership changed to project %s for %s; retry", target.projectID, common.FormatDatabase(target.instanceID, target.databaseName))+ }+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" && !slices.Contains(fenceProjects, instanceProject) {+ return common.Errorf(common.Conflict, "database ownership changed to project instance %s for %s; retry", instanceProject, common.FormatDatabase(target.instanceID, target.databaseName))+ }+ }+ // Lock every affected project, including the destination, after the+ // database rows and instances. A purged destination fails cleanly with+ // NotFound instead of an FK violation.+ projectIDs := make([]string, 0, len(locked)+1)+ for _, target := range locked {+ projectIDs = append(projectIDs, target.projectID)+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" {+ projectIDs = append(projectIDs, instanceProject)+ }+ }+ if update.ProjectID != nil {+ projectIDs = append(projectIDs, *update.ProjectID)+ }+ slices.Sort(projectIDs)+ projectIDs = slices.Compact(projectIDs)+ for _, projectID := range projectIDs {+ var foundProjectID string+ if err := tx.QueryRowContext(ctx, "SELECT resource_id FROM project WHERE resource_id = $1 FOR UPDATE", projectID).Scan(&foundProjectID); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return common.Errorf(common.NotFound, "project %s not found", projectID)+ }+ return errors.Wrapf(err, "failed to lock project %s", projectID)+ }+ }+ // Atomic validation: any invalid target rejects the whole batch before+ // any row is updated.+ if update.ProjectID != nil {+ for _, target := range locked {+ if instanceProject := instances[target.instanceID].projectID; instanceProject != "" {+ if target.projectID != instanceProject || instanceProject != *update.ProjectID {+ return common.Errorf(common.Invalid, "cannot move a project instance database to another project")+ }+ }+ }+ }+ if len(locked) == 0 {+ return nil+ }+ updateInstances, updateNames := make([]string, 0, len(locked)), make([]string, 0, len(locked))+ for _, target := range locked {+ updateInstances = append(updateInstances, target.instanceID)+ updateNames = append(updateNames, target.databaseName)+ }+ q := qb.Q().Space(`+ UPDATE db SET ?+ FROM unnest(?::TEXT[], ?::TEXT[]) AS u(instance, name)+ WHERE db.instance = u.instance AND db.name = u.name+ `, set, updateInstances, updateNames)+ query, args, err := q.ToSQL()+ if err != nil {+ return errors.Wrap(err, "failed to build batch database update sql")+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return errors.Wrap(err, "failed to update databases")+ }+ updated = locked+ return nil+ })+ if err != nil {+ return err+ }+ for _, target := range updated {+ s.removeDatabaseCache(ctx, target.instanceID, target.databaseName)+ }+ return nil+}++type databaseBatchTarget struct {+ instanceID string+ databaseName string+ projectID string+ instanceProject string+}++// listDatabaseBatchTargets finds the databases matched by the batch update+// where clause. It runs outside the purge fences; the result only determines+// which fences to acquire and which rows to lock inside the transaction.+func (s *Store) listDatabaseBatchTargets(ctx context.Context, where *qb.Query) ([]databaseBatchTarget, error) {q := qb.Q().Space(`- WITH targets AS MATERIALIZED (- SELECT db.instance, db.name, db.project, instance.project AS instance_project- FROM db- JOIN instance ON instance.resource_id = db.instance- WHERE ?- ),- validation AS (- SELECT ? AS valid- ),- updated AS (- UPDATE db- SET ?- FROM targets, validation- WHERE validation.valid- AND db.instance = targets.instance- AND db.name = targets.name- RETURNING db.instance, db.name- )- SELECT validation.valid, updated.instance, updated.name- FROM validation- LEFT JOIN updated ON TRUE- `, where, validOwnership, set)+ SELECT db.instance, db.name, db.project, instance.project+ FROM db+ JOIN instance ON instance.resource_id = db.instance+ WHERE ?+ ORDER BY db.instance, db.name+ `, where)query, args, err := q.ToSQL()if err != nil {- return errors.Wrap(err, "failed to build sql")+ return nil, errors.Wrap(err, "failed to build batch database target query")}rows, err := s.GetDB().QueryContext(ctx, query, args...)if err != nil {- return errors.Wrap(err, "failed to update databases")+ return nil, errors.Wrap(err, "failed to find batch database targets")+ }+ defer rows.Close()+ var targets []databaseBatchTarget+ for rows.Next() {+ var target databaseBatchTarget+ var instanceProject sql.NullString+ if err := rows.Scan(&target.instanceID, &target.databaseName, &target.projectID, &instanceProject); err != nil {+ return nil, errors.Wrap(err, "failed to scan batch database target")+ }+ if instanceProject.Valid {+ target.instanceProject = instanceProject.String+ }+ targets = append(targets, target)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read batch database targets")+ }+ return targets, nil+}++// withDatabaseBatchPurgeFence serializes a batch database write with direct+// project and instance purge. Advisory fences are acquired in deterministic+// sorted order (all project fences, then all instance fences) before any row+// lock.+func (s *Store) withDatabaseBatchPurgeFence(+ ctx context.Context,+ instances []string,+ projects []string,+ write func(*sql.Tx) error,+) error {+ tx, err := s.GetDB().BeginTx(ctx, nil)+ if err != nil {+ return errors.Wrap(err, "failed to begin batch database write transaction")+ }+ defer tx.Rollback()+ for _, projectID := range projects {+ if err := acquireProjectPurgeLock(ctx, tx, projectID); err != nil {+ return errors.Wrapf(err, "failed to lock project purge fence for %s", projectID)+ }+ }+ for _, instanceID := range instances {+ if err := acquireInstancePurgeLock(ctx, tx, instanceID); err != nil {+ return errors.Wrapf(err, "failed to lock instance purge fence for %s", instanceID)+ }+ }+ if err := write(tx); err != nil {+ return err+ }+ return errors.Wrap(tx.Commit(), "failed to commit batch database write transaction")+}++// lockDatabaseBatchTargets locks the batch targets in full primary-key order.+// Dynamic environment queries re-scan the original where clause so rows that+// start matching after the pre-read are detected. Static targets re-apply the+// clause to the pre-read identities, so rows that no longer match are skipped.+// Only db rows are locked.+func lockDatabaseBatchTargets(ctx context.Context, tx *sql.Tx, where *qb.Query, targets []databaseBatchTarget, dynamic bool) ([]databaseBatchTarget, error) {+ if len(targets) == 0 && !dynamic {+ return nil, nil+ }+ q := qb.Q().Space(`+ SELECT db.instance, db.name, db.project, instance.project+ FROM db+ JOIN instance ON instance.resource_id = db.instance+ WHERE ?+ `, where)+ if !dynamic {+ instances, names := make([]string, 0, len(targets)), make([]string, 0, len(targets))+ for _, target := range targets {+ instances, names = append(instances, target.instanceID), append(names, target.databaseName)+ }+ q.And(`(db.instance, db.name) IN (SELECT * FROM unnest(?::TEXT[], ?::TEXT[]))`, instances, names)+ }+ q.Space("ORDER BY db.instance, db.name FOR UPDATE OF db")+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrap(err, "failed to build batch database lock query")+ }+ rows, err := tx.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrap(err, "failed to lock batch database targets")}defer rows.Close()+ var locked []databaseBatchTarget+ for rows.Next() {+ var target databaseBatchTarget+ var instanceProject sql.NullString+ if err := rows.Scan(&target.instanceID, &target.databaseName, &target.projectID, &instanceProject); err != nil {+ return nil, errors.Wrap(err, "failed to scan locked batch database target")+ }+ if instanceProject.Valid {+ target.instanceProject = instanceProject.String+ }+ locked = append(locked, target)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read locked batch database targets")+ }+ return locked, nil+}- var updated []*DatabaseMessage+type databaseBatchInstance struct {+ projectID string+ deleted bool+}++// lockDatabaseBatchInstances locks the instances of the locked targets in+// sorted order and returns their current lifecycle and owning project.+func lockDatabaseBatchInstances(ctx context.Context, tx *sql.Tx, targets []databaseBatchTarget) (map[string]databaseBatchInstance, error) {+ instanceIDs := make([]string, 0, len(targets))+ for _, target := range targets {+ instanceIDs = append(instanceIDs, target.instanceID)+ }+ slices.Sort(instanceIDs)+ instanceIDs = slices.Compact(instanceIDs)+ instances := make(map[string]databaseBatchInstance, len(instanceIDs))+ if len(instanceIDs) == 0 {+ return instances, nil+ }+ q := qb.Q().Space(`+ SELECT resource_id, project, deleted+ FROM instance+ WHERE resource_id IN (SELECT * FROM unnest(?::TEXT[]))+ ORDER BY resource_id+ FOR NO KEY UPDATE+ `, instanceIDs)+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrap(err, "failed to build batch database instance lock query")+ }+ rows, err := tx.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrap(err, "failed to lock batch database instances")+ }+ defer rows.Close()for rows.Next() {- var valid bool- var instanceID, databaseName sql.NullString- if err := rows.Scan(&valid, &instanceID, &databaseName); err != nil {- return errors.Wrap(err, "failed to scan updated database")- }- if !valid {- return common.Errorf(common.Invalid, "cannot move a project instance database to another project")- }- if instanceID.Valid && databaseName.Valid {- updated = append(updated, &DatabaseMessage{- InstanceID: instanceID.String,- DatabaseName: databaseName.String,- })+ var instanceID string+ var project sql.NullString+ var deleted bool+ if err := rows.Scan(&instanceID, &project, &deleted); err != nil {+ return nil, errors.Wrap(err, "failed to scan locked batch database instance")+ }+ instance := databaseBatchInstance{deleted: deleted}+ if project.Valid {+ instance.projectID = project.String}+ instances[instanceID] = instance}if err := rows.Err(); err != nil {- return errors.Wrap(err, "failed to update databases")+ return nil, errors.Wrap(err, "failed to read locked batch database instances")}- for _, database := range updated {- s.removeDatabaseCache(ctx, database.InstanceID, database.DatabaseName)+ for _, instanceID := range instanceIDs {+ if _, ok := instances[instanceID]; !ok {+ return nil, common.Errorf(common.NotFound, "instance %s not found", instanceID)+ }}- return nil+ return instances, nil}type databaseOwnership struct {diff --git a/backend/store/instance.go b/backend/store/instance.goindex 2037deaf21..844002a4d8 100644--- a/backend/store/instance.go+++ b/backend/store/instance.go@@ -4,6 +4,7 @@ import ("context""database/sql""fmt"+ "slices""strings""github.com/google/cel-go/cel"@@ -40,6 +41,10 @@ type UpdateInstanceMessage struct {Deleted *boolEnvironmentID *stringMetadata *storepb.Instance+ // MoveDatabasesToProjectID atomically transfers all databases while changing+ // the instance lifecycle. It is only valid when archiving a resource-scoped+ // workspace instance.+ MoveDatabasesToProjectID *string}// FindInstanceMessage is the message for finding instances.@@ -280,6 +285,9 @@ func (s *Store) CreateInstance(ctx context.Context, instanceCreate *InstanceMess// UpdateInstance updates an instance.func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessage) (*InstanceMessage, error) {+ if patch.MoveDatabasesToProjectID != nil && (patch.ResourceID == nil || patch.Deleted == nil || !*patch.Deleted) {+ return nil, errors.New("moving instance databases is only supported while archiving a resource-scoped instance")+ }set := qb.Q()if v := patch.EnvironmentID; v != nil {@@ -328,6 +336,9 @@ func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessageSpace("WHERE ?", where)if v := patch.ResourceID; v != nil {+ if patch.Deleted != nil {+ return s.updateInstanceLifecycle(ctx, patch, q)+ }q.Space("RETURNING workspace, project")query, args, err := q.ToSQL()if err != nil {@@ -359,6 +370,220 @@ func (s *Store) UpdateInstance(ctx context.Context, patch *UpdateInstanceMessagereturn nil, nil}+func (s *Store) updateInstanceLifecycle(ctx context.Context, patch *UpdateInstanceMessage, q *qb.Query) (*InstanceMessage, error) {+ resourceID := *patch.ResourceID+ var projectFences []string+ if destinationProjectID := patch.MoveDatabasesToProjectID; destinationProjectID != nil {+ var err error+ projectFences, err = s.listInstanceDatabaseProjects(ctx, resourceID)+ if err != nil {+ return nil, err+ }+ projectFences = append(projectFences, *destinationProjectID)+ slices.Sort(projectFences)+ projectFences = slices.Compact(projectFences)+ }++ tx, err := s.GetDB().BeginTx(ctx, nil)+ if err != nil {+ return nil, errors.Wrap(err, "failed to begin instance lifecycle transaction")+ }+ defer tx.Rollback()+ for _, projectID := range projectFences {+ if err := acquireProjectPurgeLock(ctx, tx, projectID); err != nil {+ return nil, errors.Wrapf(err, "failed to lock project purge fence for %s", projectID)+ }+ }+ if err := acquireInstancePurgeLock(ctx, tx, resourceID); err != nil {+ return nil, errors.Wrapf(err, "failed to lock instance lifecycle fence for %s", resourceID)+ }++ activeTaskRunCount, err := lockActiveTaskRunsForInstance(ctx, tx, resourceID)+ if err != nil {+ return nil, err+ }+ if activeTaskRunCount > 0 {+ operation := "archiving"+ if !*patch.Deleted {+ operation = "restoring"+ }+ return nil, common.Errorf(common.Conflict, "instance %s has %d active task run(s); cancel them or wait for them to finish before %s", resourceID, activeTaskRunCount, operation)+ }++ var movedDatabases []instanceDatabaseTarget+ if destinationProjectID := patch.MoveDatabasesToProjectID; destinationProjectID != nil {+ movedDatabases, err = lockInstanceDatabases(ctx, tx, resourceID)+ if err != nil {+ return nil, err+ }+ for _, database := range movedDatabases {+ if !slices.Contains(projectFences, database.projectID) {+ return nil, common.Errorf(common.Conflict, "database ownership changed to project %s for %s; retry", database.projectID, common.FormatDatabase(resourceID, database.name))+ }+ }++ var instanceProject sql.NullString+ if err := tx.QueryRowContext(ctx, `+ SELECT project FROM instance WHERE resource_id = $1 FOR NO KEY UPDATE+ `, resourceID).Scan(&instanceProject); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return nil, common.Errorf(common.NotFound, "instance %s not found", resourceID)+ }+ return nil, errors.Wrapf(err, "failed to lock instance %s", resourceID)+ }+ if instanceProject.Valid {+ return nil, common.Errorf(common.Invalid, "cannot transfer databases while archiving project instance %s", resourceID)+ }++ projectIDs := append([]string(nil), *destinationProjectID)+ for _, database := range movedDatabases {+ projectIDs = append(projectIDs, database.projectID)+ }+ slices.Sort(projectIDs)+ projectIDs = slices.Compact(projectIDs)+ for _, projectID := range projectIDs {+ var foundProjectID string+ if err := tx.QueryRowContext(ctx, "SELECT resource_id FROM project WHERE resource_id = $1 FOR UPDATE", projectID).Scan(&foundProjectID); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return nil, common.Errorf(common.NotFound, "project %s not found", projectID)+ }+ return nil, errors.Wrapf(err, "failed to lock project %s", projectID)+ }+ }+ if _, err := tx.ExecContext(ctx, "UPDATE db SET project = $1 WHERE instance = $2", *destinationProjectID, resourceID); err != nil {+ return nil, errors.Wrapf(err, "failed to transfer databases for instance %s", resourceID)+ }+ }++ q.Space("RETURNING workspace, project")+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrap(err, "failed to build instance lifecycle query")+ }+ var workspace string+ var project sql.NullString+ if err := tx.QueryRowContext(ctx, query, args...).Scan(&workspace, &project); err != nil {+ return nil, err+ }+ if err := tx.Commit(); err != nil {+ return nil, errors.Wrap(err, "failed to commit instance lifecycle transaction")+ }++ s.instanceCache.Remove(getInstanceCacheKey(resourceID))+ for _, database := range movedDatabases {+ s.removeDatabaseCache(ctx, resourceID, database.name)+ }+ find := &FindInstanceMessage{Workspace: workspace, ResourceID: patch.ResourceID}+ if project.Valid {+ find.ProjectID = &project.String+ } else {+ find.WorkspaceOnly = true+ }+ return s.GetInstance(ctx, find)+}++func (s *Store) listInstanceDatabaseProjects(ctx context.Context, resourceID string) ([]string, error) {+ rows, err := s.GetDB().QueryContext(ctx, `+ SELECT DISTINCT project FROM db WHERE instance = $1 ORDER BY project+ `, resourceID)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to find database projects for instance %s", resourceID)+ }+ defer rows.Close()+ var projects []string+ for rows.Next() {+ var projectID string+ if err := rows.Scan(&projectID); err != nil {+ return nil, errors.Wrap(err, "failed to scan instance database project")+ }+ projects = append(projects, projectID)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read instance database projects")+ }+ return projects, nil+}++type instanceDatabaseTarget struct {+ name string+ projectID string+}++func lockInstanceDatabases(ctx context.Context, tx *sql.Tx, resourceID string) ([]instanceDatabaseTarget, error) {+ rows, err := tx.QueryContext(ctx, `+ SELECT name, project+ FROM db+ WHERE instance = $1+ ORDER BY instance, name+ FOR UPDATE+ `, resourceID)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to lock databases for instance %s", resourceID)+ }+ defer rows.Close()+ var databases []instanceDatabaseTarget+ for rows.Next() {+ var database instanceDatabaseTarget+ if err := rows.Scan(&database.name, &database.projectID); err != nil {+ return nil, errors.Wrap(err, "failed to scan instance database")+ }+ databases = append(databases, database)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read instance databases")+ }+ return databases, nil+}++// GetActiveTaskRunCountForInstance returns the number of task runs that would+// block a direct instance archive or restore.+func (s *Store) GetActiveTaskRunCountForInstance(ctx context.Context, resourceID string) (int, error) {+ var count int+ if err := s.GetDB().QueryRowContext(ctx, `+ SELECT COUNT(*)+ FROM task_run+ JOIN task+ ON task.project = task_run.project+ AND task.id = task_run.task_id+ WHERE task.instance = $1+ AND task_run.status IN ('PENDING', 'AVAILABLE', 'RUNNING')+ `, resourceID).Scan(&count); err != nil {+ return 0, errors.Wrapf(err, "failed to count active task runs for instance %s", resourceID)+ }+ return count, nil+}++func lockActiveTaskRunsForInstance(ctx context.Context, tx *sql.Tx, resourceID string) (int, error) {+ rows, err := tx.QueryContext(ctx, `+ SELECT task_run.project, task_run.id+ FROM task_run+ JOIN task+ ON task.project = task_run.project+ AND task.id = task_run.task_id+ WHERE task.instance = $1+ AND task_run.status IN ('PENDING', 'AVAILABLE', 'RUNNING')+ ORDER BY task_run.project, task_run.id+ FOR UPDATE OF task_run+ `, resourceID)+ if err != nil {+ return 0, errors.Wrapf(err, "failed to lock active task runs for instance %s", resourceID)+ }+ defer rows.Close()+ activeTaskRunCount := 0+ for rows.Next() {+ var projectID string+ var taskRunID int64+ if err := rows.Scan(&projectID, &taskRunID); err != nil {+ return 0, errors.Wrap(err, "failed to scan active task run")+ }+ activeTaskRunCount+++ }+ if err := rows.Err(); err != nil {+ return 0, errors.Wrap(err, "failed to read active task runs")+ }+ return activeTaskRunCount, nil+}+// GetActivatedInstanceCount gets the number of activated instances.func (s *Store) GetActivatedInstanceCount(ctx context.Context, workspaceID string) (int, error) {q := qb.Q().Space("SELECT COUNT(1) FROM instance WHERE workspace = ?", workspaceID).And("(metadata ?? 'activation') AND (metadata->>'activation')::boolean = TRUE AND deleted = FALSE")diff --git a/backend/store/instance_archive_task_run_test.go b/backend/store/instance_archive_task_run_test.gonew file mode 100644index 0000000000..d2409e4212--- /dev/null+++ b/backend/store/instance_archive_task_run_test.go@@ -0,0 +1,236 @@+package store_test++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/store"+)++const instanceArchiveTaskRunSeed = `+ UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a';+ INSERT INTO instance (resource_id, workspace, project)+ VALUES ('instance-a', 'default', 'project-a');+ INSERT INTO plan (id, creator, project, name, description)+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_CREATE');+`++func archiveInstanceA(fixture *projectDeletionLockOrderFixture) error {+ deleted := true+ _, err := fixture.store.UpdateInstance(fixture.ctx, &store.UpdateInstanceMessage{+ ResourceID: new("instance-a"),+ Workspace: "default",+ Deleted: &deleted,+ })+ return err+}++func createInstanceATaskRun(fixture *projectDeletionLockOrderFixture) error {+ return fixture.store.CreatePendingTaskRuns(fixture.ctx, "[email protected]", &store.TaskRunMessage{+ ProjectID: "project-a",+ TaskUID: 101,+ })+}++func forceArchiveWorkspaceInstanceA(fixture *projectDeletionLockOrderFixture) error {+ deleted := true+ destinationProjectID := "default"+ _, err := fixture.store.UpdateInstance(fixture.ctx, &store.UpdateInstanceMessage{+ ResourceID: new("instance-a"),+ Workspace: "default",+ Deleted: &deleted,+ MoveDatabasesToProjectID: &destinationProjectID,+ })+ return err+}++func createInstanceADatabase(fixture *projectDeletionLockOrderFixture) error {+ _, err := fixture.store.CreateDatabaseDefault(fixture.ctx, &store.DatabaseMessage{+ InstanceID: "instance-a",+ DatabaseName: "db-a",+ ProjectID: "project-a",+ })+ return err+}++func TestArchiveInstanceAndCreateTaskRunSerialize(t *testing.T) {+ t.Run("archive first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, instanceArchiveTaskRunSeed)+ const barrierID = 9945+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE OF deleted ON instance FOR EACH ROW")++ archiveResult := make(chan error, 1)+ go func() { archiveResult <- archiveInstanceA(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ createResult := make(chan error, 1)+ go func() { createResult <- createInstanceATaskRun(fixture) }()+ barrierReleased := false+ defer func() {+ if !barrierReleased {+ barrier.release(t)+ <-archiveResult+ <-createResult+ }+ }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)+ barrierReleased = true++ requireMaintenanceResult(t, archiveResult)+ createErr := <-createResult+ require.Error(t, createErr)+ require.Equal(t, common.Conflict, common.ErrorCode(createErr))++ var deleted bool+ var activeTaskRunCount int+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",+ ).Scan(&deleted))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx, `+ SELECT COUNT(*) FROM task_run+ WHERE project = 'project-a' AND status IN ('PENDING', 'AVAILABLE', 'RUNNING')+ `).Scan(&activeTaskRunCount))+ require.True(t, deleted)+ require.Zero(t, activeTaskRunCount)+ })++ t.Run("creation first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, instanceArchiveTaskRunSeed)+ const barrierID = 9946+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER INSERT ON task_run FOR EACH ROW")++ createResult := make(chan error, 1)+ go func() { createResult <- createInstanceATaskRun(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ archiveResult := make(chan error, 1)+ go func() { archiveResult <- archiveInstanceA(fixture) }()+ barrierReleased := false+ defer func() {+ if !barrierReleased {+ barrier.release(t)+ <-createResult+ <-archiveResult+ }+ }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)+ barrierReleased = true++ requireMaintenanceResult(t, createResult)+ archiveErr := <-archiveResult+ require.Error(t, archiveErr)+ require.Equal(t, common.Conflict, common.ErrorCode(archiveErr))++ var deleted bool+ var activeTaskRunCount int+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",+ ).Scan(&deleted))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx, `+ SELECT COUNT(*) FROM task_run+ WHERE project = 'project-a' AND status IN ('PENDING', 'AVAILABLE', 'RUNNING')+ `).Scan(&activeTaskRunCount))+ require.False(t, deleted)+ require.Equal(t, 1, activeTaskRunCount)+ })+}++func TestForceArchiveInstanceAndCreateFirstDatabaseSerialize(t *testing.T) {+ const seedSQL = `+ UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a';+ INSERT INTO instance (resource_id, workspace, project)+ VALUES ('instance-a', 'default', NULL);+ `++ t.Run("archive first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)+ const barrierID = 9947+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER UPDATE OF deleted ON instance FOR EACH ROW")++ archiveResult := make(chan error, 1)+ go func() { archiveResult <- forceArchiveWorkspaceInstanceA(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ createResult := make(chan error, 1)+ go func() { createResult <- createInstanceADatabase(fixture) }()+ barrierReleased := false+ defer func() {+ if !barrierReleased {+ barrier.release(t)+ <-archiveResult+ <-createResult+ }+ }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)+ barrierReleased = true++ requireMaintenanceResult(t, archiveResult)+ createErr := <-createResult+ require.Error(t, createErr)+ require.NotContains(t, strings.ToLower(createErr.Error()), "foreign key")++ var deleted bool+ var databaseCount int+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",+ ).Scan(&deleted))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT COUNT(*) FROM db WHERE instance = 'instance-a' AND name = 'db-a'",+ ).Scan(&databaseCount))+ require.True(t, deleted)+ require.Zero(t, databaseCount)+ })++ t.Run("creation first", func(t *testing.T) {+ fixture := newProjectDeletionLockOrderFixture(t, seedSQL)+ const barrierID = 9948+ barrier := newMaintenanceLockBarrier(fixture.ctx, t, fixture.db, barrierID)+ installMaintenanceLockBarrier(t, fixture.db, barrierID, "AFTER INSERT ON db FOR EACH ROW")++ createResult := make(chan error, 1)+ go func() { createResult <- createInstanceADatabase(fixture) }()+ waitForMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)++ archiveResult := make(chan error, 1)+ go func() { archiveResult <- forceArchiveWorkspaceInstanceA(fixture) }()+ barrierReleased := false+ defer func() {+ if !barrierReleased {+ barrier.release(t)+ <-createResult+ <-archiveResult+ }+ }()+ waitForOperationBehindMaintenanceBarrier(fixture.ctx, t, fixture.db, barrierID)+ barrier.release(t)+ barrierReleased = true++ requireMaintenanceResult(t, createResult)+ archiveErr := <-archiveResult+ require.Error(t, archiveErr)+ require.Equal(t, common.Conflict, common.ErrorCode(archiveErr))+ require.NotContains(t, strings.ToLower(archiveErr.Error()), "foreign key")++ var deleted bool+ var projectID string+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT deleted FROM instance WHERE resource_id = 'instance-a'",+ ).Scan(&deleted))+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT project FROM db WHERE instance = 'instance-a' AND name = 'db-a'",+ ).Scan(&projectID))+ require.False(t, deleted)+ require.Equal(t, "project-a", projectID)+ })+}diff --git a/backend/store/project.go b/backend/store/project.goindex 4a3f8b99d4..2338191bf3 100644--- a/backend/store/project.go+++ b/backend/store/project.go@@ -331,6 +331,120 @@ func (s *Store) removeProjectCache(resourceID string) {s.projectCache.Remove(resourceID)}+// projectDescendantCacheKeys snapshots the cache keys of every instance,+// database, and schema row that a project purge is about to remove or+// reassign, so their cache entries can be invalidated after the purge commits.+type projectDescendantCacheKeys struct {+ instanceIDs []string+ databases []projectDatabaseCacheKey+ schemas []projectSchemaCacheKey+}++type projectDatabaseCacheKey struct {+ workspace string+ instanceID string+ databaseName string+}++type projectSchemaCacheKey struct {+ instanceID string+ databaseName string+}++// captureProjectDescendantCacheKeys reads the cache keys of every descendant+// row that DeleteProject will delete (project instances and their databases+// and schemas) or reassign (workspace-instance databases owned by the+// project). It must run inside the purge transaction before any of those rows+// are removed.+func captureProjectDescendantCacheKeys(ctx context.Context, tx *stdsql.Tx, projectID string) (*projectDescendantCacheKeys, error) {+ keys := &projectDescendantCacheKeys{}++ rows, err := tx.QueryContext(ctx, `+ SELECT resource_id+ FROM instance+ WHERE project = $1+ ORDER BY resource_id+ `, projectID)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to capture project instance cache keys for project %s", projectID)+ }+ defer rows.Close()+ for rows.Next() {+ var instanceID string+ if err := rows.Scan(&instanceID); err != nil {+ return nil, errors.Wrap(err, "failed to scan project instance cache key")+ }+ keys.instanceIDs = append(keys.instanceIDs, instanceID)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read project instance cache keys")+ }++ rows, err = tx.QueryContext(ctx, `+ SELECT db.instance, db.name, instance.workspace+ FROM db+ JOIN instance ON instance.resource_id = db.instance+ WHERE instance.project = $1 OR db.project = $1+ ORDER BY db.instance, db.name+ `, projectID)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to capture project database cache keys for project %s", projectID)+ }+ defer rows.Close()+ for rows.Next() {+ var key projectDatabaseCacheKey+ if err := rows.Scan(&key.instanceID, &key.databaseName, &key.workspace); err != nil {+ return nil, errors.Wrap(err, "failed to scan project database cache key")+ }+ keys.databases = append(keys.databases, key)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read project database cache keys")+ }++ rows, err = tx.QueryContext(ctx, `+ SELECT db_schema.instance, db_schema.db_name+ FROM db_schema+ JOIN instance ON instance.resource_id = db_schema.instance+ WHERE instance.project = $1+ ORDER BY db_schema.instance, db_schema.db_name+ `, projectID)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to capture project schema cache keys for project %s", projectID)+ }+ defer rows.Close()+ for rows.Next() {+ var key projectSchemaCacheKey+ if err := rows.Scan(&key.instanceID, &key.databaseName); err != nil {+ return nil, errors.Wrap(err, "failed to scan project schema cache key")+ }+ keys.schemas = append(keys.schemas, key)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read project schema cache keys")+ }++ return keys, nil+}++// removeProjectDescendantCaches invalidates the captured descendant cache+// entries and the project entry itself. It must only run after the purge+// transaction commits; a failed purge must not publish invalidation.+func (s *Store) removeProjectDescendantCaches(keys *projectDescendantCacheKeys, projectID string) {+ for _, instanceID := range keys.instanceIDs {+ s.instanceCache.Remove(getInstanceCacheKey(instanceID))+ }+ for _, key := range keys.databases {+ // Remove both the unscoped (runner) and workspace-scoped (API) entries.+ s.databaseCache.Remove(getDatabaseCacheKey("", key.instanceID, key.databaseName))+ s.databaseCache.Remove(getDatabaseCacheKey(key.workspace, key.instanceID, key.databaseName))+ }+ for _, key := range keys.schemas {+ s.dbSchemaCache.Remove(getDBSchemaCacheKey(key.instanceID, key.databaseName))+ }+ s.projectCache.Remove(projectID)+}+// DeleteProject permanently purges a soft-deleted project and all related resources.// This operation is irreversible and should only be used for:// - Administrative cleanup of old soft-deleted projects@@ -351,6 +465,14 @@ func (s *Store) DeleteProject(ctx context.Context, workspace string, resourceIDreturn errors.Wrapf(err, "failed to lock project purge fence for %s", resourceID)}+ // Capture descendant cache keys before any descendant rows are removed so+ // the post-commit invalidation is precise and does not require re-reading+ // rows that no longer exist.+ cacheKeys, err := captureProjectDescendantCacheKeys(ctx, tx, resourceID)+ if err != nil {+ return errors.Wrap(err, "failed to capture project descendant cache keys")+ }+// Delete query history before locking database-scoped rows to preserve the// canonical sibling-branch order.q := qb.Q().Space("DELETE FROM query_history WHERE project = ?", resourceID)@@ -723,8 +845,9 @@ func (s *Store) DeleteProject(ctx context.Context, workspace string, resourceIDreturn errors.Wrap(err, "failed to commit transaction")}- // Clear the project from cache- s.projectCache.Remove(resourceID)+ // Publish invalidation only after the purge commits: every captured+ // descendant entry is removed while unrelated cache entries survive.+ s.removeProjectDescendantCaches(cacheKeys, resourceID)return nil}diff --git a/backend/store/project_purge_cache_test.go b/backend/store/project_purge_cache_test.gonew file mode 100644index 0000000000..e73b7a72f3--- /dev/null+++ b/backend/store/project_purge_cache_test.go@@ -0,0 +1,239 @@+package store_test++import (+ "context"+ "database/sql"+ "fmt"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ "github.com/bytebase/bytebase/backend/migrator"+ "github.com/bytebase/bytebase/backend/store"+)++// newProjectPurgeCacheFixture boots a real PostgreSQL with caches ENABLED and+// seeds a workspace, the default project, a deleted purgeable project, and an+// active project for ID-reuse scenarios.+func newProjectPurgeCacheFixture(t *testing.T) (context.Context, *sql.DB, *store.Store) {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ t.Cleanup(cancel)+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(context.Background()) })+ db := container.GetDB()+ require.NoError(t, migrator.MigrateSchema(ctx, db))+ _, err := db.ExecContext(ctx, `+ INSERT INTO workspace (resource_id) VALUES ('default');+ INSERT INTO project (resource_id, workspace, name, deleted) VALUES+ ('default', 'default', 'Default', FALSE),+ ('project-a', 'default', 'Project A', TRUE),+ ('project-b', 'default', 'Project B', FALSE);+ `)+ require.NoError(t, err)+ pgURL := fmt.Sprintf(+ "host=%s port=%s user=postgres password=root-password database=postgres",+ container.GetHost(), container.GetPort(),+ )+ s, err := store.New(ctx, pgURL, true)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, s.Close()) })+ return ctx, db, s+}++// seedProjectPurgeFixture creates a project instance and a workspace instance+// owned by project-a, a database on each, and schema rows for both databases.+func seedProjectPurgeFixture(ctx context.Context, t *testing.T, db *sql.DB) {+ t.Helper()+ _, err := db.ExecContext(ctx, `+ INSERT INTO instance (resource_id, workspace, project) VALUES+ ('project-instance', 'default', 'project-a'),+ ('workspace-instance', 'default', NULL);+ INSERT INTO db (instance, name, project) VALUES+ ('project-instance', 'project-db', 'project-a'),+ ('workspace-instance', 'workspace-db', 'project-a');+ INSERT INTO db_schema (instance, db_name) VALUES+ ('project-instance', 'project-db'),+ ('workspace-instance', 'workspace-db');+ `)+ require.NoError(t, err)+}++// warmProjectPurgeCaches populates the instance, scoped and unscoped database,+// and schema cache entries for every seeded descendant.+func warmProjectPurgeCaches(ctx context.Context, t *testing.T, s *store.Store) {+ t.Helper()+ projectInstanceID := "project-instance"+ projectDBName := "project-db"+ workspaceInstanceID := "workspace-instance"+ workspaceDBName := "workspace-db"+ for _, find := range []*store.FindInstanceMessage{+ {Workspace: "default", ResourceID: &projectInstanceID},+ {Workspace: "default", ResourceID: &workspaceInstanceID},+ } {+ instance, err := s.GetInstance(ctx, find)+ require.NoError(t, err)+ require.NotNil(t, instance)+ }+ for _, find := range []*store.FindDatabaseMessage{+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ {Workspace: "default", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},+ {Workspace: "", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},+ } {+ database, err := s.GetDatabase(ctx, find)+ require.NoError(t, err)+ require.NotNil(t, database)+ }+ for _, schema := range []struct {+ instanceID string+ databaseName string+ }{+ {instanceID: projectInstanceID, databaseName: projectDBName},+ {instanceID: workspaceInstanceID, databaseName: workspaceDBName},+ } {+ metadata, err := s.GetDBSchemaSnapshot(ctx, "default", schema.instanceID, schema.databaseName)+ require.NoError(t, err)+ require.NotNil(t, metadata)+ }+}++// TestDeleteProjectPurgeInvalidatesDescendantCaches warms every descendant+// cache entry, purges the project, and requires the purged descendants to be+// immediately unavailable through the cached getters while surviving rows+// still resolve correctly.+func TestDeleteProjectPurgeInvalidatesDescendantCaches(t *testing.T) {+ ctx, db, s := newProjectPurgeCacheFixture(t)+ seedProjectPurgeFixture(ctx, t, db)+ warmProjectPurgeCaches(ctx, t, s)++ projectInstanceID := "project-instance"+ projectDBName := "project-db"+ workspaceInstanceID := "workspace-instance"+ workspaceDBName := "workspace-db"++ require.NoError(t, s.DeleteProject(ctx, "default", "project-a"))++ // Observable seam: purged descendants are immediately unavailable through+ // the cached getters.+ instance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})+ require.NoError(t, err)+ require.Nil(t, instance)+ for _, find := range []*store.FindDatabaseMessage{+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ } {+ database, err := s.GetDatabase(ctx, find)+ require.NoError(t, err)+ require.Nil(t, database)+ }+ schema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)+ require.NoError(t, err)+ require.Nil(t, schema)++ // Surviving rows still resolve: the workspace instance was not flushed and+ // its schema snapshot is still readable.+ workspaceInstance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &workspaceInstanceID})+ require.NoError(t, err)+ require.NotNil(t, workspaceInstance)+ workspaceSchema, err := s.GetDBSchemaSnapshot(ctx, "default", workspaceInstanceID, workspaceDBName)+ require.NoError(t, err)+ require.NotNil(t, workspaceSchema)++ // The workspace-instance database moved to the default project is re-read+ // from the database instead of being served from a stale cached entry.+ for _, find := range []*store.FindDatabaseMessage{+ {Workspace: "default", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},+ {Workspace: "", InstanceID: &workspaceInstanceID, DatabaseName: &workspaceDBName},+ } {+ movedDB, err := s.GetDatabase(ctx, find)+ require.NoError(t, err)+ require.NotNil(t, movedDB)+ require.Equal(t, "default", movedDB.ProjectID)+ }+}++// TestDeleteProjectPurgeSupportsDescendantIDReuse proves that reusing a+// purged instance resource ID and database name cannot expose stale cached+// data. Reused rows are inserted directly so a stale cache entry would still+// be observable through the getters.+func TestDeleteProjectPurgeSupportsDescendantIDReuse(t *testing.T) {+ ctx, db, s := newProjectPurgeCacheFixture(t)+ seedProjectPurgeFixture(ctx, t, db)+ warmProjectPurgeCaches(ctx, t, s)++ require.NoError(t, s.DeleteProject(ctx, "default", "project-a"))++ projectInstanceID := "project-instance"+ projectDBName := "project-db"+ _, err := db.ExecContext(ctx, `+ INSERT INTO instance (resource_id, workspace, project) VALUES+ ('project-instance', 'default', 'project-b');+ INSERT INTO db (instance, name, project) VALUES+ ('project-instance', 'project-db', 'project-b');+ `)+ require.NoError(t, err)++ reusedInstance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})+ require.NoError(t, err)+ require.NotNil(t, reusedInstance)+ require.Equal(t, "project-b", *reusedInstance.ProjectID)+ for _, find := range []*store.FindDatabaseMessage{+ {Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ {Workspace: "", InstanceID: &projectInstanceID, DatabaseName: &projectDBName},+ } {+ reusedDB, err := s.GetDatabase(ctx, find)+ require.NoError(t, err)+ require.NotNil(t, reusedDB)+ require.Equal(t, "project-b", reusedDB.ProjectID)+ }+ // The reused database has no schema row; a stale schema cache entry for the+ // purged database must not leak.+ reusedSchema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)+ require.NoError(t, err)+ require.Nil(t, reusedSchema)+}++// TestDeleteProjectFailedTransactionKeepsDescendantCaches pins that a purge+// transaction that deletes descendants but fails before commit publishes no+// invalidation. After the failed purge the underlying rows are removed+// directly, so any surviving getter result can only come from the still-warm+// cache entries.+func TestDeleteProjectFailedTransactionKeepsDescendantCaches(t *testing.T) {+ ctx, db, s := newProjectPurgeCacheFixture(t)+ seedProjectPurgeFixture(ctx, t, db)+ warmProjectPurgeCaches(ctx, t, s)++ projectInstanceID := "project-instance"+ projectDBName := "project-db"+ // project-a is not marked deleted, so DeleteProject removes all descendant+ // rows inside the transaction and then fails on the deleted guard, rolling+ // the whole purge back.+ _, err := db.ExecContext(ctx, `UPDATE project SET deleted = FALSE WHERE resource_id = 'project-a'`)+ require.NoError(t, err)+ err = s.DeleteProject(ctx, "default", "project-a")+ require.Error(t, err)++ // Remove the descendant rows behind the store's back so that only a+ // surviving cache entry can satisfy the getters.+ _, err = db.ExecContext(ctx, `+ DELETE FROM db_schema WHERE instance = 'project-instance' AND db_name = 'project-db';+ DELETE FROM db WHERE instance = 'project-instance' AND name = 'project-db';+ DELETE FROM instance WHERE resource_id = 'project-instance';+ `)+ require.NoError(t, err)++ instance, err := s.GetInstance(ctx, &store.FindInstanceMessage{Workspace: "default", ResourceID: &projectInstanceID})+ require.NoError(t, err)+ require.NotNil(t, instance)+ require.Equal(t, "project-a", *instance.ProjectID)+ database, err := s.GetDatabase(ctx, &store.FindDatabaseMessage{Workspace: "default", InstanceID: &projectInstanceID, DatabaseName: &projectDBName})+ require.NoError(t, err)+ require.NotNil(t, database)+ require.Equal(t, "project-a", database.ProjectID)+ schema, err := s.GetDBSchemaSnapshot(ctx, "default", projectInstanceID, projectDBName)+ require.NoError(t, err)+ require.NotNil(t, schema)+}diff --git a/backend/store/task_run.go b/backend/store/task_run.goindex 83baa7c24b..dbab7f0fb3 100644--- a/backend/store/task_run.go+++ b/backend/store/task_run.go@@ -3,6 +3,7 @@ package storeimport ("context""database/sql"+ "slices""time""github.com/pkg/errors"@@ -241,7 +242,9 @@ func (s *Store) ClaimAvailableTaskRuns(ctx context.Context, replicaID string) ([SELECT task_run.project, task_run.idFROM task_runJOIN project ON project.resource_id = task_run.project- WHERE task_run.status = ? AND project.deleted = FALSE+ JOIN task ON task.project = task_run.project AND task.id = task_run.task_id+ JOIN instance ON instance.resource_id = task.instance+ WHERE task_run.status = ? AND project.deleted = FALSE AND instance.deleted = FALSEFOR UPDATE OF task_run SKIP LOCKED)RETURNING id, task_id, project@@ -342,15 +345,24 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creatpayloadStr = s}}+ instanceIDs, err := s.listTaskRunCreationInstances(ctx, projects, taskUIDs)+ if err != nil {+ return err+ }tx, err := s.GetDB().BeginTx(ctx, nil)if err != nil {return errors.Wrapf(err, "failed to begin tx")}defer tx.Rollback()+ for _, instanceID := range instanceIDs {+ if err := acquireInstancePurgeLock(ctx, tx, instanceID); err != nil {+ return errors.Wrapf(err, "failed to lock instance lifecycle fence for %s", instanceID)+ }+ }lockQ := qb.Q().Space(`- SELECT task.project, task.id+ SELECT task.project, task.id, task.instanceFROM (SELECTunnest(CAST(? AS TEXT[])) AS project,@@ -364,6 +376,7 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creatreturn errors.Wrapf(err, "failed to build task lock sql")}lockedTaskCount := 0+ lockedInstanceIDs := make([]string, 0, len(instanceIDs))if err := func() error {rows, err := tx.QueryContext(ctx, lockQuery, lockArgs...)if err != nil {@@ -373,9 +386,14 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creatfor rows.Next() {var lockedProjectID stringvar lockedTaskUID int64- if err := rows.Scan(&lockedProjectID, &lockedTaskUID); err != nil {+ var lockedInstanceID string+ if err := rows.Scan(&lockedProjectID, &lockedTaskUID, &lockedInstanceID); err != nil {return errors.Wrapf(err, "failed to scan locked task")}+ if !slices.Contains(instanceIDs, lockedInstanceID) {+ return common.Errorf(common.Conflict, "task %s/%d changed to instance %s; retry", lockedProjectID, lockedTaskUID, lockedInstanceID)+ }+ lockedInstanceIDs = append(lockedInstanceIDs, lockedInstanceID)lockedTaskCount++}if err := rows.Err(); err != nil {@@ -388,8 +406,24 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creatif lockedTaskCount == 0 {return nil}+ slices.Sort(lockedInstanceIDs)+ lockedInstanceIDs = slices.Compact(lockedInstanceIDs)+ for _, instanceID := range lockedInstanceIDs {+ var deleted bool+ if err := tx.QueryRowContext(ctx, `+ SELECT deleted FROM instance WHERE resource_id = $1 FOR UPDATE+ `, instanceID).Scan(&deleted); err != nil {+ if errors.Is(err, sql.ErrNoRows) {+ return common.Errorf(common.NotFound, "instance %s not found", instanceID)+ }+ return errors.Wrapf(err, "failed to lock instance %s", instanceID)+ }+ if deleted {+ return common.Errorf(common.Conflict, "instance %s is archived", instanceID)+ }+ }- // Keep the child-to-parent lock order used by project deletion: task, then project.+ // Keep the child-to-parent lock order used by project deletion: task, instance, then project.baseID, err := nextProjectID(ctx, tx, "task_run", projectID)if err != nil {return err@@ -465,6 +499,40 @@ func (s *Store) CreatePendingTaskRuns(ctx context.Context, creator string, creatreturn nil}+func (s *Store) listTaskRunCreationInstances(ctx context.Context, projects []string, taskUIDs []int64) ([]string, error) {+ q := qb.Q().Space(`+ SELECT DISTINCT task.instance+ FROM (+ SELECT+ unnest(CAST(? AS TEXT[])) AS project,+ unnest(CAST(? AS BIGINT[])) AS task_id+ ) requested_tasks+ JOIN task ON task.project = requested_tasks.project AND task.id = requested_tasks.task_id+ ORDER BY task.instance+ `, projects, taskUIDs)+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrap(err, "failed to build task run instance query")+ }+ rows, err := s.GetDB().QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrap(err, "failed to list task run instances")+ }+ defer rows.Close()+ var instanceIDs []string+ for rows.Next() {+ var instanceID string+ if err := rows.Scan(&instanceID); err != nil {+ return nil, errors.Wrap(err, "failed to scan task run instance")+ }+ instanceIDs = append(instanceIDs, instanceID)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to read task run instances")+ }+ return instanceIDs, nil+}+// patchTaskRunStatusImpl updates a taskRun status. Returns the new state of the taskRun after update.func (*Store) patchTaskRunStatusImpl(ctx context.Context, txn *sql.Tx, patch *TaskRunStatusPatch) (*TaskRunMessage, error) {set := qb.Q()diff --git a/backend/store/task_run_lock_order_test.go b/backend/store/task_run_lock_order_test.goindex 25a3208f03..1ddac9f5d7 100644--- a/backend/store/task_run_lock_order_test.go+++ b/backend/store/task_run_lock_order_test.go@@ -3,12 +3,15 @@ package store_testimport ("context""database/sql"+ "fmt""testing""time""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/testcontainer"+ "github.com/bytebase/bytebase/backend/migrator""github.com/bytebase/bytebase/backend/store")@@ -40,6 +43,87 @@ func TestClaimAvailableTaskRunsSkipsDeletedProjects(t *testing.T) {require.False(t, replicaID.Valid)}+func TestClaimAvailableTaskRunsSkipsArchivedInstances(t *testing.T) {+ fixture := newTaskRunArchivalLockOrderFixture(t, `+ INSERT INTO instance (resource_id, workspace, deleted) VALUES ('instance-a', 'default', TRUE);+ INSERT INTO plan (id, creator, project, name, description)+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_MIGRATE');+ INSERT INTO task_run (id, project, task_id, attempt, status)+ VALUES (101, 'project-a', 101, 0, 'AVAILABLE');+ `)++ claimed, err := fixture.store.ClaimAvailableTaskRuns(fixture.ctx, "replica-a")+ require.Empty(t, claimed)+ require.NoError(t, err)++ var status string+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT status FROM task_run WHERE project = 'project-a' AND id = 101",+ ).Scan(&status))+ require.Equal(t, "AVAILABLE", status)++ var replicaID sql.NullString+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT replica_id FROM task_run WHERE project = 'project-a' AND id = 101",+ ).Scan(&replicaID))+ require.False(t, replicaID.Valid)+}++func TestClaimAvailableTaskRunsClaimsLiveInstanceSpecialTaskTypes(t *testing.T) {+ fixture := newTaskRunArchivalLockOrderFixture(t, `+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');+ INSERT INTO plan (id, creator, project, name, description)+ VALUES (101, '[email protected]', 'project-a', 'Plan A', '');+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (101, 'project-a', 101, 'instance-a', 'DATABASE_MIGRATE');+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (102, 'project-a', 101, 'instance-a', 'DATABASE_CREATE');+ INSERT INTO task_run (id, project, task_id, attempt, status)+ VALUES (101, 'project-a', 101, 0, 'AVAILABLE');+ INSERT INTO task_run (id, project, task_id, attempt, status)+ VALUES (102, 'project-a', 102, 0, 'AVAILABLE');+ `)++ claimed, err := fixture.store.ClaimAvailableTaskRuns(fixture.ctx, "replica-a")+ require.NoError(t, err)+ require.Len(t, claimed, 2, "live-instance task runs of any task type should be claimable")++ var status string+ require.NoError(t, fixture.db.QueryRowContext(fixture.ctx,+ "SELECT status FROM task_run WHERE project = 'project-a' AND id = 101",+ ).Scan(&status))+ require.Equal(t, "RUNNING", status)+}++func newTaskRunArchivalLockOrderFixture(t *testing.T, seedSQL string) *projectDeletionLockOrderFixture {+ t.Helper()+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ t.Cleanup(cancel)+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(context.Background()) })+ db := container.GetDB()+ require.NoError(t, migrator.MigrateSchema(ctx, db))++ _, err := db.ExecContext(ctx, `+ INSERT INTO workspace (resource_id) VALUES ('default');+ INSERT INTO project (resource_id, workspace, name) VALUES ('default', 'default', 'Default');+ INSERT INTO project (resource_id, workspace, name) VALUES ('project-a', 'default', 'Project A');+ `+seedSQL)+ require.NoError(t, err)++ pgURL := fmt.Sprintf(+ "host=%s port=%s user=postgres password=root-password database=postgres",+ container.GetHost(), container.GetPort(),+ )+ s, err := store.New(ctx, pgURL, false)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, s.Close()) })++ return &projectDeletionLockOrderFixture{ctx: ctx, db: db, store: s}+}+func TestCreatePendingTaskRunsDoesNotDeadlockWithProjectDeletion(t *testing.T) {fixture := newProjectDeletionLockOrderFixture(t, `INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');diff --git a/backend/tests/project_instance_archive_task_run_test.go b/backend/tests/project_instance_archive_task_run_test.gonew file mode 100644index 0000000000..136540431d--- /dev/null+++ b/backend/tests/project_instance_archive_task_run_test.go@@ -0,0 +1,200 @@+package tests++import (+ "context"+ "database/sql"+ "testing"++ "connectrpc.com/connect"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++//nolint:tparallel // Subtests mutate one task run through successive lifecycle states.+func TestArchiveInstanceRejectsActiveTaskRuns(t *testing.T) {+ t.Parallel()+ a := require.New(t)+ ctx := context.Background()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ pg, err := provisionPgInstance(ctx, t)+ a.NoError(err)+ instance := createProjectInstanceTestInstance(+ ctx,+ t,+ ctl,+ &ctl.project.Name,+ "archive-active-task-runs",+ "archive active task runs",+ pg,+ )+ projectID, err := common.GetProjectID(ctl.project.Name)+ a.NoError(err)+ _, instanceID, err := common.GetInstanceResourceName(instance.Name)+ a.NoError(err)++ metadataDB, err := sql.Open("pgx", ctl.profile.PgURL)+ a.NoError(err)+ defer metadataDB.Close()+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO plan (id, creator, project, name, description)+ VALUES (101, '[email protected]', $1, 'Archive precondition plan', '')+ `, projectID)+ a.NoError(err)+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (101, $1, 101, $2, 'DATABASE_CREATE')+ `, projectID, instanceID)+ a.NoError(err)+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO task_run (id, project, task_id, attempt, status)+ VALUES (101, $1, 101, 0, 'PENDING')+ `, projectID)+ a.NoError(err)++ for _, status := range []string{"PENDING", "AVAILABLE", "RUNNING"} {+ t.Run(status, func(t *testing.T) {+ _, err := metadataDB.ExecContext(ctx, `+ UPDATE task_run SET status = $1 WHERE project = $2 AND id = 101+ `, status, projectID)+ require.NoError(t, err)++ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{+ Name: instance.Name,+ }))+ require.Error(t, err)+ require.Equal(t, connect.CodeFailedPrecondition, connect.CodeOf(err))++ got, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))+ require.NoError(t, err)+ require.Equal(t, v1pb.State_ACTIVE, got.Msg.State)+ })+ }++ _, err = metadataDB.ExecContext(ctx, `+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101+ `, projectID)+ a.NoError(err)+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{+ Name: instance.Name,+ }))+ a.NoError(err)+ archived, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))+ a.NoError(err)+ a.Equal(v1pb.State_DELETED, archived.Msg.State)++ // Legacy data may contain an active task run on an already archived+ // instance. Restoration must not make that work runnable without an+ // explicit cancel-and-retry decision from the user.+ _, err = metadataDB.ExecContext(ctx, `+ UPDATE task_run SET status = 'PENDING' WHERE project = $1 AND id = 101+ `, projectID)+ a.NoError(err)+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{+ Name: instance.Name,+ }))+ a.Error(err)+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))+ _, err = ctl.instanceServiceClient.UndeleteInstance(ctx, connect.NewRequest(&v1pb.UndeleteInstanceRequest{+ Name: instance.Name,+ }))+ a.Error(err)+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))+ stillArchived, err := ctl.instanceServiceClient.GetInstance(ctx, connect.NewRequest(&v1pb.GetInstanceRequest{Name: instance.Name}))+ a.NoError(err)+ a.Equal(v1pb.State_DELETED, stillArchived.Msg.State)++ _, err = metadataDB.ExecContext(ctx, `+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101+ `, projectID)+ a.NoError(err)+ restored, err := ctl.instanceServiceClient.UndeleteInstance(ctx, connect.NewRequest(&v1pb.UndeleteInstanceRequest{+ Name: instance.Name,+ }))+ a.NoError(err)+ a.Equal(v1pb.State_ACTIVE, restored.Msg.State)+}++func TestForceArchiveInstanceRejectsActiveTaskRunsWithoutTransferringDatabases(t *testing.T) {+ t.Parallel()+ a := require.New(t)+ ctx := context.Background()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ pg, err := provisionPgInstance(ctx, t)+ a.NoError(err)+ instance := createProjectInstanceTestInstance(+ ctx,+ t,+ ctl,+ nil,+ "force-archive-active-task-runs",+ "force archive active task runs",+ pg,+ )+ projectID, err := common.GetProjectID(ctl.project.Name)+ a.NoError(err)+ _, instanceID, err := common.GetInstanceResourceName(instance.Name)+ a.NoError(err)++ metadataDB, err := sql.Open("pgx", ctl.profile.PgURL)+ a.NoError(err)+ defer metadataDB.Close()+ var workspaceID string+ a.NoError(metadataDB.QueryRowContext(ctx, `+ SELECT workspace FROM instance WHERE resource_id = $1+ `, instanceID).Scan(&workspaceID))+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO db (instance, name, project) VALUES ($1, 'db-a', $2)+ `, instanceID, projectID)+ a.NoError(err)+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO plan (id, creator, project, name, description)+ VALUES (101, '[email protected]', $1, 'Force archive precondition plan', '')+ `, projectID)+ a.NoError(err)+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO task (id, project, plan_id, instance, type)+ VALUES (101, $1, 101, $2, 'DATABASE_CREATE')+ `, projectID, instanceID)+ a.NoError(err)+ _, err = metadataDB.ExecContext(ctx, `+ INSERT INTO task_run (id, project, task_id, attempt, status)+ VALUES (101, $1, 101, 0, 'RUNNING')+ `, projectID)+ a.NoError(err)++ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{+ Name: instance.Name,+ Force: true,+ }))+ a.Error(err)+ a.Equal(connect.CodeFailedPrecondition, connect.CodeOf(err))+ var databaseProjectID string+ a.NoError(metadataDB.QueryRowContext(ctx, `+ SELECT project FROM db WHERE instance = $1 AND name = 'db-a'+ `, instanceID).Scan(&databaseProjectID))+ a.Equal(projectID, databaseProjectID, "a rejected force archive must not transfer databases")++ _, err = metadataDB.ExecContext(ctx, `+ UPDATE task_run SET status = 'CANCELED' WHERE project = $1 AND id = 101+ `, projectID)+ a.NoError(err)+ _, err = ctl.instanceServiceClient.DeleteInstance(ctx, connect.NewRequest(&v1pb.DeleteInstanceRequest{+ Name: instance.Name,+ Force: true,+ }))+ a.NoError(err)+ a.NoError(metadataDB.QueryRowContext(ctx, `+ SELECT project FROM db WHERE instance = $1 AND name = 'db-a'+ `, instanceID).Scan(&databaseProjectID))+ a.True(common.IsDefaultProject(workspaceID, databaseProjectID))+}diff --git a/docs/adr/0002-support-workspace-and-project-instances.md b/docs/adr/0002-support-workspace-and-project-instances.mdindex 71d027af9e..4d0bc43e66 100644--- a/docs/adr/0002-support-workspace-and-project-instances.md+++ b/docs/adr/0002-support-workspace-and-project-instances.md@@ -189,17 +189,20 @@ writers so that no writer can commit after the owning project is removed.Directly archiving a project instance leaves its databases assigned to theowning project, but they are unavailable to data and workflow operations until-the instance is restored. Instance restore and purge operations remain-available. Archival does not stop or delete the physical databases and does not-fence already-started descendant writers; their committed results remain-unavailable until restoration. Purging serializes against those writers, then-deletes the databases, their metadata and history, and instance-targeting tasks-and task runs. Project-level issues, plans, and releases remain, retaining-canonical target names that may later identify a replacement resource if the-purged instance ID is reused; audit logs also remain under the workspace-retention policy. This matches the workspace-instance purge boundary, except-that the workspace-instance `force` behavior that transfers databases to the-default project does not apply. `DeleteInstance` rejects `force=true` with+the instance is restored. Archive and restore return `FAILED_PRECONDITION` while+any targeting task run is `PENDING`, `AVAILABLE`, or `RUNNING`; operators must+cancel that work or wait for it to finish before retrying the lifecycle request.+Applying the same guard to restoration prevents legacy queued work from+silently resuming. Archival does not stop or delete the physical databases.+Purging serializes against descendant writers, then deletes the databases,+their metadata and history, and instance-targeting tasks and task runs.+Project-level issues, plans, and releases remain, retaining canonical target+names that may later identify a replacement resource if the purged instance ID+is reused; audit logs also remain under the workspace retention policy. This+matches the workspace-instance purge boundary, except that the+workspace-instance `force` behavior that transfers databases to the default+project does not apply. For workspace instances, that transfer and archival+commit atomically after the active-task-run check. `DeleteInstance` rejects `force=true` with`INVALID_ARGUMENT` for a project instance; ordinary archival needs no forcebecause its databases remain owned by the project.diff --git a/frontend/src/types/proto-es/v1/instance_service_pb.d.ts b/frontend/src/types/proto-es/v1/instance_service_pb.d.tsindex 96533a80f9..0785d7d6b8 100644--- a/frontend/src/types/proto-es/v1/instance_service_pb.d.ts+++ b/frontend/src/types/proto-es/v1/instance_service_pb.d.ts@@ -1822,6 +1822,8 @@ export declare const InstanceService: GenService<{},/*** Deletes or soft-deletes a database instance.+ * Soft-delete requests fail with FAILED_PRECONDITION while any task run+ * targeting the instance is pending, available, or running.* Permissions required: bb.instances.delete** @generated from rpc bytebase.v1.InstanceService.DeleteInstance@@ -1833,6 +1835,8 @@ export declare const InstanceService: GenService<{},/*** Restores a soft-deleted database instance.+ * Restore requests fail with FAILED_PRECONDITION while any task run targeting+ * the instance is pending, available, or running.* Permissions required: bb.instances.undelete** @generated from rpc bytebase.v1.InstanceService.UndeleteInstancediff --git a/proto/gen/grpc-doc/openapi.yaml b/proto/gen/grpc-doc/openapi.yamlindex c07531efa6..f097cd19b7 100644--- a/proto/gen/grpc-doc/openapi.yaml+++ b/proto/gen/grpc-doc/openapi.yaml@@ -1872,6 +1872,8 @@ paths:summary: DeleteInstancedescription: |-Deletes or soft-deletes a database instance.+ Soft-delete requests fail with FAILED_PRECONDITION while any task run+ targeting the instance is pending, available, or running.Permissions required: bb.instances.deleteoperationId: InstanceService_DeleteInstanceparameters:@@ -5123,6 +5125,8 @@ paths:summary: UndeleteInstancedescription: |-Restores a soft-deleted database instance.+ Restore requests fail with FAILED_PRECONDITION while any task run targeting+ the instance is pending, available, or running.Permissions required: bb.instances.undeleteoperationId: InstanceService_UndeleteInstanceparameters:@@ -6936,6 +6940,8 @@ paths:summary: DeleteInstancedescription: |-Deletes or soft-deletes a database instance.+ Soft-delete requests fail with FAILED_PRECONDITION while any task run+ targeting the instance is pending, available, or running.Permissions required: bb.instances.deleteoperationId: InstanceService_DeleteInstance2parameters:@@ -10196,6 +10202,8 @@ paths:summary: UndeleteInstancedescription: |-Restores a soft-deleted database instance.+ Restore requests fail with FAILED_PRECONDITION while any task run targeting+ the instance is pending, available, or running.Permissions required: bb.instances.undeleteoperationId: InstanceService_UndeleteInstance2parameters:diff --git a/proto/gen/grpc-doc/v1/README.md b/proto/gen/grpc-doc/v1/README.mdindex 6abb05f76e..3a3e1a3ccd 100644--- a/proto/gen/grpc-doc/v1/README.md+++ b/proto/gen/grpc-doc/v1/README.md@@ -7219,8 +7219,8 @@ InstanceService manages database instances and their connections.| ListInstances | [ListInstancesRequest](#bytebase-v1-ListInstancesRequest) | [ListInstancesResponse](#bytebase-v1-ListInstancesResponse) | Lists database instances, optionally within a project. Permissions required: bb.instances.list || CreateInstance | [CreateInstanceRequest](#bytebase-v1-CreateInstanceRequest) | [Instance](#bytebase-v1-Instance) | Creates a new database instance. Permissions required: bb.instances.create || UpdateInstance | [UpdateInstanceRequest](#bytebase-v1-UpdateInstanceRequest) | [Instance](#bytebase-v1-Instance) | Updates a database instance. Permissions required: bb.instances.update |-| DeleteInstance | [DeleteInstanceRequest](#bytebase-v1-DeleteInstanceRequest) | [.google.protobuf.Empty](#google-protobuf-Empty) | Deletes or soft-deletes a database instance. Permissions required: bb.instances.delete |-| UndeleteInstance | [UndeleteInstanceRequest](#bytebase-v1-UndeleteInstanceRequest) | [Instance](#bytebase-v1-Instance) | Restores a soft-deleted database instance. Permissions required: bb.instances.undelete |+| DeleteInstance | [DeleteInstanceRequest](#bytebase-v1-DeleteInstanceRequest) | [.google.protobuf.Empty](#google-protobuf-Empty) | Deletes or soft-deletes a database instance. Soft-delete requests fail with FAILED_PRECONDITION while any task run targeting the instance is pending, available, or running. Permissions required: bb.instances.delete |+| UndeleteInstance | [UndeleteInstanceRequest](#bytebase-v1-UndeleteInstanceRequest) | [Instance](#bytebase-v1-Instance) | Restores a soft-deleted database instance. Restore requests fail with FAILED_PRECONDITION while any task run targeting the instance is pending, available, or running. Permissions required: bb.instances.undelete || SyncInstance | [SyncInstanceRequest](#bytebase-v1-SyncInstanceRequest) | [SyncInstanceResponse](#bytebase-v1-SyncInstanceResponse) | Syncs database schemas and metadata from an instance. Permissions required: bb.instances.sync || ListInstanceDatabase | [ListInstanceDatabaseRequest](#bytebase-v1-ListInstanceDatabaseRequest) | [ListInstanceDatabaseResponse](#bytebase-v1-ListInstanceDatabaseResponse) | Lists all databases within an instance without creating them. Permissions required: bb.instances.get || BatchSyncInstances | [BatchSyncInstancesRequest](#bytebase-v1-BatchSyncInstancesRequest) | [BatchSyncInstancesResponse](#bytebase-v1-BatchSyncInstancesResponse) | Syncs multiple instances in a single request. Permissions required: bb.instances.sync |diff --git a/proto/gen/grpc-doc/v1/index.html b/proto/gen/grpc-doc/v1/index.htmlindex 88021dda46..87e3159283 100644--- a/proto/gen/grpc-doc/v1/index.html+++ b/proto/gen/grpc-doc/v1/index.html@@ -20325,6 +20325,8 @@ Permissions required: bb.instances.update</p></td><td><a href="#bytebase.v1.DeleteInstanceRequest">DeleteInstanceRequest</a></td><td><a href="#google.protobuf.Empty">.google.protobuf.Empty</a></td><td><p>Deletes or soft-deletes a database instance.+Soft-delete requests fail with FAILED_PRECONDITION while any task run+targeting the instance is pending, available, or running.Permissions required: bb.instances.delete</p></td></tr>@@ -20333,6 +20335,8 @@ Permissions required: bb.instances.delete</p></td><td><a href="#bytebase.v1.UndeleteInstanceRequest">UndeleteInstanceRequest</a></td><td><a href="#bytebase.v1.Instance">Instance</a></td><td><p>Restores a soft-deleted database instance.+Restore requests fail with FAILED_PRECONDITION while any task run targeting+the instance is pending, available, or running.Permissions required: bb.instances.undelete</p></td></tr>diff --git a/proto/v1/v1/instance_service.proto b/proto/v1/v1/instance_service.protoindex 3d13235951..d9df5f7428 100644--- a/proto/v1/v1/instance_service.proto+++ b/proto/v1/v1/instance_service.proto@@ -78,6 +78,8 @@ service InstanceService {}// Deletes or soft-deletes a database instance.+ // Soft-delete requests fail with FAILED_PRECONDITION while any task run+ // targeting the instance is pending, available, or running.// Permissions required: bb.instances.deleterpc DeleteInstance(DeleteInstanceRequest) returns (google.protobuf.Empty) {option (google.api.http) = {@@ -91,6 +93,8 @@ service InstanceService {}// Restores a soft-deleted database instance.+ // Restore requests fail with FAILED_PRECONDITION while any task run targeting+ // the instance is pending, available, or running.// Permissions required: bb.instances.undeleterpc UndeleteInstance(UndeleteInstanceRequest) returns (Instance) {option (google.api.http) = {