Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: backfill UI plan draft review issues (#20928)
p0ny3aa20920c2be
diff --git a/backend/api/v1/plan_service_test.go b/backend/api/v1/plan_service_test.goindex 254d794368..794a31ba57 100644--- a/backend/api/v1/plan_service_test.go+++ b/backend/api/v1/plan_service_test.go@@ -16,32 +16,66 @@ import ("github.com/bytebase/bytebase/backend/store")-func TestPlanServiceListPlansHidesIssueLessDatabaseDrafts(t *testing.T) {+func TestPlanServiceListPlansHidesMalformedUIPlans(t *testing.T) {ctx := context.Background()stores := setupPlanServiceTestStore(ctx, t)service := NewPlanService(stores, nil, nil, nil, nil)- changeDraft, err := stores.CreatePlan(ctx, &store.PlanMessage{- ProjectID: "project-a",- Name: "change draft",- Config: &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{{- Id: "change",+ createPlan := func(name string, config *storepb.PlanConfig) *store.PlanMessage {+ t.Helper()+ plan, err := stores.CreatePlan(ctx, &store.PlanMessage{+ ProjectID: "project-a",+ Name: name,+ Config: config,+ }, "[email protected]")+ require.NoError(t, err)+ return plan+ }+ changeConfig := func(id, release string) *storepb.PlanConfig {+ return &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{{+ Id: id,Config: &storepb.PlanConfig_Spec_ChangeDatabaseConfig{- ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{},+ ChangeDatabaseConfig: &storepb.PlanConfig_ChangeDatabaseConfig{Release: release},},- }}},- }, "[email protected]")- require.NoError(t, err)- createDraft, err := stores.CreatePlan(ctx, &store.PlanMessage{- ProjectID: "project-a",- Name: "create draft",- Config: &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{{- Id: "create",+ }}}+ }+ createConfig := func(id string) *storepb.PlanConfig {+ return &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{{+ Id: id,Config: &storepb.PlanConfig_Spec_CreateDatabaseConfig{CreateDatabaseConfig: &storepb.PlanConfig_CreateDatabaseConfig{},},- }}},- }, "[email protected]")+ }}}+ }++ createPlan("malformed change", changeConfig("malformed-change", ""))+ createPlan("malformed create", createConfig("malformed-create"))+ createPlan("malformed mixed", &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{+ createConfig("mixed-create").Specs[0],+ changeConfig("mixed-change", "").Specs[0],+ }})+ oldMalformed := createPlan("old malformed", changeConfig("old", ""))+ _, err := stores.GetDB().ExecContext(ctx, `+ UPDATE plan SET created_at = CURRENT_TIMESTAMP - INTERVAL '31 days'+ WHERE project = $1 AND id = $2`, oldMalformed.ProjectID, oldMalformed.UID)+ require.NoError(t, err)+ gitOps := createPlan("GitOps", changeConfig("gitops", "projects/project-a/releases/release-a"))+ export := createPlan("export", &storepb.PlanConfig{Specs: []*storepb.PlanConfig_Spec{{+ Id: "export",+ Config: &storepb.PlanConfig_Spec_ExportDataConfig{+ ExportDataConfig: &storepb.PlanConfig_ExportDataConfig{},+ },+ }}})+ deleted := createPlan("deleted", changeConfig("deleted", ""))+ _, err = stores.UpdatePlan(ctx, &store.UpdatePlanMessage{+ UID: deleted.UID, ProjectID: deleted.ProjectID, Deleted: new(true),+ })+ require.NoError(t, err)+ linked := createPlan("linked", changeConfig("linked", ""))+ _, err = stores.CreateIssue(ctx, &store.IssueMessage{+ ProjectID: linked.ProjectID, CreatorEmail: "[email protected]", PlanUID: &linked.UID,+ Title: "linked issue", Type: storepb.Issue_DATABASE_CHANGE, Payload: &storepb.Issue{},+ })require.NoError(t, err)response, err := service.ListPlans(ctx, connect.NewRequest(&v1pb.ListPlansRequest{@@ -53,14 +87,13 @@ func TestPlanServiceListPlansHidesIssueLessDatabaseDrafts(t *testing.T) {for _, plan := range response.Msg.Plans {got = append(got, plan.Title)}- require.Empty(t, got)- for _, plan := range []*store.PlanMessage{changeDraft, createDraft} {- gotPlan, err := service.GetPlan(ctx, connect.NewRequest(&v1pb.GetPlanRequest{- Name: fmt.Sprintf("projects/project-a/plans/%d", plan.UID),- }))- require.NoError(t, err)- require.Equal(t, plan.Name, gotPlan.Msg.Title)- }+ require.ElementsMatch(t, []string{gitOps.Name, export.Name, deleted.Name, linked.Name}, got)++ gotMalformed, err := service.GetPlan(ctx, connect.NewRequest(&v1pb.GetPlanRequest{+ Name: fmt.Sprintf("projects/project-a/plans/%d", oldMalformed.UID),+ }))+ require.NoError(t, err)+ require.Equal(t, oldMalformed.Name, gotMalformed.Msg.Title)}func TestPlanServiceCreatePlanRejectsMixedDatabaseSpecs(t *testing.T) {diff --git a/backend/migrator/migration/3.21/0001##backfill_ui_plan_draft_issues.sql b/backend/migrator/migration/3.21/0001##backfill_ui_plan_draft_issues.sqlnew file mode 100644index 0000000000..3534509d5d--- /dev/null+++ b/backend/migrator/migration/3.21/0001##backfill_ui_plan_draft_issues.sql@@ -0,0 +1,2 @@+-- The data migration is implemented by migrate3_21_1.+SELECT 1;diff --git a/backend/migrator/migration_3_21_1.go b/backend/migrator/migration_3_21_1.gonew file mode 100644index 0000000000..8e22ee6e96--- /dev/null+++ b/backend/migrator/migration_3_21_1.go@@ -0,0 +1,341 @@+package migrator++import (+ "context"+ "database/sql"+ "strconv"+ "time"++ "github.com/lib/pq"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/store"+)++const migration3_21_1BatchSize = 100++type uiPlanDraftCandidate struct {+ planID int64+ creator string+ projectID string+ title string+ description string+}++type migration3_21_1Queryer interface {+ QueryContext(context.Context, string, ...any) (*sql.Rows, error)+}++func migrate3_21_1(ctx context.Context, conn *sql.Conn) error {+ var migrationTime time.Time+ if err := conn.QueryRowContext(ctx, `SELECT CURRENT_TIMESTAMP`).Scan(&migrationTime); err != nil {+ return errors.Wrap(err, "failed to get migration time")+ }+ return migrate3_21_1At(ctx, conn, migrationTime)+}++func migrate3_21_1At(ctx context.Context, conn *sql.Conn, migrationTime time.Time) error {+ afterProjectID := ""+ for {+ projectIDs, err := listMigration3_21_1ProjectPage(ctx, conn, afterProjectID)+ if err != nil {+ return err+ }+ if len(projectIDs) == 0 {+ return nil+ }+ highWaterByProject, err := captureMigration3_21_1HighWater(ctx, conn, projectIDs)+ if err != nil {+ return err+ }+ for _, projectID := range projectIDs {+ highWater, ok := highWaterByProject[projectID]+ if !ok {+ continue+ }+ var afterPlanID int64+ for {+ candidates, err := findMigration3_21_1Candidates(+ ctx,+ conn,+ migrationTime,+ projectID,+ nil,+ &afterPlanID,+ &highWater,+ migration3_21_1BatchSize,+ false,+ )+ if err != nil {+ return err+ }+ if len(candidates) == 0 {+ break+ }+ if err := migrate3_21_1Batch(ctx, conn, migrationTime, candidates); err != nil {+ return err+ }+ afterPlanID = candidates[len(candidates)-1].planID+ }+ }+ afterProjectID = projectIDs[len(projectIDs)-1]+ }+}++func listMigration3_21_1ProjectPage(ctx context.Context, conn *sql.Conn, afterProjectID string) ([]string, error) {+ rows, err := conn.QueryContext(ctx, `+ SELECT resource_id+ FROM project+ WHERE resource_id > $1+ ORDER BY resource_id+ LIMIT $2`, afterProjectID, migration3_21_1BatchSize)+ if err != nil {+ return nil, errors.Wrap(err, "failed to list projects for UI Plan draft backfill")+ }+ defer rows.Close()++ var projectIDs []string+ for rows.Next() {+ var projectID string+ if err := rows.Scan(&projectID); err != nil {+ return nil, errors.Wrap(err, "failed to scan project for UI Plan draft backfill")+ }+ projectIDs = append(projectIDs, projectID)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to iterate projects for UI Plan draft backfill")+ }+ return projectIDs, nil+}++func captureMigration3_21_1HighWater(ctx context.Context, conn *sql.Conn, projectIDs []string) (map[string]int64, error) {+ tx, err := conn.BeginTx(ctx, nil)+ if err != nil {+ return nil, errors.Wrap(err, "failed to begin UI Plan high-water capture")+ }+ defer tx.Rollback()++ var lockedProjectIDs []string+ if err := func() error {+ rows, err := tx.QueryContext(ctx, `+ SELECT resource_id+ FROM project+ WHERE resource_id = ANY($1)+ ORDER BY resource_id+ FOR UPDATE`, pq.Array(projectIDs))+ if err != nil {+ return errors.Wrap(err, "failed to lock projects for UI Plan high-water capture")+ }+ defer rows.Close()+ for rows.Next() {+ var projectID string+ if err := rows.Scan(&projectID); err != nil {+ return errors.Wrap(err, "failed to scan locked project")+ }+ lockedProjectIDs = append(lockedProjectIDs, projectID)+ }+ return errors.Wrap(rows.Err(), "failed to iterate locked projects")+ }(); err != nil {+ return nil, err+ }++ highWaterByProject := make(map[string]int64)+ if len(lockedProjectIDs) > 0 {+ rows, err := tx.QueryContext(ctx, `+ SELECT project, MAX(id)+ FROM plan+ WHERE project = ANY($1)+ GROUP BY project`, pq.Array(lockedProjectIDs))+ if err != nil {+ return nil, errors.Wrap(err, "failed to capture UI Plan high-water marks")+ }+ defer rows.Close()+ for rows.Next() {+ var projectID string+ var highWater int64+ if err := rows.Scan(&projectID, &highWater); err != nil {+ return nil, errors.Wrap(err, "failed to scan UI Plan high-water mark")+ }+ highWaterByProject[projectID] = highWater+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to iterate UI Plan high-water marks")+ }+ }++ if err := tx.Commit(); err != nil {+ return nil, errors.Wrap(err, "failed to commit UI Plan high-water capture")+ }+ return highWaterByProject, nil+}++func migrate3_21_1Batch(ctx context.Context, conn *sql.Conn, migrationTime time.Time, candidates []uiPlanDraftCandidate) error {+ tx, err := conn.BeginTx(ctx, nil)+ if err != nil {+ return errors.Wrap(err, "failed to begin UI Plan draft backfill batch")+ }+ defer tx.Rollback()++ projectID := candidates[0].projectID+ planIDs := make([]int64, 0, len(candidates))+ for _, candidate := range candidates {+ key := candidate.projectID + "/" + strconv.FormatInt(candidate.planID, 10)+ if err := store.AcquireAdvisoryXactLockWithStringKey(+ ctx,+ tx,+ store.AdvisoryLockKeyPlanIssueRollout,+ key,+ ); err != nil {+ return errors.Wrapf(err, "failed to lock Plan %s", key)+ }+ planIDs = append(planIDs, candidate.planID)+ }++ eligible, err := findMigration3_21_1Candidates(ctx, tx, migrationTime, projectID, planIDs, nil, nil, 0, true)+ if err != nil {+ return err+ }+ if len(eligible) == 0 {+ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit empty UI Plan draft backfill batch")+ }+ return nil+ }++ var lockedProjectID string+ if err := tx.QueryRowContext(ctx, `+ SELECT resource_id+ FROM project+ WHERE resource_id = $1+ FOR UPDATE`, projectID).Scan(&lockedProjectID); err != nil {+ return errors.Wrapf(err, "failed to lock project %s", projectID)+ }++ var issueID int64+ if err := tx.QueryRowContext(ctx, `+ SELECT GREATEST(COALESCE(MAX(id), 0), 100)+ FROM issue+ WHERE project = $1`, projectID).Scan(&issueID); err != nil {+ return errors.Wrapf(err, "failed to get maximum issue ID for project %s", projectID)+ }++ statement, err := tx.PrepareContext(ctx, `+ INSERT INTO issue (+ id, creator, created_at, updated_at, project, plan_id,+ name, status, type, description, payload, ts_vector+ ) VALUES ($1, $2, $3, $3, $4, $5, $6, 'OPEN', 'DATABASE_CHANGE', $7, '{"draft":true}', $8)`)+ if err != nil {+ return errors.Wrap(err, "failed to prepare UI Plan draft backfill")+ }+ defer statement.Close()++ for _, candidate := range eligible {+ issueID+++ if _, err := statement.ExecContext(+ ctx,+ issueID,+ candidate.creator,+ migrationTime,+ candidate.projectID,+ candidate.planID,+ candidate.title,+ candidate.description,+ store.IssueSearchVector(candidate.title, candidate.description),+ ); err != nil {+ return errors.Wrapf(err, "failed to backfill draft issue for Plan %s/%d", candidate.projectID, candidate.planID)+ }+ }++ if err := tx.Commit(); err != nil {+ return errors.Wrap(err, "failed to commit UI Plan draft backfill batch")+ }+ return nil+}++func findMigration3_21_1Candidates(+ ctx context.Context,+ queryer migration3_21_1Queryer,+ migrationTime time.Time,+ projectID string,+ planIDs []int64,+ afterPlanID *int64,+ highWater *int64,+ limit int,+ lock bool,+) ([]uiPlanDraftCandidate, error) {+ query := `+ SELECT plan.id, plan.creator, plan.project, plan.name, plan.description+ FROM plan+ WHERE NOT plan.deleted+ AND COALESCE(plan.config->>'hasRollout', 'false') = 'false'+ AND plan.created_at >= $1::TIMESTAMPTZ - INTERVAL '30 days'+ AND plan.project = $2+ AND ($3::BIGINT[] IS NULL OR plan.id = ANY($3))+ AND ($4::BIGINT IS NULL OR plan.id > $4)+ AND ($5::BIGINT IS NULL OR plan.id <= $5)+ AND NOT EXISTS (+ SELECT 1+ FROM issue linked_issue+ WHERE linked_issue.project = plan.project+ AND linked_issue.plan_id = plan.id+ )+ AND jsonb_typeof(plan.config->'specs') = 'array'+ AND jsonb_array_length(plan.config->'specs') > 0+ AND (+ NOT EXISTS (+ SELECT 1+ FROM jsonb_array_elements(plan.config->'specs') AS spec+ WHERE jsonb_typeof(spec->'createDatabaseConfig') IS DISTINCT FROM 'object'+ OR spec ? 'changeDatabaseConfig'+ OR spec ? 'exportDataConfig'+ )+ OR NOT EXISTS (+ SELECT 1+ FROM jsonb_array_elements(plan.config->'specs') AS spec+ WHERE jsonb_typeof(spec->'changeDatabaseConfig') IS DISTINCT FROM 'object'+ OR spec ? 'createDatabaseConfig'+ OR spec ? 'exportDataConfig'+ OR NULLIF(spec->'changeDatabaseConfig'->>'release', '') IS NOT NULL+ )+ )+ ORDER BY plan.project, plan.id`+ if limit > 0 {+ query += " LIMIT $6"+ }+ if lock {+ query += " FOR UPDATE OF plan"+ }++ var planIDsArg any+ if len(planIDs) > 0 {+ planIDsArg = pq.Array(planIDs)+ }+ args := []any{migrationTime, projectID, planIDsArg, afterPlanID, highWater}+ if limit > 0 {+ args = append(args, limit)+ }+ rows, err := queryer.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, errors.Wrap(err, "failed to list UI Plan draft candidates")+ }+ defer rows.Close()++ var candidates []uiPlanDraftCandidate+ for rows.Next() {+ var candidate uiPlanDraftCandidate+ if err := rows.Scan(+ &candidate.planID,+ &candidate.creator,+ &candidate.projectID,+ &candidate.title,+ &candidate.description,+ ); err != nil {+ return nil, errors.Wrap(err, "failed to scan UI Plan draft candidate")+ }+ candidates = append(candidates, candidate)+ }+ if err := rows.Err(); err != nil {+ return nil, errors.Wrap(err, "failed to iterate UI Plan draft candidates")+ }+ return candidates, nil+}diff --git a/backend/migrator/migrator.go b/backend/migrator/migrator.goindex b7d2a5f4b5..98df19e99b 100644--- a/backend/migrator/migrator.go+++ b/backend/migrator/migrator.go@@ -38,6 +38,7 @@ type GoMigrationFunc func(ctx context.Context, conn *sql.Conn) error// - Operations that benefit from programmatic controlvar goMigrations = map[string]GoMigrationFunc{"3.13.21": migrate3_13_21,+ "3.21.1": migrate3_21_1,}// MigrateSchema migrates the schema for metadata database.diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 2affbac84c..ffc88271c2 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -3,6 +3,7 @@ package migratorimport ("context""testing"+ "time""github.com/blang/semver/v4""github.com/stretchr/testify/assert"@@ -10,13 +11,14 @@ import ("github.com/bytebase/bytebase/backend/common/testcontainer"_ "github.com/bytebase/bytebase/backend/plugin/db/pg"+ "github.com/bytebase/bytebase/backend/store")func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.21.0"), *files[len(files)-1].version)- require.Equal(t, "migration/3.21/0000##add_query_history_project_created_at_index.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.21.1"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.21/0001##backfill_ui_plan_draft_issues.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {@@ -34,6 +36,259 @@ func TestVersionUnique(t *testing.T) {}}+func TestMigration3_21_1_BackfillUIPlanDraftIssues(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(ctx) })++ tx, err := container.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ defer tx.Rollback()+ var migrationTime time.Time+ require.NoError(t, tx.QueryRowContext(ctx, `SELECT CURRENT_TIMESTAMP`).Scan(&migrationTime))++ _, err = tx.ExecContext(ctx, `+ CREATE TABLE project (+ resource_id TEXT PRIMARY KEY+ );+ CREATE TABLE plan (+ id BIGINT NOT NULL,+ deleted BOOLEAN NOT NULL DEFAULT FALSE,+ creator TEXT NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ project TEXT NOT NULL,+ name TEXT NOT NULL,+ description TEXT NOT NULL,+ config JSONB NOT NULL,+ PRIMARY KEY (project, id)+ );+ CREATE TABLE issue (+ id BIGINT NOT NULL,+ creator TEXT NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,+ project TEXT NOT NULL,+ plan_id BIGINT,+ name TEXT NOT NULL,+ status TEXT NOT NULL,+ type TEXT NOT NULL,+ description TEXT NOT NULL DEFAULT '',+ payload JSONB NOT NULL DEFAULT '{}',+ ts_vector TSVECTOR,+ PRIMARY KEY (project, id),+ UNIQUE (project, plan_id)+ );++ INSERT INTO project (resource_id) VALUES ('project-a'), ('project-b');+ INSERT INTO plan (id, creator, created_at, project, name, description, config) VALUES+ (1, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '29 days', 'project-a', 'recent change', 'change description',+ '{"specs":[{"changeDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),+ (2, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days', 'project-a', 'boundary create', 'create description',+ '{"specs":[{"createDatabaseConfig":{}}]}'),+ (3, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'release GitOps', '',+ '{"specs":[{"changeDatabaseConfig":{"release":"projects/project-a/releases/release-a"}}]}'),+ (4, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'export', '',+ '{"specs":[{"exportDataConfig":{}}]}'),+ (5, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'mixed', '',+ '{"specs":[{"createDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),+ (6, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'deleted', '',+ '{"specs":[{"changeDatabaseConfig":{}}]}'),+ (7, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days 1 microsecond', 'project-a', 'old', '',+ '{"specs":[{"changeDatabaseConfig":{}}]}'),+ (8, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'linked', '',+ '{"specs":[{"changeDatabaseConfig":{}}]}'),+ (9, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-b', 'other project change', 'other description',+ '{"specs":[{"changeDatabaseConfig":{}}]}'),+ (10, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'rolled out', '',+ '{"specs":[{"changeDatabaseConfig":{}}],"hasRollout":true}');+ UPDATE plan SET deleted = TRUE WHERE project = 'project-a' AND id = 6;+ INSERT INTO issue (id, creator, project, plan_id, name, status, type, payload)+ VALUES (150, '[email protected]', 'project-a', 8, 'existing issue', 'OPEN', 'DATABASE_CHANGE', '{}');+ `)+ require.NoError(t, err)+ require.NoError(t, tx.Commit())++ conn, err := container.GetDB().Conn(ctx)+ require.NoError(t, err)+ defer conn.Close()+ var migrationPID int+ require.NoError(t, conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&migrationPID))++ createPlanTx, err := container.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ defer createPlanTx.Rollback()+ var lockedProjectID string+ require.NoError(t, createPlanTx.QueryRowContext(ctx, `+ SELECT resource_id FROM project WHERE resource_id = 'project-a' FOR UPDATE`).Scan(&lockedProjectID))+ _, err = createPlanTx.ExecContext(ctx, `+ INSERT INTO plan (id, creator, created_at, project, name, description, config)+ VALUES (11, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'late commit', 'late description',+ '{"specs":[{"changeDatabaseConfig":{}}]}')`)+ require.NoError(t, err)++ discoveryDone := make(chan error, 1)+ go func() {+ discoveryDone <- migrate3_21_1At(ctx, conn, migrationTime)+ }()+ require.Eventually(t, func() bool {+ var waiting bool+ err := container.GetDB().QueryRowContext(ctx, `+ SELECT wait_event_type = 'Lock'+ FROM pg_stat_activity+ WHERE pid = $1`, migrationPID).Scan(&waiting)+ return err == nil && waiting+ }, 5*time.Second, 10*time.Millisecond)+ var hasPlanTableLock bool+ require.NoError(t, container.GetDB().QueryRowContext(ctx, `+ SELECT EXISTS (+ SELECT 1+ FROM pg_locks+ WHERE pid = $1+ AND locktype = 'relation'+ AND relation = 'plan'::regclass+ AND mode = 'ShareLock'+ )`, migrationPID).Scan(&hasPlanTableLock))+ assert.False(t, hasPlanTableLock)+ require.NoError(t, createPlanTx.Commit())+ require.NoError(t, <-discoveryDone)++ rows, err := container.GetDB().QueryContext(ctx, `+ SELECT+ plan.name,+ issue.creator,+ issue.name,+ issue.description,+ issue.status,+ issue.type,+ COALESCE((issue.payload->>'draft')::boolean, FALSE),+ COALESCE(jsonb_array_length(issue.payload->'labels'), 0),+ issue.created_at = $1,+ issue.ts_vector::TEXT+ FROM issue+ JOIN plan ON plan.project = issue.project AND plan.id = issue.plan_id+ WHERE COALESCE((issue.payload->>'draft')::boolean, FALSE)+ ORDER BY plan.project, plan.id+ `, migrationTime)+ require.NoError(t, err)+ defer rows.Close()++ type draft struct {+ creator string+ name string+ description string+ status string+ issueType string+ draft bool+ labelCount int+ createdAtMigration bool+ searchVector string+ }+ got := make(map[string]draft)+ for rows.Next() {+ var planName string+ var value draft+ require.NoError(t, rows.Scan(+ &planName,+ &value.creator,+ &value.name,+ &value.description,+ &value.status,+ &value.issueType,+ &value.draft,+ &value.labelCount,+ &value.createdAtMigration,+ &value.searchVector,+ ))+ got[planName] = value+ }+ require.NoError(t, rows.Err())++ require.Equal(t, map[string]draft{+ "recent change": {+ creator: "[email protected]", name: "recent change", description: "change description",+ status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,+ searchVector: "'change':2,3 'description':4 'recent':1",+ },+ "boundary create": {+ creator: "[email protected]", name: "boundary create", description: "create description",+ status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,+ searchVector: "'boundary':1 'create':2,3 'description':4",+ },+ "other project change": {+ creator: "[email protected]", name: "other project change", description: "other description",+ status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,+ searchVector: "'change':3 'description':5 'other':1,4 'project':2",+ },+ "late commit": {+ creator: "[email protected]", name: "late commit", description: "late description",+ status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,+ searchVector: "'commit':2 'description':4 'late':1,3",+ },+ }, got)++ _, err = container.GetDB().ExecContext(ctx, `+ INSERT INTO plan (id, creator, created_at, project, name, description, config)+ SELECT id, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'paged plan', '',+ '{"specs":[{"changeDatabaseConfig":{}}]}'+ FROM generate_series(1000, 1100) AS id`)+ require.NoError(t, err)+ require.NoError(t, migrate3_21_1At(ctx, conn, migrationTime))+ var pagedIssueCount int+ require.NoError(t, container.GetDB().QueryRowContext(ctx, `+ SELECT COUNT(*)+ FROM issue+ WHERE project = 'project-a' AND plan_id BETWEEN 1000 AND 1100`).Scan(&pagedIssueCount))+ require.Equal(t, 101, pagedIssueCount)++ _, err = container.GetDB().ExecContext(ctx, `+ INSERT INTO plan (id, creator, created_at, project, name, description, config)+ VALUES (12, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'racing rollout', '',+ '{"specs":[{"changeDatabaseConfig":{}}]}')`)+ require.NoError(t, err)++ rolloutTx, err := container.GetDB().BeginTx(ctx, nil)+ require.NoError(t, err)+ defer rolloutTx.Rollback()+ require.NoError(t, store.AcquireAdvisoryXactLockWithStringKey(+ ctx,+ rolloutTx,+ store.AdvisoryLockKeyPlanIssueRollout,+ "project-a/12",+ ))++ raceConn, err := container.GetDB().Conn(ctx)+ require.NoError(t, err)+ defer raceConn.Close()+ migrationDone := make(chan error, 1)+ go func() {+ migrationDone <- migrate3_21_1At(ctx, raceConn, migrationTime)+ }()+ require.Eventually(t, func() bool {+ var waiting bool+ err := container.GetDB().QueryRowContext(ctx, `+ SELECT EXISTS (+ SELECT 1+ FROM pg_locks+ WHERE locktype = 'advisory'+ AND classid = $1+ AND NOT granted+ )`, store.AdvisoryLockKeyPlanIssueRollout).Scan(&waiting)+ return err == nil && waiting+ }, 5*time.Second, 10*time.Millisecond)+ _, err = rolloutTx.ExecContext(ctx, `+ UPDATE plan+ SET config = jsonb_set(config, '{hasRollout}', 'true'::jsonb, true)+ WHERE project = 'project-a' AND id = 12`)+ require.NoError(t, err)+ require.NoError(t, rolloutTx.Commit())+ require.NoError(t, <-migrationDone)++ var racingIssueCount int+ require.NoError(t, container.GetDB().QueryRowContext(ctx, `+ SELECT COUNT(*) FROM issue WHERE project = 'project-a' AND plan_id = 12`).Scan(&racingIssueCount))+ require.Zero(t, racingIssueCount)+}+func TestMigration3_17_15_DedupeReadOnlyDataSources(t *testing.T) {ctx := context.Background()container := testcontainer.GetTestPgContainer(ctx, t)