Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
perf(tests): drop the container-backed tests in migrator and starrocks (#21341)
Danny Xu6e9709695ecc
diff --git a/AGENTS.md b/AGENTS.mdindex 7bff084844..a3f99eb86d 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -45,10 +45,15 @@ fidelity belong in [omni](https://github.com/bytebase/omni), so never write`TestFooForMySQL` beside `TestFooForPostgreSQL`. A test earns a Bytebase serverboot only if it needs a background runner, a real rollout, or the audit trail —those live in `backend/tests`, the only package that may start one; everything-else asserts in `backend/api/v1` or `backend/store`. Packages needing a-real database share one container via `TestMain` and give each test its own-`CREATE DATABASE`, never a container per test — a Postgres container costs 4.3s-against a 0.7s server boot.+else asserts in `backend/api/v1` or `backend/store`.++Only `backend/store` and `backend/tests` may run a real metadata Postgres.+Everywhere else, extract the decide-and-convert half of a handler as functions+over plain data, and where state is genuinely read declare an interface beside+the handler and fake it, as `backend/api/mcp` does with `serverStore` — never+one interface over the whole store, which has 305 methods.+Every fake needs a contract test run against the real store too, or it drifts+into testing itself.## Development Workflowdiff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 30cee0c68f..11115a18c8 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -15,8 +15,6 @@ import ("net""os""path/filepath"- "runtime"- "strings""testing""time"@@ -463,132 +461,6 @@ func GetTestOracleContainer(ctx context.Context, t testing.TB) *Container {return container}-// GetStarRocksContainer creates a StarRocks (allin1) container for testing. The allin1-// image bundles the FE and BE; information_schema is served by the BE, so this waits until-// the backend actually serves queries rather than only the FE port being open.-//-// NOTE: requires an amd64 host — the StarRocks BE has no working arm64 build (it fails to-// come alive under emulation). bytebase CI runs on amd64; locally, run without -short on an-// amd64 machine, or the test is skipped via testing.Short().-func GetStarRocksContainer(ctx context.Context) (retC *Container, retErr error) {- req := testcontainers.ContainerRequest{- Image: "starrocks/allin1-ubuntu:3.4.10",- ExposedPorts: []string{"9030/tcp"},- WaitingFor: wait.ForListeningPort("9030/tcp").WithStartupTimeout(5 * time.Minute),- }- c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{- ContainerRequest: req,- Started: true,- })- if err != nil {- return nil, err- }- host, err := c.Host(ctx)- if err != nil {- return nil, err- }- port, err := c.MappedPort(ctx, "9030/tcp")- if err != nil {- return nil, err- }- dsn := fmt.Sprintf("root@tcp(%s:%s)/?multiStatements=true", host, port.Port())- db, err := sql.Open("mysql", dsn)- if err != nil {- return nil, err- }- defer func() {- if retErr != nil {- db.Close()- }- }()- if err := waitStarRocksReady(ctx, db); err != nil {- return nil, err- }- return &Container{- container: c,- host: host,- port: port.Port(),- db: db,- }, nil-}--// GetTestStarRocksContainer creates a StarRocks container and fails the test on error. It-// skips on non-amd64 hosts: the StarRocks all-in-one BE has no working arm64 build (it never-// comes alive under emulation), so the readiness wait would otherwise time out.-func GetTestStarRocksContainer(ctx context.Context, t testing.TB) *Container {- t.Helper()- if runtime.GOARCH != "amd64" {- t.Skipf("StarRocks requires an amd64 host; the all-in-one BE has no working arm64 build (GOARCH=%s)", runtime.GOARCH)- }- container, err := GetStarRocksContainer(ctx)- if err != nil {- t.Fatalf("failed to create StarRocks container: %v", err)- }- return container-}--// waitStarRocksReady blocks until at least one StarRocks backend is registered and alive,-// which is required before tablet-allocating DDL (CREATE TABLE) succeeds. The check is-// read-only on purpose: issuing DDL during allin1's first-boot disrupts backend-// registration, so poll SHOW BACKENDS rather than probing with CREATE TABLE.-func waitStarRocksReady(ctx context.Context, db *sql.DB) error {- ticker := time.NewTicker(3 * time.Second)- defer ticker.Stop()- timeout := time.After(10 * time.Minute)- for {- select {- case <-ticker.C:- if starRocksBackendAlive(ctx, db) {- return nil- }- case <-timeout:- return errors.Errorf("StarRocks backend did not become ready")- }- }-}--// starRocksBackendAlive reports whether SHOW BACKENDS lists a backend with Alive=true.-func starRocksBackendAlive(ctx context.Context, db *sql.DB) bool {- rows, err := db.QueryContext(ctx, "SHOW BACKENDS")- if err != nil {- return false- }- defer rows.Close()- cols, err := rows.Columns()- if err != nil {- return false- }- aliveIdx := -1- for i, c := range cols {- if strings.EqualFold(c, "Alive") {- aliveIdx = i- break- }- }- if aliveIdx < 0 {- return false- }- alive := false- for rows.Next() {- vals := make([]sql.RawBytes, len(cols))- ptrs := make([]any, len(cols))- for i := range vals {- ptrs[i] = &vals[i]- }- if err := rows.Scan(ptrs...); err != nil {- return false- }- if strings.EqualFold(string(vals[aliveIdx]), "true") {- alive = true- break- }- }- if err := rows.Err(); err != nil {- return false- }- return alive-}-// GetMSSQLContainer creates a Microsoft SQL Server container for testingfunc GetMSSQLContainer(ctx context.Context) (retC *Container, retErr error) {req := testcontainers.ContainerRequest{diff --git a/backend/migrator/changelog_utf8_test.go b/backend/migrator/changelog_utf8_test.godeleted file mode 100644index b246de68bd..0000000000--- a/backend/migrator/changelog_utf8_test.go+++ /dev/null@@ -1,112 +0,0 @@-package migrator--import (- "context"- "strings"- "testing"- "unicode/utf8"-- "github.com/stretchr/testify/assert"- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"-)--// TestListChangelogsWithInvalidUTF8 verifies that reading sync_history.raw_dump-// containing invalid UTF-8 bytes does not crash. Regression test for SQLSTATE 22021-// caused by LEFT(raw_dump, N) triggering PostgreSQL UTF-8 validation.-//-// The customer reported: "failed to ensure baseline changelog: failed to check for-// existing changelogs: failed to query: ERROR: invalid byte sequence for encoding-// "UTF8": 0xe4 0xb8 (SQLSTATE 22021)"-func TestListChangelogsWithInvalidUTF8(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- // Create minimal schema with raw_dump as bytea initially.- // We insert the invalid bytes as bytea (no encoding validation), then flip- // the column type to text via pg_attribute. This simulates how corrupted data- // ends up in production (e.g., via server-side INSERT INTO...SELECT during- // the 3.2 migration, or older PostgreSQL versions with laxer validation).- setup := `- CREATE TABLE sync_history (- id bigserial PRIMARY KEY,- created_at timestamptz NOT NULL DEFAULT now(),- instance text NOT NULL,- db_name text NOT NULL,- metadata json NOT NULL DEFAULT '{}',- raw_dump bytea NOT NULL DEFAULT ''::bytea- );-- CREATE TABLE changelog (- id bigserial PRIMARY KEY,- created_at timestamptz NOT NULL DEFAULT now(),- instance text NOT NULL,- db_name text NOT NULL,- status text NOT NULL,- sync_history_id bigint REFERENCES sync_history(id),- payload jsonb NOT NULL DEFAULT '{}'- );-- CREATE TABLE task (- id serial PRIMARY KEY,- plan_id bigint NOT NULL- );-- CREATE TABLE plan (- id bigserial PRIMARY KEY,- name text NOT NULL- );-- -- Insert invalid UTF-8 as bytea (no encoding validation).- -- 0xe4 0xb8 is a truncated 3-byte Chinese UTF-8 character (e.g. 中 = 0xe4 0xb8 0xad).- INSERT INTO sync_history (instance, db_name, metadata, raw_dump)- VALUES ('test-instance', 'test-db', '{}', '\xE4B8'::bytea);-- -- Flip column type from bytea to text via system catalog.- -- bytea and text share the same varlena storage format, so the raw bytes- -- are preserved but now treated as text — bypassing UTF-8 validation.- UPDATE pg_attribute- SET atttypid = 'text'::regtype- WHERE attrelid = 'sync_history'::regclass- AND attname = 'raw_dump';-- INSERT INTO changelog (instance, db_name, status, sync_history_id, payload)- VALUES ('test-instance', 'test-db', 'DONE', 1, '{}');- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- // Reproduce the bug: LEFT() on text containing invalid UTF-8 triggers SQLSTATE 22021.- // This mirrors the query that ListChangelogs generates when ShowFull is false.- var schema string- err = db.QueryRowContext(ctx,- `SELECT COALESCE(LEFT(sh_cur.raw_dump, 512), '')- FROM changelog- LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id- WHERE changelog.id = $1`,- 1,- ).Scan(&schema)- assert.Error(t, err, "LEFT() on invalid UTF-8 text must fail — confirming the bug exists")- if err != nil {- assert.Contains(t, err.Error(), "22021", "error must be SQLSTATE 22021 (invalid byte sequence)")- }-- // Verify the fix: reading the raw column without LEFT() succeeds because- // a plain column reference does not trigger PostgreSQL's UTF-8 string-function validation.- err = db.QueryRowContext(ctx,- `SELECT COALESCE(sh_cur.raw_dump, '')- FROM changelog- LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id- WHERE changelog.id = $1`,- 1,- ).Scan(&schema)- require.NoError(t, err, "reading raw_dump without LEFT() must succeed even with invalid UTF-8")-- // After Go-side sanitization, the result must be valid UTF-8.- sanitized := strings.ToValidUTF8(schema, "")- assert.True(t, utf8.ValidString(sanitized), "sanitized schema must be valid UTF-8")-}diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex ace3c59cf5..74fd26e243 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -1,21 +1,10 @@package migratorimport (- "context"- "crypto/sha256"- "encoding/hex"- "fmt"- "strings""testing"- "time""github.com/blang/semver/v4"- "github.com/stretchr/testify/assert""github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"- _ "github.com/bytebase/bytebase/backend/plugin/db/pg"- "github.com/bytebase/bytebase/backend/store")func TestLatestVersion(t *testing.T) {@@ -25,81 +14,6 @@ func TestLatestVersion(t *testing.T) {require.Equal(t, "migration/3.23/0003##identity_provider_list_permission.sql", files[len(files)-1].path)}-func TestMigration3_23_1_IssueCommentThreads(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- // Minimal pre-3.23.1 shape of issue_comment.- setup := `- CREATE TABLE issue_comment (- resource_id text NOT NULL,- project text NOT NULL DEFAULT 'p1',- issue_id integer NOT NULL DEFAULT 101,- payload jsonb NOT NULL DEFAULT '{}'- );- CREATE UNIQUE INDEX idx_issue_comment_unique_resource_id ON issue_comment(resource_id);- ALTER TABLE issue_comment ADD PRIMARY KEY (resource_id);- INSERT INTO issue_comment (resource_id, payload) VALUES- ('comment-only', '{"comment": "hello"}'),- ('empty-comment', '{"comment": ""}'),- ('empty-object', '{}'),- ('hybrid', '{"comment": "approved", "approval": {}}'),- ('event-only', '{"issueUpdate": {"toTitle": "t"}}'),- ('legacy-event', '{"planSpecUpdate": {}}'),- ('numeric-comment', '{"comment": 123}'),- ('null-comment', '{"comment": null}'),- ('scalar', '"comment"');- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- migration, err := migrationFS.ReadFile("migration/3.23/0001##issue_comment_thread.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(migration))- require.NoError(t, err)-- // Legacy comments have no statement anchor, so none becomes a thread root.- var rowCount, stateless int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT count(*), count(*) FILTER (WHERE parent_id IS NULL AND thread_state IS NULL)- FROM issue_comment- `).Scan(&rowCount, &stateless))- require.Equal(t, 9, rowCount)- require.Equal(t, rowCount, stateless, "the migration must not classify any legacy row as a thread root")-- var indexes int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT count(*) FROM pg_indexes- WHERE indexname IN ('idx_issue_comment_parent_id', 'idx_issue_comment_open_thread')- `).Scan(&indexes))- require.Equal(t, 2, indexes)-- // On upgraded databases the explicit-column FK binds the pre-existing- // duplicate unique index, not the primary key (this fixture recreates- // that index order). Accepted for now; the deferred cleanup that drops- // idx_issue_comment_unique_resource_id must swap the FK dependency and- // flip this assertion to issue_comment_pkey.- var referencedIndex string- require.NoError(t, db.QueryRowContext(ctx, `- SELECT referenced_index.relname- FROM pg_constraint- JOIN pg_class AS referenced_index ON referenced_index.oid = pg_constraint.conindid- WHERE pg_constraint.conname = 'issue_comment_parent_id_fkey'- AND pg_constraint.conrelid = 'issue_comment'::regclass- `).Scan(&referencedIndex))- require.Equal(t, "idx_issue_comment_unique_resource_id", referencedIndex)-- _, err = db.ExecContext(ctx, `INSERT INTO issue_comment (resource_id, payload, parent_id) VALUES ('reply', '{"comment": "re"}', 'comment-only')`)- require.NoError(t, err)- _, err = db.ExecContext(ctx, `INSERT INTO issue_comment (resource_id, payload, parent_id) VALUES ('orphan', '{"comment": "re"}', 'missing')`)- require.Error(t, err, "parent_id must reference an existing comment")- _, err = db.ExecContext(ctx, `UPDATE issue_comment SET thread_state = 'INVALID' WHERE resource_id = 'comment-only'`)- require.Error(t, err, "thread_state must reject unknown values")-}-func TestVersionUnique(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)@@ -114,1243 +28,3 @@ func TestVersionUnique(t *testing.T) {versions[file.version.String()] = struct{}{}}}--func TestMigration3_23_2BackfillsLastPlanEditorApproval(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })- db := container.GetDB()- _, err := db.ExecContext(ctx, `- CREATE TABLE project (- resource_id text PRIMARY KEY,- deleted boolean NOT NULL DEFAULT FALSE,- setting jsonb NOT NULL DEFAULT '{}'- );- CREATE TABLE plan (id bigint NOT NULL, project text NOT NULL, creator text NOT NULL);- INSERT INTO project (resource_id, deleted) VALUES ('active', false), ('deleted', true);- `)- require.NoError(t, err)- statement, err := migrationFS.ReadFile("migration/3.23/0002##plan_last_editor_approval.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- for _, projectID := range []string{"active", "deleted"} {- var enabled bool- err := db.QueryRowContext(ctx, `- SELECT (setting->>'allowLastPlanEditorApproval')::boolean- FROM project- WHERE resource_id = $1`, projectID).Scan(&enabled)- require.NoError(t, err)- require.True(t, enabled)- }- _, err = db.ExecContext(ctx, "UPDATE project SET deleted = false WHERE resource_id = 'deleted'")- require.NoError(t, err)- var restoredEnabled bool- err = db.QueryRowContext(ctx, `- SELECT (setting->>'allowLastPlanEditorApproval')::boolean- FROM project- WHERE resource_id = 'deleted' AND NOT deleted`).Scan(&restoredEnabled)- require.NoError(t, err)- require.True(t, restoredEnabled)- var nullable string- require.NoError(t, db.QueryRowContext(ctx, `- SELECT is_nullable- FROM information_schema.columns- WHERE table_name = 'plan' AND column_name = 'last_plan_editor'`).Scan(&nullable))- require.Equal(t, "YES", nullable)-}--func TestMigration3_22_12MigratesSampleProjectInstance(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })- db := container.GetDB()- _, err := db.ExecContext(ctx, `- CREATE TABLE workspace (resource_id text PRIMARY KEY);- INSERT INTO workspace (resource_id) VALUES ('workspace-a');- `)- require.NoError(t, err)- statement, err := migrationFS.ReadFile("migration/3.22/0011##sample_project_instance.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)- createdAt := time.Date(2026, time.August, 1, 12, 0, 0, 0, time.UTC)- expiresAt := createdAt.Add(7 * 24 * time.Hour)- _, err = db.ExecContext(ctx, `- INSERT INTO sample_project_instance (- workspace, project, instance, db_name, role_name, replica_id,- created_at, expires_at- ) VALUES ('workspace-a', 'project-a', 'sample-a', 'db-a', 'role-a', 'replica-a', $1, $2)- `, createdAt, expiresAt)- require.NoError(t, err)- statement, err = migrationFS.ReadFile("migration/3.22/0013##sample_instance_setup.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- var instance string- var activatedAt time.Time- err = db.QueryRowContext(ctx, `- SELECT activated_at, payload->>'instanceId'- FROM sample_instance_setup- WHERE workspace = 'workspace-a'- `).Scan(&activatedAt, &instance)- require.NoError(t, err)- require.Equal(t, createdAt, activatedAt)- require.Equal(t, "sample-a", instance)-- var oldTable *string- require.NoError(t, db.QueryRowContext(ctx, "SELECT to_regclass('sample_project_instance')").Scan(&oldTable))- require.Nil(t, oldTable)-}--// TestMigration3_16_2_TaskRunLogDuplicateTimestamps verifies that the 3.16.2-// id-column cleanup succeeds on legacy task_run_log data holding duplicate-// (task_run_id, created_at) pairs, which were legal under the old id primary-// key. Regression test for BYT-10035.-func TestMigration3_16_2_TaskRunLogDuplicateTimestamps(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- // Minimal pre-3.16.2 shapes for every table the migration touches.- setup := `- CREATE TABLE project (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);- CREATE TABLE instance (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);- CREATE TABLE db (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, name TEXT NOT NULL);- CREATE TABLE setting (id BIGSERIAL PRIMARY KEY, name TEXT NOT NULL);- CREATE TABLE policy (id BIGSERIAL PRIMARY KEY, resource_type TEXT NOT NULL, resource TEXT NOT NULL, type TEXT NOT NULL);- CREATE TABLE idp (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);- CREATE TABLE role (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);- CREATE TABLE db_schema (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, db_name TEXT NOT NULL);- CREATE TABLE db_group (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, resource_id TEXT NOT NULL);- CREATE TABLE release (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, train TEXT NOT NULL, iteration INT NOT NULL);- CREATE TABLE task_run_log (- id BIGSERIAL PRIMARY KEY,- task_run_id INTEGER NOT NULL,- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),- payload JSONB NOT NULL DEFAULT '{}'- );- CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);-- INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),- (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.16/0002##drop_unused_id_columns.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- var rowCount int- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_run_log`).Scan(&rowCount))- require.Equal(t, 3, rowCount)-- var hasPkey bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)- `).Scan(&hasPkey))- require.False(t, hasPkey)-- var hasIDColumn bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'task_run_log' AND column_name = 'id')- `).Scan(&hasIDColumn))- require.False(t, hasIDColumn)-- var hasIndex bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_task_run_id_created_at')- `).Scan(&hasIndex))- require.True(t, hasIndex)-- var hasOldIndex bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_task_run_id')- `).Scan(&hasOldIndex))- require.False(t, hasOldIndex)-}--// TestMigration3_17_1_TaskRunLogDuplicateTimestamps verifies that the 3.17.1-// project-scoping migration accepts duplicate (task_run_id, created_at) pairs:-// it must backfill project and index the table without imposing uniqueness.-// Regression test for BYT-10035.-func TestMigration3_17_1_TaskRunLogDuplicateTimestamps(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- // Minimal post-3.16.2 shapes for the plan-chain tables the migration touches;- // task_run_log has no id column and no primary key at this point.- setup := `- CREATE TABLE project (resource_id TEXT PRIMARY KEY);- CREATE TABLE plan (id BIGINT PRIMARY KEY, project TEXT NOT NULL);- CREATE TABLE issue (id BIGINT PRIMARY KEY, project TEXT NOT NULL, plan_id BIGINT);- CREATE TABLE task (id INT PRIMARY KEY, plan_id BIGINT NOT NULL, environment TEXT NOT NULL);- CREATE TABLE task_run (id INT PRIMARY KEY, task_id INT NOT NULL, attempt INT NOT NULL);- CREATE TABLE plan_check_run (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);- CREATE TABLE plan_webhook_delivery (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);- CREATE TABLE task_run_log (- task_run_id INTEGER NOT NULL,- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),- payload JSONB NOT NULL DEFAULT '{}'- );- CREATE INDEX idx_task_run_log_task_run_id_created_at ON task_run_log(task_run_id, created_at);-- INSERT INTO project VALUES ('proj-a');- INSERT INTO plan VALUES (1, 'proj-a');- INSERT INTO task VALUES (10, 1, 'environments/prod');- INSERT INTO task_run VALUES (100, 10, 1);- INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),- (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.17/0001##add_project_to_plan_chain.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- var backfilledCount int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT COUNT(*) FROM task_run_log WHERE project = 'proj-a'- `).Scan(&backfilledCount))- require.Equal(t, 3, backfilledCount)-- var hasPkey bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)- `).Scan(&hasPkey))- require.False(t, hasPkey)-- var hasIndex bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')- `).Scan(&hasIndex))- require.True(t, hasIndex)-- var oldIndexCount int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public' AND indexname IN ('idx_task_run_log_task_run_id', 'idx_task_run_log_task_run_id_created_at')- `).Scan(&oldIndexCount))- require.Zero(t, oldIndexCount)-}--// TestMigration3_22_4_DropTaskRunLogPkey verifies that the pkey drop converges-// every historical shape — the 2-column PK left by upgrading through the-// original 3.16.2, the 3-column PK of fresh installs, and the no-PK state left-// by a release-branch backport of the 3.16.2 fix — is idempotent, and that-// same-microsecond entries insert afterwards instead of colliding.-func TestMigration3_22_4_DropTaskRunLogPkey(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- statement, err := migrationFS.ReadFile("migration/3.22/0004##drop_task_run_log_pkey.sql")- require.NoError(t, err)-- const tableDDL = `- CREATE TABLE task_run_log (- project TEXT NOT NULL,- task_run_id INTEGER NOT NULL,- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),- payload JSONB NOT NULL DEFAULT '{}'`-- scenarios := []struct {- name string- setup string- }{- {- "2-col pkey from the original 3.16.2",- tableDDL + `, PRIMARY KEY (task_run_id, created_at));- CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);`,- },- {- "3-col pkey from a fresh install",- tableDDL + `, PRIMARY KEY (project, task_run_id, created_at));`,- },- {- "no pkey from a release-branch backport of the 3.16.2 fix",- tableDDL + `);- CREATE INDEX idx_task_run_log_task_run_id_created_at ON task_run_log(task_run_id, created_at);`,- },- }- for _, scenario := range scenarios {- _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS task_run_log;`+scenario.setup)- require.NoError(t, err, scenario.name)-- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err, scenario.name)- // Re-running must be a no-op.- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err, scenario.name)-- var hasPkey bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)- `).Scan(&hasPkey))- require.False(t, hasPkey, scenario.name)-- var hasIndex bool- require.NoError(t, db.QueryRowContext(ctx, `- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')- `).Scan(&hasIndex))- require.True(t, hasIndex, scenario.name)-- var oldIndexCount int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public' AND indexname IN ('idx_task_run_log_task_run_id', 'idx_task_run_log_task_run_id_created_at')- `).Scan(&oldIndexCount))- require.Zero(t, oldIndexCount, scenario.name)-- _, err = db.ExecContext(ctx, `- INSERT INTO task_run_log (project, task_run_id, created_at) VALUES- ('proj-a', 100, '2026-01-01 00:00:00.000001+00'),- ('proj-a', 100, '2026-01-01 00:00:00.000001+00')`)- require.NoError(t, err, "same-microsecond entries must both insert")- }-}--func TestMigration3_21_2_MigrateInstanceSyncDatabases(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- setup := `- CREATE TABLE instance (- resource_id TEXT PRIMARY KEY,- metadata JSONB NOT NULL DEFAULT '{}'- );-- INSERT INTO instance (resource_id, metadata) VALUES- (- 'instance-with-selected-databases',- '{"engine":"POSTGRES","syncDatabases":["db1","db2"]}'- ),- (- 'instance-with-empty-databases',- '{"engine":"POSTGRES","syncDatabases":[]}'- ),- (- 'instance-with-new-shape',- '{"engine":"POSTGRES","syncDatabases":{"databases":["db3"]}}'- ),- (- 'instance-without-sync-databases',- '{"engine":"POSTGRES"}'- );- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.21/0002##migrate_instance_sync_databases.sql")- require.NoError(t, err)-- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- getMetadata := func(resourceID string) string {- t.Helper()- var metadata string- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)- require.NoError(t, err)- return metadata- }-- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":["db1","db2"]}}`, getMetadata("instance-with-selected-databases"))- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":[]}}`, getMetadata("instance-with-empty-databases"))- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":["db3"]}}`, getMetadata("instance-with-new-shape"))- require.JSONEq(t, `{"engine":"POSTGRES"}`, getMetadata("instance-without-sync-databases"))-}--func TestMigration3_22_2_AddInstanceProject(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()- _, err := db.ExecContext(ctx, `- CREATE TABLE project (- resource_id TEXT PRIMARY KEY- );- CREATE TABLE instance (- resource_id TEXT PRIMARY KEY,- metadata JSONB NOT NULL DEFAULT '{}'- );- INSERT INTO instance (resource_id) VALUES ('legacy-workspace-instance');- `)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.22/0002##add_instance_project.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- var projectID *string- require.NoError(t, db.QueryRowContext(ctx, `- SELECT project FROM instance WHERE resource_id = 'legacy-workspace-instance'- `).Scan(&projectID))- require.Nil(t, projectID)-- var indexDefinition string- require.NoError(t, db.QueryRowContext(ctx, `- SELECT indexdef- FROM pg_indexes- WHERE schemaname = 'public' AND indexname = 'idx_instance_project'- `).Scan(&indexDefinition))- require.Contains(t, indexDefinition, "WHERE (project IS NOT NULL)")-}--func TestMigration3_21_3_MigrateGCPDataSourceFields(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- setup := `- CREATE TABLE instance (- resource_id TEXT PRIMARY KEY,- metadata JSONB NOT NULL DEFAULT '{}'- );-- INSERT INTO instance (resource_id, metadata) VALUES- (- 'spanner-legacy',- '{"engine":"SPANNER","dataSources":[{"id":"admin","host":"projects/my-proj/instances/my-inst"}]}'- ),- (- 'spanner-multi-data-source',- '{"engine":"SPANNER","dataSources":[{"id":"admin","host":"projects/my-proj/instances/my-inst"},{"id":"ro-1","type":"READ_ONLY","host":"projects/my-proj/instances/my-inst"},{"id":"ro-2","type":"READ_ONLY","host":"projects/my-proj/instances/my-inst"}]}'- ),- (- 'spanner-new-shape',- '{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst","host":"spanner-nonprod.p.googleapis.com"}]}'- ),- (- 'spanner-no-data-sources',- '{"engine":"SPANNER","dataSources":[]}'- ),- (- 'bigquery-legacy',- '{"engine":"BIGQUERY","dataSources":[{"id":"admin","host":"my-proj"}]}'- ),- (- 'bigquery-new-shape',- '{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj","host":"bigquery-nonprod.p.googleapis.com"}]}'- ),- (- 'postgres-untouched',- '{"engine":"POSTGRES","dataSources":[{"id":"admin","host":"localhost","port":"5432"}]}'- );- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.21/0003##migrate_gcp_data_source_fields.sql")- require.NoError(t, err)-- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- getMetadata := func(resourceID string) string {- t.Helper()- var metadata string- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)- require.NoError(t, err)- return metadata- }-- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst"}]}`, getMetadata("spanner-legacy"))- // JSONEq treats arrays as ordered: this also asserts the data source order is preserved.- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst"},{"id":"ro-1","type":"READ_ONLY","projectId":"my-proj","instanceId":"my-inst"},{"id":"ro-2","type":"READ_ONLY","projectId":"my-proj","instanceId":"my-inst"}]}`, getMetadata("spanner-multi-data-source"))- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst","host":"spanner-nonprod.p.googleapis.com"}]}`, getMetadata("spanner-new-shape"))- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[]}`, getMetadata("spanner-no-data-sources"))- require.JSONEq(t, `{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj"}]}`, getMetadata("bigquery-legacy"))- require.JSONEq(t, `{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj","host":"bigquery-nonprod.p.googleapis.com"}]}`, getMetadata("bigquery-new-shape"))- require.JSONEq(t, `{"engine":"POSTGRES","dataSources":[{"id":"admin","host":"localhost","port":"5432"}]}`, getMetadata("postgres-untouched"))-}--func TestMigration3_21_1_BackfillUIPlanDraftIssues(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- tx, err := container.GetDB().BeginTx(ctx, nil)- require.NoError(t, err)- defer tx.Rollback()- var migrationTime time.Time- require.NoError(t, tx.QueryRowContext(ctx, `SELECT CURRENT_TIMESTAMP`).Scan(&migrationTime))-- _, err = tx.ExecContext(ctx, `- CREATE TABLE project (- resource_id TEXT PRIMARY KEY- );- CREATE TABLE plan (- id BIGINT NOT NULL,- deleted BOOLEAN NOT NULL DEFAULT FALSE,- creator TEXT NOT NULL,- created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,- project TEXT NOT NULL,- name TEXT NOT NULL,- description TEXT NOT NULL,- config JSONB NOT NULL,- PRIMARY KEY (project, id)- );- CREATE TABLE issue (- id BIGINT NOT NULL,- creator TEXT NOT NULL,- created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,- project TEXT NOT NULL,- plan_id BIGINT,- name TEXT NOT NULL,- status TEXT NOT NULL,- type TEXT NOT NULL,- description TEXT NOT NULL DEFAULT '',- payload JSONB NOT NULL DEFAULT '{}',- ts_vector TSVECTOR,- PRIMARY KEY (project, id),- UNIQUE (project, plan_id)- );-- INSERT INTO project (resource_id) VALUES ('project-a'), ('project-b');- INSERT INTO plan (id, creator, created_at, project, name, description, config) VALUES- (1, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '29 days', 'project-a', 'recent change', 'change description',- '{"specs":[{"changeDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),- (2, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days', 'project-a', 'boundary create', 'create description',- '{"specs":[{"createDatabaseConfig":{}}]}'),- (3, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'release GitOps', '',- '{"specs":[{"changeDatabaseConfig":{"release":"projects/project-a/releases/release-a"}}]}'),- (4, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'export', '',- '{"specs":[{"exportDataConfig":{}}]}'),- (5, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'mixed', '',- '{"specs":[{"createDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),- (6, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'deleted', '',- '{"specs":[{"changeDatabaseConfig":{}}]}'),- (7, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days 1 microsecond', 'project-a', 'old', '',- '{"specs":[{"changeDatabaseConfig":{}}]}'),- (8, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'linked', '',- '{"specs":[{"changeDatabaseConfig":{}}]}'),- (9, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-b', 'other project change', 'other description',- '{"specs":[{"changeDatabaseConfig":{}}]}'),- (10, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'rolled out', '',- '{"specs":[{"changeDatabaseConfig":{}}],"hasRollout":true}');- UPDATE plan SET deleted = TRUE WHERE project = 'project-a' AND id = 6;- INSERT INTO issue (id, creator, project, plan_id, name, status, type, payload)- VALUES (150, '[email protected]', 'project-a', 8, 'existing issue', 'OPEN', 'DATABASE_CHANGE', '{}');- `)- require.NoError(t, err)- require.NoError(t, tx.Commit())-- conn, err := container.GetDB().Conn(ctx)- require.NoError(t, err)- defer conn.Close()- var migrationPID int- require.NoError(t, conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&migrationPID))-- createPlanTx, err := container.GetDB().BeginTx(ctx, nil)- require.NoError(t, err)- defer createPlanTx.Rollback()- var lockedProjectID string- require.NoError(t, createPlanTx.QueryRowContext(ctx, `- SELECT resource_id FROM project WHERE resource_id = 'project-a' FOR UPDATE`).Scan(&lockedProjectID))- _, err = createPlanTx.ExecContext(ctx, `- INSERT INTO plan (id, creator, created_at, project, name, description, config)- VALUES (11, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'late commit', 'late description',- '{"specs":[{"changeDatabaseConfig":{}}]}')`)- require.NoError(t, err)-- discoveryDone := make(chan error, 1)- go func() {- discoveryDone <- migrate3_21_1At(ctx, conn, migrationTime)- }()- require.Eventually(t, func() bool {- var waiting bool- err := container.GetDB().QueryRowContext(ctx, `- SELECT wait_event_type = 'Lock'- FROM pg_stat_activity- WHERE pid = $1`, migrationPID).Scan(&waiting)- return err == nil && waiting- }, 5*time.Second, 10*time.Millisecond)- var hasPlanTableLock bool- require.NoError(t, container.GetDB().QueryRowContext(ctx, `- SELECT EXISTS (- SELECT 1- FROM pg_locks- WHERE pid = $1- AND locktype = 'relation'- AND relation = 'plan'::regclass- AND mode = 'ShareLock'- )`, migrationPID).Scan(&hasPlanTableLock))- assert.False(t, hasPlanTableLock)- require.NoError(t, createPlanTx.Commit())- require.NoError(t, <-discoveryDone)-- rows, err := container.GetDB().QueryContext(ctx, `- SELECT- plan.name,- issue.creator,- issue.name,- issue.description,- issue.status,- issue.type,- COALESCE((issue.payload->>'draft')::boolean, FALSE),- COALESCE(jsonb_array_length(issue.payload->'labels'), 0),- issue.created_at = $1,- issue.ts_vector::TEXT- FROM issue- JOIN plan ON plan.project = issue.project AND plan.id = issue.plan_id- WHERE COALESCE((issue.payload->>'draft')::boolean, FALSE)- ORDER BY plan.project, plan.id- `, migrationTime)- require.NoError(t, err)- defer rows.Close()-- type draft struct {- creator string- name string- description string- status string- issueType string- draft bool- labelCount int- createdAtMigration bool- searchVector string- }- got := make(map[string]draft)- for rows.Next() {- var planName string- var value draft- require.NoError(t, rows.Scan(- &planName,- &value.creator,- &value.name,- &value.description,- &value.status,- &value.issueType,- &value.draft,- &value.labelCount,- &value.createdAtMigration,- &value.searchVector,- ))- got[planName] = value- }- require.NoError(t, rows.Err())-- require.Equal(t, map[string]draft{- "recent change": {- creator: "[email protected]", name: "recent change", description: "change description",- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,- searchVector: "'change':2,3 'description':4 'recent':1",- },- "boundary create": {- creator: "[email protected]", name: "boundary create", description: "create description",- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,- searchVector: "'boundary':1 'create':2,3 'description':4",- },- "other project change": {- creator: "[email protected]", name: "other project change", description: "other description",- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,- searchVector: "'change':3 'description':5 'other':1,4 'project':2",- },- "late commit": {- creator: "[email protected]", name: "late commit", description: "late description",- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,- searchVector: "'commit':2 'description':4 'late':1,3",- },- }, got)-- _, err = container.GetDB().ExecContext(ctx, `- INSERT INTO plan (id, creator, created_at, project, name, description, config)- SELECT id, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'paged plan', '',- '{"specs":[{"changeDatabaseConfig":{}}]}'- FROM generate_series(1000, 1100) AS id`)- require.NoError(t, err)- require.NoError(t, migrate3_21_1At(ctx, conn, migrationTime))- var pagedIssueCount int- require.NoError(t, container.GetDB().QueryRowContext(ctx, `- SELECT COUNT(*)- FROM issue- WHERE project = 'project-a' AND plan_id BETWEEN 1000 AND 1100`).Scan(&pagedIssueCount))- require.Equal(t, 101, pagedIssueCount)-- _, err = container.GetDB().ExecContext(ctx, `- INSERT INTO plan (id, creator, created_at, project, name, description, config)- VALUES (12, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'racing rollout', '',- '{"specs":[{"changeDatabaseConfig":{}}]}')`)- require.NoError(t, err)-- rolloutTx, err := container.GetDB().BeginTx(ctx, nil)- require.NoError(t, err)- defer rolloutTx.Rollback()- require.NoError(t, store.AcquireAdvisoryXactLockWithStringKey(- ctx,- rolloutTx,- store.AdvisoryLockKeyPlanIssueRollout,- "project-a/12",- ))-- raceConn, err := container.GetDB().Conn(ctx)- require.NoError(t, err)- defer raceConn.Close()- migrationDone := make(chan error, 1)- go func() {- migrationDone <- migrate3_21_1At(ctx, raceConn, migrationTime)- }()- require.Eventually(t, func() bool {- var waiting bool- err := container.GetDB().QueryRowContext(ctx, `- SELECT EXISTS (- SELECT 1- FROM pg_locks- WHERE locktype = 'advisory'- AND classid = $1- AND NOT granted- )`, store.AdvisoryLockKeyPlanIssueRollout).Scan(&waiting)- return err == nil && waiting- }, 5*time.Second, 10*time.Millisecond)- _, err = rolloutTx.ExecContext(ctx, `- UPDATE plan- SET config = jsonb_set(config, '{hasRollout}', 'true'::jsonb, true)- WHERE project = 'project-a' AND id = 12`)- require.NoError(t, err)- require.NoError(t, rolloutTx.Commit())- require.NoError(t, <-migrationDone)-- var racingIssueCount int- require.NoError(t, container.GetDB().QueryRowContext(ctx, `- SELECT COUNT(*) FROM issue WHERE project = 'project-a' AND plan_id = 12`).Scan(&racingIssueCount))- require.Zero(t, racingIssueCount)-}--func TestMigration3_17_15_DedupeReadOnlyDataSources(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- setup := `- CREATE TABLE instance (- resource_id TEXT PRIMARY KEY,- metadata JSONB NOT NULL DEFAULT '{}'- );-- INSERT INTO instance (resource_id, metadata) VALUES- (- 'instance-with-duplicate-read-only',- '{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"},{"id":"read-only-2","type":"READ_ONLY","username":"readonly-2"}]}'- ),- (- 'instance-with-single-read-only',- '{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}'- ),- (- 'instance-with-non-array-data-sources',- '{"dataSources":{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}}'- ),- (- 'instance-without-data-sources',- '{"engine":"POSTGRES"}'- );- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- statement, err := migrationFS.ReadFile("migration/3.17/0015##dedupe_read_only_data_sources.sql")- require.NoError(t, err)-- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- getMetadata := func(resourceID string) string {- t.Helper()- var metadata string- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)- require.NoError(t, err)- return metadata- }-- require.JSONEq(t, `{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}`, getMetadata("instance-with-duplicate-read-only"))- require.JSONEq(t, `{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}`, getMetadata("instance-with-single-read-only"))- require.JSONEq(t, `{"dataSources":{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}}`, getMetadata("instance-with-non-array-data-sources"))- require.JSONEq(t, `{"engine":"POSTGRES"}`, getMetadata("instance-without-data-sources"))-}--// TestMigration3_7_20_ScalarTaskUpdateTasks verifies that the migration 3.7.20-// UPDATE on issue_comment handles scalar (non-array) taskUpdate.tasks values-// without error. Regression test for "cannot get array length of a scalar".-func TestMigration3_7_20_ScalarTaskUpdateTasks(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()-- // Create minimal schema.- setup := `- CREATE TABLE stage (- id INT PRIMARY KEY,- environment TEXT NOT NULL- );- INSERT INTO stage (id, environment) VALUES (101, 'environments/prod');-- CREATE OR REPLACE FUNCTION update_stage_reference(resource_path text) RETURNS text AS $$- DECLARE- stage_match text;- stage_id int;- environment_id text;- BEGIN- IF resource_path !~ '/stages/[0-9]+' THEN- RETURN resource_path;- END IF;- stage_match := substring(resource_path from '/stages/([0-9]+)');- IF stage_match IS NULL THEN- RETURN resource_path;- END IF;- stage_id := stage_match::int;- SELECT s.environment INTO environment_id FROM stage s WHERE s.id = stage_id;- IF environment_id IS NULL THEN- RETURN resource_path;- END IF;- RETURN regexp_replace(resource_path, '/stages/' || stage_id, '/stages/' || environment_id);- END;- $$ LANGUAGE plpgsql;-- CREATE TABLE issue_comment (- id SERIAL PRIMARY KEY,- payload JSONB NOT NULL- );-- INSERT INTO issue_comment (payload) VALUES- ('{"taskUpdate":{"tasks":["projects/p1/rollouts/1/stages/101/tasks/1"]}}'),- ('{"taskUpdate":{"tasks":"projects/p1/rollouts/1/stages/101/tasks/1"}}'),- ('{"taskUpdate":{"tasks":null}}');- `- _, err := db.ExecContext(ctx, setup)- require.NoError(t, err)-- // Run the exact UPDATE from migration 3.7.20 with the fixed WHERE clause.- migrate := `- UPDATE issue_comment- SET payload = jsonb_set(- payload,- '{taskUpdate,tasks}',- (- SELECT jsonb_agg(update_stage_reference(task_ref))- FROM jsonb_array_elements_text(payload->'taskUpdate'->'tasks') AS task_ref- )- )- WHERE payload->'taskUpdate' IS NOT NULL- AND jsonb_typeof(payload->'taskUpdate'->'tasks') = 'array'- AND CASE WHEN jsonb_typeof(payload->'taskUpdate'->'tasks') = 'array'- THEN jsonb_array_length(payload->'taskUpdate'->'tasks') > 0- ELSE false END;- `- _, err = db.ExecContext(ctx, migrate)- require.NoError(t, err, "migration UPDATE must not fail on scalar tasks values")-- // Verify: valid array row was rewritten with environment ID.- var arrayPayload string- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 1`).Scan(&arrayPayload)- require.NoError(t, err)- assert.Contains(t, arrayPayload, "environments/prod", "array row should have rewritten stage reference")- assert.NotContains(t, arrayPayload, "stages/101", "array row should no longer have numeric stage ID")-- // Verify: scalar row was NOT modified.- var scalarPayload string- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 2`).Scan(&scalarPayload)- require.NoError(t, err)- assert.Contains(t, scalarPayload, "stages/101", "scalar row should be unchanged")-- // Verify: null row was NOT modified.- var nullPayload string- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 3`).Scan(&nullPayload)- require.NoError(t, err)- assert.Contains(t, nullPayload, "null", "null row should be unchanged")-}--// TestMigration3_22_5_ScopeSheetBlob verifies the sheet_blob_ref backfill.-//-// The four project-scoped sources (plan, task, release, plan_check_run) each-// yield one (project, hash) ref; references naming a hash with no blob are-// skipped; unreferenced blobs end with zero refs; and a hash claimed by more-// than one project is recorded per project so the multi-project audit reports-// it — detected, not silently merged.-//-// The revision branch derives the authoring project from corroborated-// provenance (never db.project). A corroborating row is almost always itself-// a source granting the same (project, hash), so the observable probe for-// each corroboration rule is the one additive shape: task-run provenance-// whose task routes its hash through a release in a DIFFERENT project. There-// the revision branch grants (task-run project, hash), which no source scan-// produces, so a rule failure is visible as that ref's absence.-func TestMigration3_22_5_ScopeSheetBlob(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })- db := container.GetDB()-- _, err := db.ExecContext(ctx, `- CREATE TABLE project (resource_id TEXT PRIMARY KEY);- CREATE TABLE sheet_blob (sha256 BYTEA NOT NULL PRIMARY KEY, content TEXT NOT NULL);- CREATE TABLE plan (project TEXT NOT NULL, config JSONB NOT NULL DEFAULT '{}');- CREATE TABLE task (project TEXT NOT NULL, id BIGINT NOT NULL, payload JSONB NOT NULL DEFAULT '{}');- CREATE TABLE task_run (project TEXT NOT NULL, id BIGINT NOT NULL, task_id INTEGER NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now());- CREATE TABLE release (project TEXT NOT NULL, release_id TEXT NOT NULL DEFAULT '', created_at TIMESTAMPTZ NOT NULL DEFAULT now(), payload JSONB NOT NULL DEFAULT '{}');- CREATE TABLE plan_check_run (project TEXT NOT NULL, result JSONB NOT NULL DEFAULT '{}');- CREATE TABLE revision (resource_id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, instance TEXT NOT NULL, db_name TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), payload JSONB NOT NULL DEFAULT '{}');- INSERT INTO project VALUES- ('p1'), ('p2'), ('p3'), ('p4'),- ('rel-ok'), ('rel-upper'), ('tr-ok'), ('rel-old'), ('tr-late'), ('rel-mismatch'), ('tr-mismatch'),- ('rel-dup'), ('tr-dup'), ('rel-task'), ('rel-taskid'), ('tr-taskid'), ('tr-direct'),- ('rel-prefer'), ('rel-nest-prefer'), ('tr-prefer'),- ('rel-newer'), ('rel-nest-fb'), ('tr-fb');- `)- require.NoError(t, err)-- sheetHex := func(content string) string {- h := sha256.Sum256([]byte(content))- return hex.EncodeToString(h[:])- }- blobs := map[string]string{}- for _, name := range []string{- "plan", "task", "release", "pcr", "shared", "orphan",- "rev-rel", "rev-upper", "rev-nest", "rev-late", "rev-mismatch", "rev-other",- "rev-dup", "rev-taskid", "rev-direct", "rev-prefer", "rev-fb", "rev-ghost", "rev-bare",- "rev-overflow", "rev-stale-stamp", "rev-empty-stamp",- } {- content := "SELECT '" + name + "';"- blobs[name] = sheetHex(content)- _, err := db.ExecContext(ctx,- "INSERT INTO sheet_blob VALUES (decode($1, 'hex'), $2)", blobs[name], content)- require.NoError(t, err)- }- missingBlob := strings.Repeat("f", 64)-- const (- older = "2026-01-01 00:00:00+00" // predates every revision- rev = "2026-02-01 00:00:00+00" // every revision's created_at- newer = "2026-03-01 00:00:00+00" // postdates every revision- )- taskRunName := func(project string, taskID, taskRunID int) string {- return fmt.Sprintf("projects/%s/plans/1/rollout/stages/prod/tasks/%d/taskRuns/%d", project, taskID, taskRunID)- }-- type seed struct {- query string- args []any- }- seeds := []seed{- // The four project-scoped sources.- {"INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",- []any{fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}},{"createDatabaseConfig":{}}]}`, blobs["plan"])}},- // The same hash referenced from two projects: honest dedup or a- // laundered reference — indistinguishable, so both refs are- // backfilled and the multi-project audit reports the hash.- {"INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",- []any{fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}}]}`, blobs["shared"])}},- {"INSERT INTO task (project, id, payload) VALUES ('p2', 1, $1::jsonb)",- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["task"])}},- {"INSERT INTO task (project, id, payload) VALUES ('p2', 2, $1::jsonb)",- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["shared"])}},- // A reference naming a hash with no blob is skipped by the EXISTS- // guard, not a foreign-key abort.- {"INSERT INTO task (project, id, payload) VALUES ('p2', 3, $1::jsonb)",- []any{fmt.Sprintf(`{"sheetSha256":%q}`, missingBlob)}},- {"INSERT INTO task (project, id, payload) VALUES ('p2', 4, '{}')", nil},- {"INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r1', $1::jsonb)",- []any{fmt.Sprintf(`{"files":[{"path":"a.sql","sheetSha256":%q},{"path":"no-sheet.sql"}]}`, blobs["release"])}},- {"INSERT INTO plan_check_run (project, result) VALUES ('p4', $1::jsonb)",- []any{fmt.Sprintf(`{"results":[{"sheetSha256":%q},{"title":"no sheet"}]}`, blobs["pcr"])}},-- // Null and malformed shapes: every one must expand to nothing, not- // abort. JSON null in place of an array is the case COALESCE alone- // would pass through to jsonb_array_elements.- {"INSERT INTO plan (project, config) VALUES ('p1', '{}')", nil},- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": null}')`, nil},- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": []}')`, nil},- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": ["garbage", null]}')`, nil},- {"INSERT INTO task (project, id, payload) VALUES ('p2', 5, 'null'::jsonb)", nil},- {`INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r-null', '{"files": null}')`, nil},- {`INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r-nullelem', '{"files": [null]}')`, nil},- {`INSERT INTO plan_check_run (project, result) VALUES ('p4', '{"results": null}')`, nil},-- // Revision: release provenance corroborates (exactly one row, older,- // carries the hash) → (rel-ok, rev-rel). Redundant with the release- // source, so presence-only.- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-ok', 'r', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-rel"])}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-ok/releases/r"}`, blobs["rev-rel"])}},-- // Legacy uppercase hex: the revision stores the hash in uppercase- // while the release file carries lowercase. Text comparisons are- // lowered, so this corroborates → (rel-upper, rev-upper).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-upper', 'r', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-upper"])}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-upper/releases/r"}`, strings.ToUpper(blobs["rev-upper"]))}},-- // Observable probe, all constraints satisfied: task-run provenance in- // tr-ok whose task routes the hash through a release in rel-task.- // Expect the additive (tr-ok, rev-nest) alongside the source-derived- // (rel-task, rev-nest).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-task', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-nest"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-ok', 10, $1::jsonb)",- []any{`{"release":"projects/rel-task/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-ok', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-nest"], taskRunName("tr-ok", 10, 100))}},-- // Temporal violation: the task run postdates the revision (the ID-reuse- // shape after a purge). No (tr-late, rev-late).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-old', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-late"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-late', 10, $1::jsonb)",- []any{`{"release":"projects/rel-old/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-late', 100, 10, $1)", []any{newer}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-late"], taskRunName("tr-late", 10, 100))}},-- // Hash mismatch: the nested release carries a different hash than the- // revision claims (laundered provenance). No (tr-mismatch, rev-mismatch).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-mismatch', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-other"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-mismatch', 10, $1::jsonb)",- []any{`{"release":"projects/rel-mismatch/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-mismatch', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-mismatch"], taskRunName("tr-mismatch", 10, 100))}},-- // (project, release_id) matching two rows: not a unique key, so- // exactly-one fails even though both rows carry the hash and predate- // the revision. No (tr-dup, rev-dup).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-dup', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-dup"])}},- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-dup', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-dup"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-dup', 10, $1::jsonb)",- []any{`{"release":"projects/rel-dup/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-dup', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-dup"], taskRunName("tr-dup", 10, 100))}},-- // Task-ID mismatch: the task run exists on (project, id) but belongs- // to a different task than the name claims. No (tr-taskid, rev-taskid).- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-taskid', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-taskid"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-taskid', 11, $1::jsonb)",- []any{`{"release":"projects/rel-taskid/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-taskid', 100, 11, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-taskid"], taskRunName("tr-taskid", 10, 100))}},-- // Direct task hash: corroborates through task.payload.sheetSha256.- // Redundant with the task source, so presence-only.- {"INSERT INTO task (project, id, payload) VALUES ('tr-direct', 10, $1::jsonb)",- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["rev-direct"])}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-direct', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-direct"], taskRunName("tr-direct", 10, 100))}},-- // Preference: both release and task-run provenance corroborate. The- // release wins, so the task-run fallback's additive grant- // (tr-prefer, rev-prefer) must NOT appear.- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-prefer', 'r', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-prefer"])}},- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-nest-prefer', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-prefer"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-prefer', 10, $1::jsonb)",- []any{`{"release":"projects/rel-nest-prefer/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-prefer', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-prefer/releases/r","taskRun":%q}`,- blobs["rev-prefer"], taskRunName("tr-prefer", 10, 100))}},-- // Fallback: the release provenance fails on age (the release postdates- // the revision), so the corroborated task run supplies the grant —- // (tr-fb, rev-fb) must appear.- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-newer', 'r', $1, $2::jsonb)",- []any{newer, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-fb"])}},- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-nest-fb', 'nest', $1, $2::jsonb)",- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-fb"])}},- {"INSERT INTO task (project, id, payload) VALUES ('tr-fb', 10, $1::jsonb)",- []any{`{"release":"projects/rel-nest-fb/releases/nest"}`}},- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-fb', 100, 10, $1)", []any{older}},- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-newer/releases/r","taskRun":%q}`,- blobs["rev-fb"], taskRunName("tr-fb", 10, 100))}},-- // Provenance naming rows that no longer exist (purged authoring- // project): no ref at all, the hash stays zero-ref.- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/ghost/releases/r","taskRun":%q}`,- blobs["rev-ghost"], taskRunName("ghost", 10, 100))}},-- // No provenance at all: no ref, the hash stays zero-ref.- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q}`, blobs["rev-bare"])}},-- // Abort hazards: neither row may fail the migration.- // A taskRun name whose IDs overflow bigint fails the bounded digit- // match and stays uncorroborated instead of erroring the cast.- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":"projects/tr-ok/plans/1/rollout/stages/prod/tasks/99999999999999999999999/taskRuns/99999999999999999999999"}`,- blobs["rev-overflow"])}},- // A pre-existing payload.project naming no live project row is kept- // as-is but granted no ref: the project EXISTS guard skips it instead- // of aborting on the foreign key.- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"project":"stale-ghost"}`, blobs["rev-stale-stamp"])}},- // Same for the empty string, which survives the IS NOT NULL filter.- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"project":""}`, blobs["rev-empty-stamp"])}},- }- for _, s := range seeds {- _, err := db.ExecContext(ctx, s.query, s.args...)- require.NoError(t, err, s.query)- }-- statement, err := migrationFS.ReadFile("migration/3.22/0005##scope_sheet_blob.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(statement))- require.NoError(t, err)-- rows, err := db.QueryContext(ctx,- "SELECT project, encode(sha256, 'hex') FROM sheet_blob_ref ORDER BY project, sha256")- require.NoError(t, err)- defer rows.Close()- var got []string- for rows.Next() {- var project, sha string- require.NoError(t, rows.Scan(&project, &sha))- got = append(got, project+"|"+sha)- }- require.NoError(t, rows.Err())- require.ElementsMatch(t, []string{- // The four project-scoped sources.- "p1|" + blobs["plan"],- "p1|" + blobs["shared"],- "p2|" + blobs["task"],- "p2|" + blobs["shared"],- "p3|" + blobs["release"],- "p4|" + blobs["pcr"],- // Release rows seeded as corroboration targets are sources themselves.- "rel-ok|" + blobs["rev-rel"],- "rel-upper|" + blobs["rev-upper"],- "rel-task|" + blobs["rev-nest"],- "rel-old|" + blobs["rev-late"],- "rel-mismatch|" + blobs["rev-other"],- "rel-dup|" + blobs["rev-dup"],- "rel-taskid|" + blobs["rev-taskid"],- "tr-direct|" + blobs["rev-direct"],- "rel-prefer|" + blobs["rev-prefer"],- "rel-nest-prefer|" + blobs["rev-prefer"],- "rel-newer|" + blobs["rev-fb"],- "rel-nest-fb|" + blobs["rev-fb"],- // The revision branch's observable grants: corroborated task-run- // provenance routing through another project's release, and the- // age-based fallback. Every negative scenario is covered by this- // list's exactness: tr-late, tr-mismatch, tr-dup, tr-taskid and- // tr-prefer hold no refs.- "tr-ok|" + blobs["rev-nest"],- "tr-fb|" + blobs["rev-fb"],- }, got)-- // Corroboration also stamps payload.project — the stored fact new- // revision writers set at creation. Each scenario's revision is- // identified by its unique hash; an empty stamp means the provenance did- // not corroborate. rev-prefer pins the release-over-taskRun preference- // directly: both branches corroborate and the release's project wins.- for sha, want := range map[string]string{- blobs["rev-rel"]: "rel-ok",- blobs["rev-upper"]: "rel-upper",- blobs["rev-nest"]: "tr-ok",- blobs["rev-late"]: "",- blobs["rev-mismatch"]: "",- blobs["rev-dup"]: "",- blobs["rev-taskid"]: "",- blobs["rev-direct"]: "tr-direct",- blobs["rev-prefer"]: "rel-prefer",- blobs["rev-fb"]: "tr-fb",- blobs["rev-ghost"]: "",- blobs["rev-bare"]: "",- blobs["rev-overflow"]: "",- blobs["rev-stale-stamp"]: "stale-ghost",- blobs["rev-empty-stamp"]: "",- } {- var got string- require.NoError(t, db.QueryRowContext(ctx,- "SELECT COALESCE(payload->>'project', '') FROM revision WHERE lower(payload->>'sheetSha256') = $1", sha).Scan(&got))- require.Equal(t, want, got, "stamped project for hash %s", sha)- }-- // The zero-ref audit counts the never-referenced blob plus the revisions- // whose provenance was absent or uncorroborated with no other source.- var zeroRefBlobs int- require.NoError(t, db.QueryRowContext(ctx, `- SELECT count(*) FROM sheet_blob b- WHERE NOT EXISTS (SELECT 1 FROM sheet_blob_ref r WHERE r.sha256 = b.sha256)- `).Scan(&zeroRefBlobs))- require.Equal(t, 7, zeroRefBlobs, "orphan, rev-mismatch, rev-ghost, rev-bare, rev-overflow, rev-stale-stamp, and rev-empty-stamp have zero refs")-- // The multi-project audit reports every hash claimed by more than one- // project, including the deliberately shared one.- auditRows, err := db.QueryContext(ctx, `- SELECT encode(sha256, 'hex')- FROM sheet_blob_ref- GROUP BY sha256- HAVING count(DISTINCT project) > 1- `)- require.NoError(t, err)- defer auditRows.Close()- multiProject := map[string]bool{}- for auditRows.Next() {- var sha string- require.NoError(t, auditRows.Scan(&sha))- multiProject[sha] = true- }- require.NoError(t, auditRows.Err())- require.True(t, multiProject[blobs["shared"]], "the cross-project reference must be reported, not blessed")-- // The source-level missing-ref audit from the design doc: a plan that- // references a pre-existing foreign hash AFTER the backfill invokes no- // CreateSheets and writes no ref, so only this query can see it.- _, err = db.ExecContext(ctx, "INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",- fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}}]}`, blobs["pcr"]))- require.NoError(t, err)- var missingProject, missingSha string- require.NoError(t, db.QueryRowContext(ctx, `- SELECT DISTINCT pl.project, spec->'changeDatabaseConfig'->>'sheetSha256' AS sha- FROM plan pl- CROSS JOIN LATERAL jsonb_array_elements(- CASE WHEN jsonb_typeof(pl.config->'specs') = 'array' THEN pl.config->'specs' ELSE '[]'::jsonb END) AS spec- WHERE spec->'changeDatabaseConfig'->>'sheetSha256' ~ '^[0-9a-fA-F]{64}$'- AND NOT EXISTS (- SELECT 1 FROM sheet_blob_ref r- WHERE r.project = pl.project- AND r.sha256 = decode(spec->'changeDatabaseConfig'->>'sheetSha256', 'hex')- )- `).Scan(&missingProject, &missingSha))- require.Equal(t, "p1", missingProject)- require.Equal(t, blobs["pcr"], missingSha)-}diff --git a/backend/migrator/retire_export_data_test.go b/backend/migrator/retire_export_data_test.godeleted file mode 100644index f26c524dfa..0000000000--- a/backend/migrator/retire_export_data_test.go+++ /dev/null@@ -1,178 +0,0 @@-package migrator--import (- "context"- "io/fs"- "testing"-- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"-)--// TestRetireExportDataMigration seeds legacy DATABASE_EXPORT rows and verifies-// the 3.22 purge deletes the whole export family — including the draft-// DATABASE_CHANGE issue attached to an export plan, which a type-only delete-// would leave behind to break the plan delete on the issue→plan FK — while-// unrelated plans, issues, tasks, and comments survive.-//-// Project p2 seeds a change workflow whose plan/issue/task/task_run ids all-// collide with p1's export family: every purge predicate must carry the-// project column of the composite PKs, so p2's rows survive untouched.-func TestRetireExportDataMigration(t *testing.T) {- ctx := context.Background()- container := testcontainer.GetTestPgContainer(ctx, t)- t.Cleanup(func() { container.Close(ctx) })-- db := container.GetDB()- require.NoError(t, MigrateSchema(ctx, db))-- // Fresh installs no longer have export_archive; recreate the pre-3.22 table- // so the DROP TABLE in the migration runs against a realistic schema.- _, err := db.ExecContext(ctx, `- CREATE TABLE export_archive (- resource_id text PRIMARY KEY DEFAULT gen_random_uuid()::text,- workspace text NOT NULL REFERENCES workspace(resource_id),- created_at timestamptz NOT NULL DEFAULT now(),- bytes bytea,- payload jsonb NOT NULL DEFAULT '{}'- );-- INSERT INTO workspace (resource_id) VALUES ('default');- INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');- INSERT INTO project (resource_id, workspace, name) VALUES ('p1', 'default', 'Project 1');- INSERT INTO instance (resource_id, workspace) VALUES ('i1', 'default');-- -- Family A: classic completed export workflow.- INSERT INTO plan (id, creator, project, name, description, config) VALUES- (100, '[email protected]', 'p1', 'export plan', '',- '{"specs":[{"id":"s1","exportDataConfig":{"targets":["instances/i1/databases/db1"]}}]}');- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (200, '[email protected]', 'p1', 100, 'export issue', 'DONE', 'DATABASE_EXPORT');- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES- (300, 'p1', 100, 'i1', 'db1', 'DATABASE_EXPORT');- INSERT INTO task_run (id, creator, project, task_id, attempt, status, result) VALUES- (400, '[email protected]', 'p1', 300, 0, 'DONE', '{"exportArchiveId":"arch-1"}');- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 400);- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (500, 'p1', 100, 'DONE');- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 200);- INSERT INTO plan_webhook_delivery (project, plan_id, event_type) VALUES ('p1', 100, 'PIPELINE_COMPLETED');- INSERT INTO export_archive (resource_id, workspace, bytes) VALUES ('arch-1', 'default', ''::bytea);-- -- Family B: export plan carrying a draft DATABASE_CHANGE issue (the FK trap).- INSERT INTO plan (id, creator, project, name, description, config) VALUES- (101, '[email protected]', 'p1', 'export plan with draft', '',- '{"specs":[{"id":"s1","exportDataConfig":{}}]}');- INSERT INTO issue (id, creator, project, plan_id, name, status, type, payload) VALUES- (201, '[email protected]', 'p1', 101, 'draft on export plan', 'OPEN', 'DATABASE_CHANGE', '{"draft":true}');- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 201);-- -- Family D: legacy mixed plan (change + export specs). The whole family- -- is purged via the plan-membership arm, including its DATABASE_MIGRATE- -- task and DATABASE_CHANGE issue.- INSERT INTO plan (id, creator, project, name, description, config) VALUES- (103, '[email protected]', 'p1', 'mixed plan', '',- '{"specs":[{"id":"s1","changeDatabaseConfig":{}},{"id":"s2","exportDataConfig":{}}]}');- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (205, '[email protected]', 'p1', 103, 'mixed issue', 'DONE', 'DATABASE_CHANGE');- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES- (303, 'p1', 103, 'i1', 'db1', 'DATABASE_MIGRATE');- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES- (403, '[email protected]', 'p1', 303, 0, 'DONE');- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 403);-- -- An export-typed issue with no plan at all.- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (204, '[email protected]', 'p1', NULL, 'orphan export issue', 'CANCELED', 'DATABASE_EXPORT');-- -- Family C: unrelated change workflow that must survive.- INSERT INTO plan (id, creator, project, name, description, config) VALUES- (102, '[email protected]', 'p1', 'change plan', '',- '{"specs":[{"id":"s1","changeDatabaseConfig":{"targets":["instances/i1/databases/db1"]}}]}');- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (202, '[email protected]', 'p1', 102, 'change issue', 'DONE', 'DATABASE_CHANGE');- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES- (302, 'p1', 102, 'i1', 'db1', 'DATABASE_MIGRATE');- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES- (402, '[email protected]', 'p1', 302, 0, 'DONE');- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 402);- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (502, 'p1', 102, 'DONE');- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 202);-- -- A plan-less role grant issue that must survive.- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (203, '[email protected]', 'p1', NULL, 'role grant', 'OPEN', 'ROLE_GRANT');-- -- Project p2: a change workflow whose ids collide with p1's export- -- family. Survives in full — proves composite-PK project scoping.- INSERT INTO project (resource_id, workspace, name) VALUES ('p2', 'default', 'Project 2');- INSERT INTO plan (id, creator, project, name, description, config) VALUES- (100, '[email protected]', 'p2', 'p2 change plan', '',- '{"specs":[{"id":"s1","changeDatabaseConfig":{"targets":["instances/i1/databases/db2"]}}]}');- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES- (200, '[email protected]', 'p2', 100, 'p2 change issue', 'DONE', 'DATABASE_CHANGE');- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES- (300, 'p2', 100, 'i1', 'db2', 'DATABASE_MIGRATE');- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES- (400, '[email protected]', 'p2', 300, 0, 'DONE');- INSERT INTO task_run_log (project, task_run_id) VALUES ('p2', 400);- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (500, 'p2', 100, 'DONE');- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p2', 200);- INSERT INTO plan_webhook_delivery (project, plan_id, event_type) VALUES ('p2', 100, 'PIPELINE_COMPLETED');- `)- require.NoError(t, err)-- // Positive preconditions: the seeds must exist before the purge, so the- // survival assertions below cannot pass vacuously.- var planCount, issueCount int- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM plan`).Scan(&planCount))- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue`).Scan(&issueCount))- require.Equal(t, 5, planCount)- require.Equal(t, 7, issueCount)-- buf, err := fs.ReadFile(migrationFS, "migration/3.22/0000##retire_export_data.sql")- require.NoError(t, err)-- conn, err := db.Conn(ctx)- require.NoError(t, err)- defer conn.Close()- require.NoError(t, executeMigration(ctx, conn, string(buf), "test-retire-export-data"))-- queryIDs := func(query string) []int64 {- rows, err := db.QueryContext(ctx, query)- require.NoError(t, err)- defer rows.Close()- var ids []int64- for rows.Next() {- var id int64- require.NoError(t, rows.Scan(&id))- ids = append(ids, id)- }- require.NoError(t, rows.Err())- return ids- }-- require.Equal(t, []int64{102}, queryIDs(`SELECT id FROM plan WHERE project = 'p1' ORDER BY id`))- require.Equal(t, []int64{202, 203}, queryIDs(`SELECT id FROM issue WHERE project = 'p1' ORDER BY id`))- require.Equal(t, []int64{302}, queryIDs(`SELECT id FROM task WHERE project = 'p1' ORDER BY id`))- require.Equal(t, []int64{402}, queryIDs(`SELECT id FROM task_run WHERE project = 'p1' ORDER BY id`))- require.Equal(t, []int64{402}, queryIDs(`SELECT task_run_id FROM task_run_log WHERE project = 'p1' ORDER BY task_run_id`))- require.Equal(t, []int64{502}, queryIDs(`SELECT id FROM plan_check_run WHERE project = 'p1' ORDER BY id`))- require.Equal(t, []int64{202}, queryIDs(`SELECT issue_id FROM issue_comment WHERE project = 'p1' ORDER BY issue_id`))- require.Empty(t, queryIDs(`SELECT plan_id FROM plan_webhook_delivery WHERE project = 'p1'`))-- // p2's colliding-id change workflow survives in full.- require.Equal(t, []int64{100}, queryIDs(`SELECT id FROM plan WHERE project = 'p2' ORDER BY id`))- require.Equal(t, []int64{200}, queryIDs(`SELECT id FROM issue WHERE project = 'p2' ORDER BY id`))- require.Equal(t, []int64{300}, queryIDs(`SELECT id FROM task WHERE project = 'p2' ORDER BY id`))- require.Equal(t, []int64{400}, queryIDs(`SELECT id FROM task_run WHERE project = 'p2' ORDER BY id`))- require.Equal(t, []int64{400}, queryIDs(`SELECT task_run_id FROM task_run_log WHERE project = 'p2' ORDER BY task_run_id`))- require.Equal(t, []int64{500}, queryIDs(`SELECT id FROM plan_check_run WHERE project = 'p2' ORDER BY id`))- require.Equal(t, []int64{200}, queryIDs(`SELECT issue_id FROM issue_comment WHERE project = 'p2' ORDER BY issue_id`))- require.Equal(t, []int64{100}, queryIDs(`SELECT plan_id FROM plan_webhook_delivery WHERE project = 'p2'`))-- var exportArchiveExists bool- require.NoError(t, db.QueryRowContext(ctx,- `SELECT to_regclass('export_archive') IS NOT NULL`).Scan(&exportArchiveExists))- require.False(t, exportArchiveExists)-}diff --git a/backend/migrator/split_saved_query_manage_test.go b/backend/migrator/split_saved_query_manage_test.godeleted file mode 100644index db159f8cf7..0000000000--- a/backend/migrator/split_saved_query_manage_test.go+++ /dev/null@@ -1,76 +0,0 @@-package migrator--import (- "context"- "encoding/json"- "testing"-- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"-)--// TestSplitSavedQueryManageMigration exercises 3.22/0009's jsonb transform:-// bb.savedQueries.manage expands to the four per-verb permissions —-// deliberately without setIamPolicy — deduplicated, while rows without-// manage and rows with a non-array permissions payload stay untouched.-func TestSplitSavedQueryManageMigration(t *testing.T) {- if testing.Short() {- t.Skip("skipping testcontainer test in short mode")- }- ctx := context.Background()- pg := testcontainer.GetTestPgContainer(ctx, t)- defer pg.Close(ctx)- db := pg.GetDB()-- _, err := db.ExecContext(ctx, `- CREATE TABLE role (- resource_id text NOT NULL PRIMARY KEY,- permissions jsonb NOT NULL DEFAULT '{}'- );- INSERT INTO role (resource_id, permissions) VALUES- ('manage-holder', '{"permissions": ["bb.savedQueries.manage", "bb.savedQueries.get", "bb.projects.get"]}'),- ('no-manage', '{"permissions": ["bb.savedQueries.search", "bb.projects.get"]}'),- ('non-array', '{"permissions": {"oops": true}}'),- ('empty', '{}');- `)- require.NoError(t, err)-- migration, err := migrationFS.ReadFile("migration/3.22/0009##split_saved_query_manage.sql")- require.NoError(t, err)- _, err = db.ExecContext(ctx, string(migration))- require.NoError(t, err)-- permissionsOf := func(roleID string) []string {- var raw []byte- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions->'permissions' FROM role WHERE resource_id = $1`, roleID).Scan(&raw))- if string(raw) == "null" {- return nil- }- var list []string- require.NoError(t, json.Unmarshal(raw, &list))- return list- }-- // manage expands to the four verbs, deduplicated against the get the- // role already held; setIamPolicy is deliberately absent.- require.ElementsMatch(t, []string{- "bb.projects.get",- "bb.savedQueries.delete",- "bb.savedQueries.get",- "bb.savedQueries.getIamPolicy",- "bb.savedQueries.update",- }, permissionsOf("manage-holder"))- require.NotContains(t, permissionsOf("manage-holder"), "bb.savedQueries.setIamPolicy")-- // Rows without manage keep their permissions verbatim.- require.Equal(t, []string{"bb.savedQueries.search", "bb.projects.get"}, permissionsOf("no-manage"))-- // Non-array and empty payloads are untouched rather than mangled.- var nonArray string- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions::text FROM role WHERE resource_id = 'non-array'`).Scan(&nonArray))- require.JSONEq(t, `{"permissions": {"oops": true}}`, nonArray)- var empty string- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions::text FROM role WHERE resource_id = 'empty'`).Scan(&empty))- require.JSONEq(t, `{}`, empty)-}diff --git a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.godeleted file mode 100644index 1cf3091538..0000000000--- a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go+++ /dev/null@@ -1,89 +0,0 @@-package starrocks--import (- "bytes"- "context"- "database/sql"- "fmt"- "testing"- "time"-- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"-)--// TestDump_StarRocksMaterializedViewRoundTrip locks the two dump fixes (BYT-9689) against a-// real StarRocks (amd64-only; skipped in -short). A materialized view defined on a regular-// view is dumped, then replayed into a fresh database of the same name, and must come back:-// - without a name collision (P1: the temporary placeholder is dropped as a view, so the-// real CREATE MATERIALIZED VIEW does not collide with a leftover placeholder), and-// - IS_ACTIVE=true (P2: the MV is emitted after its source view, so it is created against-// the real view, not inactivated when a placeholder is dropped under it).-//-// The dump is schema-only (--no-data), so IS_ACTIVE — not row data — is the P2 signal.-func TestDump_StarRocksMaterializedViewRoundTrip(t *testing.T) {- if testing.Short() {- t.Skip("skipping StarRocks testcontainer test in short mode")- }- ctx := context.Background()- c := testcontainer.GetTestStarRocksContainer(ctx, t)- defer c.Close(ctx)- db := c.GetDB()-- for _, stmt := range []string{- "CREATE DATABASE rt",- "CREATE TABLE rt.t (id INT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",- "INSERT INTO rt.t VALUES (1,10),(2,20)",- "CREATE VIEW rt.vw AS SELECT id, amt FROM rt.t",- "CREATE MATERIALIZED VIEW rt.mv REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM rt.vw GROUP BY id",- } {- if _, err := db.ExecContext(ctx, stmt); err != nil {- t.Fatalf("setup failed: %s: %v", stmt, err)- }- }- requireMVActive(ctx, t, db, "rt", "mv")-- // Produce the dump through the real driver.- d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "rt"}- var buf bytes.Buffer- require.NoError(t, d.Dump(ctx, &buf, nil))- dump := buf.String()- require.Contains(t, dump, "CREATE MATERIALIZED VIEW", "dump should emit the MV")-- // Restore into a fresh same-name database. StarRocks qualifies view/MV references with- // the original db, so the restore target must reuse the name. DROP+CREATE+USE+replay run- // in one multi-statement Exec so USE applies to the unqualified statements on the same- // pooled connection.- restore := "DROP DATABASE IF EXISTS rt;\nCREATE DATABASE rt;\nUSE rt;\n" + dump- if _, err := db.ExecContext(ctx, restore); err != nil {- t.Fatalf("dump replay failed (P1 collision or unreplayable DDL): %v", err)- }-- // P2: the restored MV must be active, not inactivated by the placeholder dance.- requireMVActive(ctx, t, db, "rt", "mv")-- // And it re-syncs cleanly: mv as a materialized view, no stale placeholder.- meta, err := d.SyncDBSchema(ctx)- require.NoError(t, err)- require.Len(t, meta.Schemas, 1)- require.Len(t, meta.Schemas[0].MaterializedViews, 1)- require.Equal(t, "mv", meta.Schemas[0].MaterializedViews[0].Name)-}--// requireMVActive waits until the named materialized view reports IS_ACTIVE=true.-func requireMVActive(ctx context.Context, t *testing.T, db *sql.DB, schema, name string) {- t.Helper()- q := fmt.Sprintf("SELECT IS_ACTIVE FROM information_schema.materialized_views WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'", schema, name)- require.Eventually(t, func() bool {- var active string- if err := db.QueryRowContext(ctx, q).Scan(&active); err != nil {- return false- }- return active == "true"- // information_schema.materialized_views (listMaterializedViewStatus, an FE->BE RPC)- // is unavailable for ~tens of seconds after a fresh allin1 boot ("BE host unknown")- // before it settles, so allow a generous window.- }, 180*time.Second, 3*time.Second, "MV %s.%s did not become active", schema, name)-}diff --git a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.godeleted file mode 100644index ae004487d9..0000000000--- a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go+++ /dev/null@@ -1,75 +0,0 @@-package starrocks--import (- "context"- "testing"- "time"-- "github.com/stretchr/testify/require"-- "github.com/bytebase/bytebase/backend/common/testcontainer"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"-)--// TestSyncDBSchema_StarRocksMaterializedView_E2E brings up a real StarRocks (amd64-only;-// runs in CI, skipped in -short) and verifies BYT-9689 end to end: an async materialized-// view is synced under MaterializedViews with its definition, while a synchronous rollup-// (REFRESH_TYPE='ROLLUP', which has no information_schema.tables row) is excluded.-func TestSyncDBSchema_StarRocksMaterializedView_E2E(t *testing.T) {- if testing.Short() {- t.Skip("skipping StarRocks testcontainer test in short mode")- }- ctx := context.Background()- c := testcontainer.GetTestStarRocksContainer(ctx, t)- defer c.Close(ctx)- db := c.GetDB()-- for _, stmt := range []string{- "CREATE DATABASE mvtest",- "CREATE TABLE mvtest.sales (id BIGINT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",- "INSERT INTO mvtest.sales VALUES (1,10),(2,20)",- "CREATE MATERIALIZED VIEW mvtest.mv_async REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM mvtest.sales GROUP BY id",- "CREATE MATERIALIZED VIEW mvtest.mv_rollup AS SELECT id, sum(amt) FROM mvtest.sales GROUP BY id",- } {- if _, err := db.ExecContext(ctx, stmt); err != nil {- t.Fatalf("setup failed: %s: %v", stmt, err)- }- }-- // The sync rollup builds asynchronously; wait until it registers in- // information_schema.materialized_views so the test deterministically exercises the- // rollup-exclusion path (it appears there as REFRESH_TYPE='ROLLUP' but never as a- // information_schema.tables row).- require.Eventually(t, func() bool {- var n int- if err := db.QueryRowContext(ctx,- "SELECT COUNT(*) FROM information_schema.materialized_views WHERE TABLE_SCHEMA='mvtest' AND REFRESH_TYPE='ROLLUP'").Scan(&n); err != nil {- return false- }- return n == 1- // information_schema.materialized_views (an FE->BE RPC) is unavailable for tens of- // seconds after a fresh allin1 boot before it settles; allow a generous window.- }, 150*time.Second, 3*time.Second, "sync rollup did not register in materialized_views")-- d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "mvtest"}- meta, err := d.SyncDBSchema(ctx)- require.NoError(t, err)- require.Len(t, meta.Schemas, 1)- schema := meta.Schemas[0]-- // The async MV is synced under MaterializedViews with its definition.- require.Len(t, schema.MaterializedViews, 1, "exactly the async MV must be synced")- require.Equal(t, "mv_async", schema.MaterializedViews[0].Name)- require.Contains(t, schema.MaterializedViews[0].Definition, "CREATE MATERIALIZED VIEW")-- // The base table is synced as a table; the sync rollup leaks nowhere; the MV is not- // misfiled as a regular view.- require.Len(t, schema.Tables, 1)- require.Equal(t, "sales", schema.Tables[0].Name)- for _, mv := range schema.MaterializedViews {- require.NotEqual(t, "mv_rollup", mv.Name, "sync rollup must be excluded from MaterializedViews")- }- for _, v := range schema.Views {- require.NotEqual(t, "mv_async", v.Name, "MV must not be misfiled as a regular view")- }-}diff --git a/docs/design/backend-test-execution-time.md b/docs/design/backend-test-execution-time.mdindex a7bc02d056..9ad63ea374 100644--- a/docs/design/backend-test-execution-time.md+++ b/docs/design/backend-test-execution-time.md@@ -92,6 +92,9 @@ cost inside a package that runs 3.7× parallel, and effort 1's saving sits insideffort 4's, so they do not simply add. Together they should take it from 635 sof wall clock to roughly 330 s.+Effort 6 was rescoped and now runs ahead of effort 2 for `api/v1`; the numbering+is kept so existing references resolve.+### [✓] 1. Close idle connections before shutting the server down**Three lines.** `httpServer.Shutdown` burns 1036 ms on every one of the 207@@ -124,10 +127,13 @@ below against 611 s.### 2. One shared container per package, then turn parallelism on-**1371 s of wall clock across four packages, about 1070 s of it container-starts.** `store`, `api/v1`, `component/review` and `migrator` start 247-containers between them, and because they run serially that cost is their wall-clock almost exactly.+**459 s of wall clock, 390 s of it container starts.** `store` starts 90+containers, and because it runs serially that cost is its wall clock almost+exactly.++`api/v1`, `component/review` and `migrator` were in this effort's original+scope. AGENTS.md now confines a real metadata Postgres to `store` and `tests`,+so all three move to effort 6 instead.Two changes, in this order:@@ -143,18 +149,18 @@ Measured on the migrated Bytebase schema (9.5 MB): migrating the template costs`CREATE DATABASE … TEMPLATE` is **44 ms** — 98× cheaper than the 4328 mscontainer it replaces.-Two details. `api/v1` shares a Postgres rather than using a fake store: it is-the smaller change, and a fake risks testing the fake. And close the template-connection after migrating, because `TEMPLATE` refuses to copy a database that-still has a session attached.+One detail. Close the template connection after migrating, because `TEMPLATE`+refuses to copy a database that still has a session attached.++This section originally called for `api/v1` to share a Postgres "rather than+using a fake store". Effort 6 reverses that.Expect tests that silently assumed a virgin database to fail. That is the point.Fix them with per-test databases, not with cleanup hooks — those reintroduceordering coupling.-Step 1 alone takes those four packages from 1371 s to roughly 320 s. Step 2-should take them well below that: what remains is about 300 s of real work-spread across 550 tests.+Step 1 alone takes `store` from 459 s to roughly 75 s. Step 2 should take it+below that; what remains is real work across 128 tests.### 3. Pool the provisioned Postgres instances@@ -250,24 +256,61 @@ Three keep their engine. Each needs a comment saying why:under test. It covers MySQL, Postgres, Oracle and MSSQL in one table, andboots no server.-### 6. Push tests down a level--**Mostly architectural.** A test earns a server boot only if it needs a-background runner, a real rollout, or the audit trail. `backend/api/mcp` shows-the payoff: 168 tests in 39.5 s, no containers.--Candidates: `mcp_capability_setting_test.go` (6 boots to read and write one-settings row), the saved-query list and filter tests (12), and GitOps-`CheckRelease` validation (11 of 17 — one stateless call against different-releases).--Effort 4 makes a boot cheap, which takes most of the money out of this. Treat it-as a rule to hold going forward, not a saving to bank.--Keep the 27 `TestCollision*` and `TestClaim*` tests where they are.+### 6. Seams instead of containers++**912 s of wall clock, 680 s of it container starts.** AGENTS.md confines a real+metadata Postgres to `store` and `tests`. `api/v1`, `component/review` and+`migrator` are not on that list and start 157 containers between them.+`backend/api/mcp` is the proof this is livable: 168 tests in 39.5 s, 16 of its+18 files containerless.++Effort 2 declined a fake store on two grounds, and both fail here. `api/v1` is+not testing its API — `mcp_info_test.go` calls `service.GetMCPInfo` as a Go+method, so the container buys a `*store.Store` and nothing the test asserts. And+"a fake store" was the wrong unit: `store.Store` has 305 methods and `api/v1`+calls 181, but `api/mcp` needed five (`serverStore`, faked in 62 lines) and+`mcp_gate.go` needed one (`mcpSettingsReader`). Interfaces go beside the handler+that reads them, never over the store.++Fourteen files in `api/v1` start containers, ~90 tests. Read from names, not+bodies — verify the back two rows:++| Disposition | Tests | Shape |+| --- | ---: | --- |+| Move to `backend/store` | ~20 | `Test*Claims`, concurrent-update and serialization, `TestMixedIssuePatchRollsBackWhenLabelsFail`, retention filter boundaries, `TestIssueApprovalFiltersRunBeforePaging` |+| Pure function over plain data | ~40 | canonical-name tests, the four `TestCheckRelease*`, converters |+| Narrow interface and a fake | ~30 | `TestGetMCPInfoHandler`, the list-and-hide tests, `TestApproveIssueFailsClosedWhenIAMLookupFails` |++`component/review` splits the same way:+`TestApplyApprovalTemplateAndCreatePlanCheckRunDoNotDeadlock` is lock ordering+and moves to `store`; the CEL, template-matching and target-unfolding tests are+pure functions already, wearing a container for the package's sake.++Prefer the middle row: a handler is fetch, decide, convert, and only the fetch+needs a store.++`migrator` is done, by deletion rather than by moving. `TestLatestVersion` and+`TestVersionUnique` need no database and stay; the other 16 booted a container+each to apply one migration to a hand-built fixture and assert the transform.+Moving them to `backend/store` would have relocated the 90 s, not removed it, so+they were deleted outright. The package now runs in under 2 s.++That is 90 s bought at a real price, and the price should be stated plainly: the+migration SQL — including irreversible data transforms over customer metadata —+now has no test coverage, and no fake can restore it. Anyone reintroducing+coverage here should bring back the fixtures against a shared container rather+than one per test.++**Every fake needs a contract test** — one table run against both the fake and+the real store — or effort 2's objection is right. Since none of these packages+may hold a container, that suite lives in `backend/store`. No `mockgen`, no SQLite.++Convert `mcp_info_test.go` first — `mcpSettingsReader` exists and the test is 60+lines — and measure before doing the rest. Server boots go the same way:+`mcp_capability_setting_test.go` spends 6 to read and write one settings row,+the saved-query list and filter tests 12, GitOps `CheckRelease` 11 of 17. Keep+the 27 `TestCollision*` and `TestClaim*` tests where they are —`backend/store/AGENTS.md` requires them for a bug class unit tests cannot catch.-25 of them already build on `setupCollidingProjects`, so effort 4 gives them a-shared workspace almost for free.### 7. Engine conformance to omni@@ -305,9 +348,12 @@ deleted rather than moved. That is 3 s of the 403 s this effort is about, whichis the measure of how much of the cost sits behind the port rather than besideit.-This plan does not address `plugin/db/starrocks` (158 s). It tests our own-driver against a heavyweight engine, every test runs on every PR, and skipping-is not an option — so it stays as accepted cost.+`plugin/db/starrocks` (158 s) is gone. Nearly all of it was two testcontainer+tests booting the `allin1` image — an FE and a BE in one container, with a+readiness wait measured in minutes — to assert materialized-view sync and dump+round-trip. That is engine fidelity, which belongs in omni, paid for on every PR+by every engineer. The two tests and the `GetStarRocksContainer` helpers were+deleted; the package now runs in under 2 s on its remaining unit tests.### One loose end