Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

perf(tests): drop the container-backed tests in migrator and starrocks (#21341)

Danny Xu6e9709695ecc
diff --git a/AGENTS.md b/AGENTS.md
index 7bff084844..a3f99eb86d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -45,10 +45,15 @@ fidelity belong in [omni](https://github.com/bytebase/omni), so never write
`TestFooForMySQL` beside `TestFooForPostgreSQL`. A test earns a Bytebase server
boot only if it needs a background runner, a real rollout, or the audit trail —
those live in `backend/tests`, the only package that may start one; everything
-else asserts in `backend/api/v1` or `backend/store`. Packages needing a
-real database share one container via `TestMain` and give each test its own
-`CREATE DATABASE`, never a container per test — a Postgres container costs 4.3s
-against a 0.7s server boot.
+else asserts in `backend/api/v1` or `backend/store`.
+
+Only `backend/store` and `backend/tests` may run a real metadata Postgres.
+Everywhere else, extract the decide-and-convert half of a handler as functions
+over plain data, and where state is genuinely read declare an interface beside
+the handler and fake it, as `backend/api/mcp` does with `serverStore` — never
+one interface over the whole store, which has 305 methods.
+Every fake needs a contract test run against the real store too, or it drifts
+into testing itself.
## Development Workflow
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.go
index 30cee0c68f..11115a18c8 100644
--- a/backend/common/testcontainer/testcontainer.go
+++ b/backend/common/testcontainer/testcontainer.go
@@ -15,8 +15,6 @@ import (
"net"
"os"
"path/filepath"
- "runtime"
- "strings"
"testing"
"time"
@@ -463,132 +461,6 @@ func GetTestOracleContainer(ctx context.Context, t testing.TB) *Container {
return container
}
-// GetStarRocksContainer creates a StarRocks (allin1) container for testing. The allin1
-// image bundles the FE and BE; information_schema is served by the BE, so this waits until
-// the backend actually serves queries rather than only the FE port being open.
-//
-// NOTE: requires an amd64 host — the StarRocks BE has no working arm64 build (it fails to
-// come alive under emulation). bytebase CI runs on amd64; locally, run without -short on an
-// amd64 machine, or the test is skipped via testing.Short().
-func GetStarRocksContainer(ctx context.Context) (retC *Container, retErr error) {
- req := testcontainers.ContainerRequest{
- Image: "starrocks/allin1-ubuntu:3.4.10",
- ExposedPorts: []string{"9030/tcp"},
- WaitingFor: wait.ForListeningPort("9030/tcp").WithStartupTimeout(5 * time.Minute),
- }
- c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
- ContainerRequest: req,
- Started: true,
- })
- if err != nil {
- return nil, err
- }
- host, err := c.Host(ctx)
- if err != nil {
- return nil, err
- }
- port, err := c.MappedPort(ctx, "9030/tcp")
- if err != nil {
- return nil, err
- }
- dsn := fmt.Sprintf("root@tcp(%s:%s)/?multiStatements=true", host, port.Port())
- db, err := sql.Open("mysql", dsn)
- if err != nil {
- return nil, err
- }
- defer func() {
- if retErr != nil {
- db.Close()
- }
- }()
- if err := waitStarRocksReady(ctx, db); err != nil {
- return nil, err
- }
- return &Container{
- container: c,
- host: host,
- port: port.Port(),
- db: db,
- }, nil
-}
-
-// GetTestStarRocksContainer creates a StarRocks container and fails the test on error. It
-// skips on non-amd64 hosts: the StarRocks all-in-one BE has no working arm64 build (it never
-// comes alive under emulation), so the readiness wait would otherwise time out.
-func GetTestStarRocksContainer(ctx context.Context, t testing.TB) *Container {
- t.Helper()
- if runtime.GOARCH != "amd64" {
- t.Skipf("StarRocks requires an amd64 host; the all-in-one BE has no working arm64 build (GOARCH=%s)", runtime.GOARCH)
- }
- container, err := GetStarRocksContainer(ctx)
- if err != nil {
- t.Fatalf("failed to create StarRocks container: %v", err)
- }
- return container
-}
-
-// waitStarRocksReady blocks until at least one StarRocks backend is registered and alive,
-// which is required before tablet-allocating DDL (CREATE TABLE) succeeds. The check is
-// read-only on purpose: issuing DDL during allin1's first-boot disrupts backend
-// registration, so poll SHOW BACKENDS rather than probing with CREATE TABLE.
-func waitStarRocksReady(ctx context.Context, db *sql.DB) error {
- ticker := time.NewTicker(3 * time.Second)
- defer ticker.Stop()
- timeout := time.After(10 * time.Minute)
- for {
- select {
- case <-ticker.C:
- if starRocksBackendAlive(ctx, db) {
- return nil
- }
- case <-timeout:
- return errors.Errorf("StarRocks backend did not become ready")
- }
- }
-}
-
-// starRocksBackendAlive reports whether SHOW BACKENDS lists a backend with Alive=true.
-func starRocksBackendAlive(ctx context.Context, db *sql.DB) bool {
- rows, err := db.QueryContext(ctx, "SHOW BACKENDS")
- if err != nil {
- return false
- }
- defer rows.Close()
- cols, err := rows.Columns()
- if err != nil {
- return false
- }
- aliveIdx := -1
- for i, c := range cols {
- if strings.EqualFold(c, "Alive") {
- aliveIdx = i
- break
- }
- }
- if aliveIdx < 0 {
- return false
- }
- alive := false
- for rows.Next() {
- vals := make([]sql.RawBytes, len(cols))
- ptrs := make([]any, len(cols))
- for i := range vals {
- ptrs[i] = &vals[i]
- }
- if err := rows.Scan(ptrs...); err != nil {
- return false
- }
- if strings.EqualFold(string(vals[aliveIdx]), "true") {
- alive = true
- break
- }
- }
- if err := rows.Err(); err != nil {
- return false
- }
- return alive
-}
-
// GetMSSQLContainer creates a Microsoft SQL Server container for testing
func GetMSSQLContainer(ctx context.Context) (retC *Container, retErr error) {
req := testcontainers.ContainerRequest{
diff --git a/backend/migrator/changelog_utf8_test.go b/backend/migrator/changelog_utf8_test.go
deleted file mode 100644
index b246de68bd..0000000000
--- a/backend/migrator/changelog_utf8_test.go
+++ /dev/null
@@ -1,112 +0,0 @@
-package migrator
-
-import (
- "context"
- "strings"
- "testing"
- "unicode/utf8"
-
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
-)
-
-// TestListChangelogsWithInvalidUTF8 verifies that reading sync_history.raw_dump
-// containing invalid UTF-8 bytes does not crash. Regression test for SQLSTATE 22021
-// caused by LEFT(raw_dump, N) triggering PostgreSQL UTF-8 validation.
-//
-// The customer reported: "failed to ensure baseline changelog: failed to check for
-// existing changelogs: failed to query: ERROR: invalid byte sequence for encoding
-// "UTF8": 0xe4 0xb8 (SQLSTATE 22021)"
-func TestListChangelogsWithInvalidUTF8(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- // Create minimal schema with raw_dump as bytea initially.
- // We insert the invalid bytes as bytea (no encoding validation), then flip
- // the column type to text via pg_attribute. This simulates how corrupted data
- // ends up in production (e.g., via server-side INSERT INTO...SELECT during
- // the 3.2 migration, or older PostgreSQL versions with laxer validation).
- setup := `
- CREATE TABLE sync_history (
- id bigserial PRIMARY KEY,
- created_at timestamptz NOT NULL DEFAULT now(),
- instance text NOT NULL,
- db_name text NOT NULL,
- metadata json NOT NULL DEFAULT '{}',
- raw_dump bytea NOT NULL DEFAULT ''::bytea
- );
-
- CREATE TABLE changelog (
- id bigserial PRIMARY KEY,
- created_at timestamptz NOT NULL DEFAULT now(),
- instance text NOT NULL,
- db_name text NOT NULL,
- status text NOT NULL,
- sync_history_id bigint REFERENCES sync_history(id),
- payload jsonb NOT NULL DEFAULT '{}'
- );
-
- CREATE TABLE task (
- id serial PRIMARY KEY,
- plan_id bigint NOT NULL
- );
-
- CREATE TABLE plan (
- id bigserial PRIMARY KEY,
- name text NOT NULL
- );
-
- -- Insert invalid UTF-8 as bytea (no encoding validation).
- -- 0xe4 0xb8 is a truncated 3-byte Chinese UTF-8 character (e.g. 中 = 0xe4 0xb8 0xad).
- INSERT INTO sync_history (instance, db_name, metadata, raw_dump)
- VALUES ('test-instance', 'test-db', '{}', '\xE4B8'::bytea);
-
- -- Flip column type from bytea to text via system catalog.
- -- bytea and text share the same varlena storage format, so the raw bytes
- -- are preserved but now treated as text — bypassing UTF-8 validation.
- UPDATE pg_attribute
- SET atttypid = 'text'::regtype
- WHERE attrelid = 'sync_history'::regclass
- AND attname = 'raw_dump';
-
- INSERT INTO changelog (instance, db_name, status, sync_history_id, payload)
- VALUES ('test-instance', 'test-db', 'DONE', 1, '{}');
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- // Reproduce the bug: LEFT() on text containing invalid UTF-8 triggers SQLSTATE 22021.
- // This mirrors the query that ListChangelogs generates when ShowFull is false.
- var schema string
- err = db.QueryRowContext(ctx,
- `SELECT COALESCE(LEFT(sh_cur.raw_dump, 512), '')
- FROM changelog
- LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id
- WHERE changelog.id = $1`,
- 1,
- ).Scan(&schema)
- assert.Error(t, err, "LEFT() on invalid UTF-8 text must fail — confirming the bug exists")
- if err != nil {
- assert.Contains(t, err.Error(), "22021", "error must be SQLSTATE 22021 (invalid byte sequence)")
- }
-
- // Verify the fix: reading the raw column without LEFT() succeeds because
- // a plain column reference does not trigger PostgreSQL's UTF-8 string-function validation.
- err = db.QueryRowContext(ctx,
- `SELECT COALESCE(sh_cur.raw_dump, '')
- FROM changelog
- LEFT JOIN sync_history sh_cur ON sh_cur.id = changelog.sync_history_id
- WHERE changelog.id = $1`,
- 1,
- ).Scan(&schema)
- require.NoError(t, err, "reading raw_dump without LEFT() must succeed even with invalid UTF-8")
-
- // After Go-side sanitization, the result must be valid UTF-8.
- sanitized := strings.ToValidUTF8(schema, "")
- assert.True(t, utf8.ValidString(sanitized), "sanitized schema must be valid UTF-8")
-}
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index ace3c59cf5..74fd26e243 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -1,21 +1,10 @@
package migrator
import (
- "context"
- "crypto/sha256"
- "encoding/hex"
- "fmt"
- "strings"
"testing"
- "time"
"github.com/blang/semver/v4"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
- _ "github.com/bytebase/bytebase/backend/plugin/db/pg"
- "github.com/bytebase/bytebase/backend/store"
)
func TestLatestVersion(t *testing.T) {
@@ -25,81 +14,6 @@ func TestLatestVersion(t *testing.T) {
require.Equal(t, "migration/3.23/0003##identity_provider_list_permission.sql", files[len(files)-1].path)
}
-func TestMigration3_23_1_IssueCommentThreads(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- // Minimal pre-3.23.1 shape of issue_comment.
- setup := `
- CREATE TABLE issue_comment (
- resource_id text NOT NULL,
- project text NOT NULL DEFAULT 'p1',
- issue_id integer NOT NULL DEFAULT 101,
- payload jsonb NOT NULL DEFAULT '{}'
- );
- CREATE UNIQUE INDEX idx_issue_comment_unique_resource_id ON issue_comment(resource_id);
- ALTER TABLE issue_comment ADD PRIMARY KEY (resource_id);
- INSERT INTO issue_comment (resource_id, payload) VALUES
- ('comment-only', '{"comment": "hello"}'),
- ('empty-comment', '{"comment": ""}'),
- ('empty-object', '{}'),
- ('hybrid', '{"comment": "approved", "approval": {}}'),
- ('event-only', '{"issueUpdate": {"toTitle": "t"}}'),
- ('legacy-event', '{"planSpecUpdate": {}}'),
- ('numeric-comment', '{"comment": 123}'),
- ('null-comment', '{"comment": null}'),
- ('scalar', '"comment"');
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- migration, err := migrationFS.ReadFile("migration/3.23/0001##issue_comment_thread.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(migration))
- require.NoError(t, err)
-
- // Legacy comments have no statement anchor, so none becomes a thread root.
- var rowCount, stateless int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT count(*), count(*) FILTER (WHERE parent_id IS NULL AND thread_state IS NULL)
- FROM issue_comment
- `).Scan(&rowCount, &stateless))
- require.Equal(t, 9, rowCount)
- require.Equal(t, rowCount, stateless, "the migration must not classify any legacy row as a thread root")
-
- var indexes int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT count(*) FROM pg_indexes
- WHERE indexname IN ('idx_issue_comment_parent_id', 'idx_issue_comment_open_thread')
- `).Scan(&indexes))
- require.Equal(t, 2, indexes)
-
- // On upgraded databases the explicit-column FK binds the pre-existing
- // duplicate unique index, not the primary key (this fixture recreates
- // that index order). Accepted for now; the deferred cleanup that drops
- // idx_issue_comment_unique_resource_id must swap the FK dependency and
- // flip this assertion to issue_comment_pkey.
- var referencedIndex string
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT referenced_index.relname
- FROM pg_constraint
- JOIN pg_class AS referenced_index ON referenced_index.oid = pg_constraint.conindid
- WHERE pg_constraint.conname = 'issue_comment_parent_id_fkey'
- AND pg_constraint.conrelid = 'issue_comment'::regclass
- `).Scan(&referencedIndex))
- require.Equal(t, "idx_issue_comment_unique_resource_id", referencedIndex)
-
- _, err = db.ExecContext(ctx, `INSERT INTO issue_comment (resource_id, payload, parent_id) VALUES ('reply', '{"comment": "re"}', 'comment-only')`)
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, `INSERT INTO issue_comment (resource_id, payload, parent_id) VALUES ('orphan', '{"comment": "re"}', 'missing')`)
- require.Error(t, err, "parent_id must reference an existing comment")
- _, err = db.ExecContext(ctx, `UPDATE issue_comment SET thread_state = 'INVALID' WHERE resource_id = 'comment-only'`)
- require.Error(t, err, "thread_state must reject unknown values")
-}
-
func TestVersionUnique(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
@@ -114,1243 +28,3 @@ func TestVersionUnique(t *testing.T) {
versions[file.version.String()] = struct{}{}
}
}
-
-func TestMigration3_23_2BackfillsLastPlanEditorApproval(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
- db := container.GetDB()
- _, err := db.ExecContext(ctx, `
- CREATE TABLE project (
- resource_id text PRIMARY KEY,
- deleted boolean NOT NULL DEFAULT FALSE,
- setting jsonb NOT NULL DEFAULT '{}'
- );
- CREATE TABLE plan (id bigint NOT NULL, project text NOT NULL, creator text NOT NULL);
- INSERT INTO project (resource_id, deleted) VALUES ('active', false), ('deleted', true);
- `)
- require.NoError(t, err)
- statement, err := migrationFS.ReadFile("migration/3.23/0002##plan_last_editor_approval.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- for _, projectID := range []string{"active", "deleted"} {
- var enabled bool
- err := db.QueryRowContext(ctx, `
- SELECT (setting->>'allowLastPlanEditorApproval')::boolean
- FROM project
- WHERE resource_id = $1`, projectID).Scan(&enabled)
- require.NoError(t, err)
- require.True(t, enabled)
- }
- _, err = db.ExecContext(ctx, "UPDATE project SET deleted = false WHERE resource_id = 'deleted'")
- require.NoError(t, err)
- var restoredEnabled bool
- err = db.QueryRowContext(ctx, `
- SELECT (setting->>'allowLastPlanEditorApproval')::boolean
- FROM project
- WHERE resource_id = 'deleted' AND NOT deleted`).Scan(&restoredEnabled)
- require.NoError(t, err)
- require.True(t, restoredEnabled)
- var nullable string
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT is_nullable
- FROM information_schema.columns
- WHERE table_name = 'plan' AND column_name = 'last_plan_editor'`).Scan(&nullable))
- require.Equal(t, "YES", nullable)
-}
-
-func TestMigration3_22_12MigratesSampleProjectInstance(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
- db := container.GetDB()
- _, err := db.ExecContext(ctx, `
- CREATE TABLE workspace (resource_id text PRIMARY KEY);
- INSERT INTO workspace (resource_id) VALUES ('workspace-a');
- `)
- require.NoError(t, err)
- statement, err := migrationFS.ReadFile("migration/3.22/0011##sample_project_instance.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
- createdAt := time.Date(2026, time.August, 1, 12, 0, 0, 0, time.UTC)
- expiresAt := createdAt.Add(7 * 24 * time.Hour)
- _, err = db.ExecContext(ctx, `
- INSERT INTO sample_project_instance (
- workspace, project, instance, db_name, role_name, replica_id,
- created_at, expires_at
- ) VALUES ('workspace-a', 'project-a', 'sample-a', 'db-a', 'role-a', 'replica-a', $1, $2)
- `, createdAt, expiresAt)
- require.NoError(t, err)
- statement, err = migrationFS.ReadFile("migration/3.22/0013##sample_instance_setup.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- var instance string
- var activatedAt time.Time
- err = db.QueryRowContext(ctx, `
- SELECT activated_at, payload->>'instanceId'
- FROM sample_instance_setup
- WHERE workspace = 'workspace-a'
- `).Scan(&activatedAt, &instance)
- require.NoError(t, err)
- require.Equal(t, createdAt, activatedAt)
- require.Equal(t, "sample-a", instance)
-
- var oldTable *string
- require.NoError(t, db.QueryRowContext(ctx, "SELECT to_regclass('sample_project_instance')").Scan(&oldTable))
- require.Nil(t, oldTable)
-}
-
-// TestMigration3_16_2_TaskRunLogDuplicateTimestamps verifies that the 3.16.2
-// id-column cleanup succeeds on legacy task_run_log data holding duplicate
-// (task_run_id, created_at) pairs, which were legal under the old id primary
-// key. Regression test for BYT-10035.
-func TestMigration3_16_2_TaskRunLogDuplicateTimestamps(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- // Minimal pre-3.16.2 shapes for every table the migration touches.
- setup := `
- CREATE TABLE project (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);
- CREATE TABLE instance (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);
- CREATE TABLE db (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, name TEXT NOT NULL);
- CREATE TABLE setting (id BIGSERIAL PRIMARY KEY, name TEXT NOT NULL);
- CREATE TABLE policy (id BIGSERIAL PRIMARY KEY, resource_type TEXT NOT NULL, resource TEXT NOT NULL, type TEXT NOT NULL);
- CREATE TABLE idp (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);
- CREATE TABLE role (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);
- CREATE TABLE db_schema (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, db_name TEXT NOT NULL);
- CREATE TABLE db_group (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, resource_id TEXT NOT NULL);
- CREATE TABLE release (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, train TEXT NOT NULL, iteration INT NOT NULL);
- CREATE TABLE task_run_log (
- id BIGSERIAL PRIMARY KEY,
- task_run_id INTEGER NOT NULL,
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
- payload JSONB NOT NULL DEFAULT '{}'
- );
- CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);
-
- INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES
- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),
- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),
- (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.16/0002##drop_unused_id_columns.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- var rowCount int
- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_run_log`).Scan(&rowCount))
- require.Equal(t, 3, rowCount)
-
- var hasPkey bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)
- `).Scan(&hasPkey))
- require.False(t, hasPkey)
-
- var hasIDColumn bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'task_run_log' AND column_name = 'id')
- `).Scan(&hasIDColumn))
- require.False(t, hasIDColumn)
-
- var hasIndex bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_task_run_id_created_at')
- `).Scan(&hasIndex))
- require.True(t, hasIndex)
-
- var hasOldIndex bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_task_run_id')
- `).Scan(&hasOldIndex))
- require.False(t, hasOldIndex)
-}
-
-// TestMigration3_17_1_TaskRunLogDuplicateTimestamps verifies that the 3.17.1
-// project-scoping migration accepts duplicate (task_run_id, created_at) pairs:
-// it must backfill project and index the table without imposing uniqueness.
-// Regression test for BYT-10035.
-func TestMigration3_17_1_TaskRunLogDuplicateTimestamps(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- // Minimal post-3.16.2 shapes for the plan-chain tables the migration touches;
- // task_run_log has no id column and no primary key at this point.
- setup := `
- CREATE TABLE project (resource_id TEXT PRIMARY KEY);
- CREATE TABLE plan (id BIGINT PRIMARY KEY, project TEXT NOT NULL);
- CREATE TABLE issue (id BIGINT PRIMARY KEY, project TEXT NOT NULL, plan_id BIGINT);
- CREATE TABLE task (id INT PRIMARY KEY, plan_id BIGINT NOT NULL, environment TEXT NOT NULL);
- CREATE TABLE task_run (id INT PRIMARY KEY, task_id INT NOT NULL, attempt INT NOT NULL);
- CREATE TABLE plan_check_run (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);
- CREATE TABLE plan_webhook_delivery (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);
- CREATE TABLE task_run_log (
- task_run_id INTEGER NOT NULL,
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
- payload JSONB NOT NULL DEFAULT '{}'
- );
- CREATE INDEX idx_task_run_log_task_run_id_created_at ON task_run_log(task_run_id, created_at);
-
- INSERT INTO project VALUES ('proj-a');
- INSERT INTO plan VALUES (1, 'proj-a');
- INSERT INTO task VALUES (10, 1, 'environments/prod');
- INSERT INTO task_run VALUES (100, 10, 1);
- INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES
- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),
- (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),
- (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.17/0001##add_project_to_plan_chain.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- var backfilledCount int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT COUNT(*) FROM task_run_log WHERE project = 'proj-a'
- `).Scan(&backfilledCount))
- require.Equal(t, 3, backfilledCount)
-
- var hasPkey bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)
- `).Scan(&hasPkey))
- require.False(t, hasPkey)
-
- var hasIndex bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')
- `).Scan(&hasIndex))
- require.True(t, hasIndex)
-
- var oldIndexCount int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public' AND indexname IN ('idx_task_run_log_task_run_id', 'idx_task_run_log_task_run_id_created_at')
- `).Scan(&oldIndexCount))
- require.Zero(t, oldIndexCount)
-}
-
-// TestMigration3_22_4_DropTaskRunLogPkey verifies that the pkey drop converges
-// every historical shape — the 2-column PK left by upgrading through the
-// original 3.16.2, the 3-column PK of fresh installs, and the no-PK state left
-// by a release-branch backport of the 3.16.2 fix — is idempotent, and that
-// same-microsecond entries insert afterwards instead of colliding.
-func TestMigration3_22_4_DropTaskRunLogPkey(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- statement, err := migrationFS.ReadFile("migration/3.22/0004##drop_task_run_log_pkey.sql")
- require.NoError(t, err)
-
- const tableDDL = `
- CREATE TABLE task_run_log (
- project TEXT NOT NULL,
- task_run_id INTEGER NOT NULL,
- created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
- payload JSONB NOT NULL DEFAULT '{}'`
-
- scenarios := []struct {
- name string
- setup string
- }{
- {
- "2-col pkey from the original 3.16.2",
- tableDDL + `, PRIMARY KEY (task_run_id, created_at));
- CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);`,
- },
- {
- "3-col pkey from a fresh install",
- tableDDL + `, PRIMARY KEY (project, task_run_id, created_at));`,
- },
- {
- "no pkey from a release-branch backport of the 3.16.2 fix",
- tableDDL + `);
- CREATE INDEX idx_task_run_log_task_run_id_created_at ON task_run_log(task_run_id, created_at);`,
- },
- }
- for _, scenario := range scenarios {
- _, err := db.ExecContext(ctx, `DROP TABLE IF EXISTS task_run_log;`+scenario.setup)
- require.NoError(t, err, scenario.name)
-
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err, scenario.name)
- // Re-running must be a no-op.
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err, scenario.name)
-
- var hasPkey bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)
- `).Scan(&hasPkey))
- require.False(t, hasPkey, scenario.name)
-
- var hasIndex bool
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')
- `).Scan(&hasIndex))
- require.True(t, hasIndex, scenario.name)
-
- var oldIndexCount int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT COUNT(*) FROM pg_indexes WHERE schemaname = 'public' AND indexname IN ('idx_task_run_log_task_run_id', 'idx_task_run_log_task_run_id_created_at')
- `).Scan(&oldIndexCount))
- require.Zero(t, oldIndexCount, scenario.name)
-
- _, err = db.ExecContext(ctx, `
- INSERT INTO task_run_log (project, task_run_id, created_at) VALUES
- ('proj-a', 100, '2026-01-01 00:00:00.000001+00'),
- ('proj-a', 100, '2026-01-01 00:00:00.000001+00')`)
- require.NoError(t, err, "same-microsecond entries must both insert")
- }
-}
-
-func TestMigration3_21_2_MigrateInstanceSyncDatabases(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- setup := `
- CREATE TABLE instance (
- resource_id TEXT PRIMARY KEY,
- metadata JSONB NOT NULL DEFAULT '{}'
- );
-
- INSERT INTO instance (resource_id, metadata) VALUES
- (
- 'instance-with-selected-databases',
- '{"engine":"POSTGRES","syncDatabases":["db1","db2"]}'
- ),
- (
- 'instance-with-empty-databases',
- '{"engine":"POSTGRES","syncDatabases":[]}'
- ),
- (
- 'instance-with-new-shape',
- '{"engine":"POSTGRES","syncDatabases":{"databases":["db3"]}}'
- ),
- (
- 'instance-without-sync-databases',
- '{"engine":"POSTGRES"}'
- );
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.21/0002##migrate_instance_sync_databases.sql")
- require.NoError(t, err)
-
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- getMetadata := func(resourceID string) string {
- t.Helper()
- var metadata string
- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)
- require.NoError(t, err)
- return metadata
- }
-
- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":["db1","db2"]}}`, getMetadata("instance-with-selected-databases"))
- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":[]}}`, getMetadata("instance-with-empty-databases"))
- require.JSONEq(t, `{"engine":"POSTGRES","syncDatabases":{"databases":["db3"]}}`, getMetadata("instance-with-new-shape"))
- require.JSONEq(t, `{"engine":"POSTGRES"}`, getMetadata("instance-without-sync-databases"))
-}
-
-func TestMigration3_22_2_AddInstanceProject(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
- _, err := db.ExecContext(ctx, `
- CREATE TABLE project (
- resource_id TEXT PRIMARY KEY
- );
- CREATE TABLE instance (
- resource_id TEXT PRIMARY KEY,
- metadata JSONB NOT NULL DEFAULT '{}'
- );
- INSERT INTO instance (resource_id) VALUES ('legacy-workspace-instance');
- `)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.22/0002##add_instance_project.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- var projectID *string
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT project FROM instance WHERE resource_id = 'legacy-workspace-instance'
- `).Scan(&projectID))
- require.Nil(t, projectID)
-
- var indexDefinition string
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT indexdef
- FROM pg_indexes
- WHERE schemaname = 'public' AND indexname = 'idx_instance_project'
- `).Scan(&indexDefinition))
- require.Contains(t, indexDefinition, "WHERE (project IS NOT NULL)")
-}
-
-func TestMigration3_21_3_MigrateGCPDataSourceFields(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- setup := `
- CREATE TABLE instance (
- resource_id TEXT PRIMARY KEY,
- metadata JSONB NOT NULL DEFAULT '{}'
- );
-
- INSERT INTO instance (resource_id, metadata) VALUES
- (
- 'spanner-legacy',
- '{"engine":"SPANNER","dataSources":[{"id":"admin","host":"projects/my-proj/instances/my-inst"}]}'
- ),
- (
- 'spanner-multi-data-source',
- '{"engine":"SPANNER","dataSources":[{"id":"admin","host":"projects/my-proj/instances/my-inst"},{"id":"ro-1","type":"READ_ONLY","host":"projects/my-proj/instances/my-inst"},{"id":"ro-2","type":"READ_ONLY","host":"projects/my-proj/instances/my-inst"}]}'
- ),
- (
- 'spanner-new-shape',
- '{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst","host":"spanner-nonprod.p.googleapis.com"}]}'
- ),
- (
- 'spanner-no-data-sources',
- '{"engine":"SPANNER","dataSources":[]}'
- ),
- (
- 'bigquery-legacy',
- '{"engine":"BIGQUERY","dataSources":[{"id":"admin","host":"my-proj"}]}'
- ),
- (
- 'bigquery-new-shape',
- '{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj","host":"bigquery-nonprod.p.googleapis.com"}]}'
- ),
- (
- 'postgres-untouched',
- '{"engine":"POSTGRES","dataSources":[{"id":"admin","host":"localhost","port":"5432"}]}'
- );
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.21/0003##migrate_gcp_data_source_fields.sql")
- require.NoError(t, err)
-
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- getMetadata := func(resourceID string) string {
- t.Helper()
- var metadata string
- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)
- require.NoError(t, err)
- return metadata
- }
-
- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst"}]}`, getMetadata("spanner-legacy"))
- // JSONEq treats arrays as ordered: this also asserts the data source order is preserved.
- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst"},{"id":"ro-1","type":"READ_ONLY","projectId":"my-proj","instanceId":"my-inst"},{"id":"ro-2","type":"READ_ONLY","projectId":"my-proj","instanceId":"my-inst"}]}`, getMetadata("spanner-multi-data-source"))
- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[{"id":"admin","projectId":"my-proj","instanceId":"my-inst","host":"spanner-nonprod.p.googleapis.com"}]}`, getMetadata("spanner-new-shape"))
- require.JSONEq(t, `{"engine":"SPANNER","dataSources":[]}`, getMetadata("spanner-no-data-sources"))
- require.JSONEq(t, `{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj"}]}`, getMetadata("bigquery-legacy"))
- require.JSONEq(t, `{"engine":"BIGQUERY","dataSources":[{"id":"admin","projectId":"my-proj","host":"bigquery-nonprod.p.googleapis.com"}]}`, getMetadata("bigquery-new-shape"))
- require.JSONEq(t, `{"engine":"POSTGRES","dataSources":[{"id":"admin","host":"localhost","port":"5432"}]}`, getMetadata("postgres-untouched"))
-}
-
-func TestMigration3_21_1_BackfillUIPlanDraftIssues(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- tx, err := container.GetDB().BeginTx(ctx, nil)
- require.NoError(t, err)
- defer tx.Rollback()
- var migrationTime time.Time
- require.NoError(t, tx.QueryRowContext(ctx, `SELECT CURRENT_TIMESTAMP`).Scan(&migrationTime))
-
- _, err = tx.ExecContext(ctx, `
- CREATE TABLE project (
- resource_id TEXT PRIMARY KEY
- );
- CREATE TABLE plan (
- id BIGINT NOT NULL,
- deleted BOOLEAN NOT NULL DEFAULT FALSE,
- creator TEXT NOT NULL,
- created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- project TEXT NOT NULL,
- name TEXT NOT NULL,
- description TEXT NOT NULL,
- config JSONB NOT NULL,
- PRIMARY KEY (project, id)
- );
- CREATE TABLE issue (
- id BIGINT NOT NULL,
- creator TEXT NOT NULL,
- created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- project TEXT NOT NULL,
- plan_id BIGINT,
- name TEXT NOT NULL,
- status TEXT NOT NULL,
- type TEXT NOT NULL,
- description TEXT NOT NULL DEFAULT '',
- payload JSONB NOT NULL DEFAULT '{}',
- ts_vector TSVECTOR,
- PRIMARY KEY (project, id),
- UNIQUE (project, plan_id)
- );
-
- INSERT INTO project (resource_id) VALUES ('project-a'), ('project-b');
- INSERT INTO plan (id, creator, created_at, project, name, description, config) VALUES
- (1, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '29 days', 'project-a', 'recent change', 'change description',
- '{"specs":[{"changeDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),
- (2, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days', 'project-a', 'boundary create', 'create description',
- '{"specs":[{"createDatabaseConfig":{}}]}'),
- (3, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'release GitOps', '',
- '{"specs":[{"changeDatabaseConfig":{"release":"projects/project-a/releases/release-a"}}]}'),
- (4, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'export', '',
- '{"specs":[{"exportDataConfig":{}}]}'),
- (5, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'mixed', '',
- '{"specs":[{"createDatabaseConfig":{}},{"changeDatabaseConfig":{}}]}'),
- (6, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'deleted', '',
- '{"specs":[{"changeDatabaseConfig":{}}]}'),
- (7, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '30 days 1 microsecond', 'project-a', 'old', '',
- '{"specs":[{"changeDatabaseConfig":{}}]}'),
- (8, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'linked', '',
- '{"specs":[{"changeDatabaseConfig":{}}]}'),
- (9, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-b', 'other project change', 'other description',
- '{"specs":[{"changeDatabaseConfig":{}}]}'),
- (10, '[email protected]', CURRENT_TIMESTAMP - INTERVAL '1 day', 'project-a', 'rolled out', '',
- '{"specs":[{"changeDatabaseConfig":{}}],"hasRollout":true}');
- UPDATE plan SET deleted = TRUE WHERE project = 'project-a' AND id = 6;
- INSERT INTO issue (id, creator, project, plan_id, name, status, type, payload)
- VALUES (150, '[email protected]', 'project-a', 8, 'existing issue', 'OPEN', 'DATABASE_CHANGE', '{}');
- `)
- require.NoError(t, err)
- require.NoError(t, tx.Commit())
-
- conn, err := container.GetDB().Conn(ctx)
- require.NoError(t, err)
- defer conn.Close()
- var migrationPID int
- require.NoError(t, conn.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&migrationPID))
-
- createPlanTx, err := container.GetDB().BeginTx(ctx, nil)
- require.NoError(t, err)
- defer createPlanTx.Rollback()
- var lockedProjectID string
- require.NoError(t, createPlanTx.QueryRowContext(ctx, `
- SELECT resource_id FROM project WHERE resource_id = 'project-a' FOR UPDATE`).Scan(&lockedProjectID))
- _, err = createPlanTx.ExecContext(ctx, `
- INSERT INTO plan (id, creator, created_at, project, name, description, config)
- VALUES (11, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'late commit', 'late description',
- '{"specs":[{"changeDatabaseConfig":{}}]}')`)
- require.NoError(t, err)
-
- discoveryDone := make(chan error, 1)
- go func() {
- discoveryDone <- migrate3_21_1At(ctx, conn, migrationTime)
- }()
- require.Eventually(t, func() bool {
- var waiting bool
- err := container.GetDB().QueryRowContext(ctx, `
- SELECT wait_event_type = 'Lock'
- FROM pg_stat_activity
- WHERE pid = $1`, migrationPID).Scan(&waiting)
- return err == nil && waiting
- }, 5*time.Second, 10*time.Millisecond)
- var hasPlanTableLock bool
- require.NoError(t, container.GetDB().QueryRowContext(ctx, `
- SELECT EXISTS (
- SELECT 1
- FROM pg_locks
- WHERE pid = $1
- AND locktype = 'relation'
- AND relation = 'plan'::regclass
- AND mode = 'ShareLock'
- )`, migrationPID).Scan(&hasPlanTableLock))
- assert.False(t, hasPlanTableLock)
- require.NoError(t, createPlanTx.Commit())
- require.NoError(t, <-discoveryDone)
-
- rows, err := container.GetDB().QueryContext(ctx, `
- SELECT
- plan.name,
- issue.creator,
- issue.name,
- issue.description,
- issue.status,
- issue.type,
- COALESCE((issue.payload->>'draft')::boolean, FALSE),
- COALESCE(jsonb_array_length(issue.payload->'labels'), 0),
- issue.created_at = $1,
- issue.ts_vector::TEXT
- FROM issue
- JOIN plan ON plan.project = issue.project AND plan.id = issue.plan_id
- WHERE COALESCE((issue.payload->>'draft')::boolean, FALSE)
- ORDER BY plan.project, plan.id
- `, migrationTime)
- require.NoError(t, err)
- defer rows.Close()
-
- type draft struct {
- creator string
- name string
- description string
- status string
- issueType string
- draft bool
- labelCount int
- createdAtMigration bool
- searchVector string
- }
- got := make(map[string]draft)
- for rows.Next() {
- var planName string
- var value draft
- require.NoError(t, rows.Scan(
- &planName,
- &value.creator,
- &value.name,
- &value.description,
- &value.status,
- &value.issueType,
- &value.draft,
- &value.labelCount,
- &value.createdAtMigration,
- &value.searchVector,
- ))
- got[planName] = value
- }
- require.NoError(t, rows.Err())
-
- require.Equal(t, map[string]draft{
- "recent change": {
- creator: "[email protected]", name: "recent change", description: "change description",
- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,
- searchVector: "'change':2,3 'description':4 'recent':1",
- },
- "boundary create": {
- creator: "[email protected]", name: "boundary create", description: "create description",
- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,
- searchVector: "'boundary':1 'create':2,3 'description':4",
- },
- "other project change": {
- creator: "[email protected]", name: "other project change", description: "other description",
- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,
- searchVector: "'change':3 'description':5 'other':1,4 'project':2",
- },
- "late commit": {
- creator: "[email protected]", name: "late commit", description: "late description",
- status: "OPEN", issueType: "DATABASE_CHANGE", draft: true, createdAtMigration: true,
- searchVector: "'commit':2 'description':4 'late':1,3",
- },
- }, got)
-
- _, err = container.GetDB().ExecContext(ctx, `
- INSERT INTO plan (id, creator, created_at, project, name, description, config)
- SELECT id, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'paged plan', '',
- '{"specs":[{"changeDatabaseConfig":{}}]}'
- FROM generate_series(1000, 1100) AS id`)
- require.NoError(t, err)
- require.NoError(t, migrate3_21_1At(ctx, conn, migrationTime))
- var pagedIssueCount int
- require.NoError(t, container.GetDB().QueryRowContext(ctx, `
- SELECT COUNT(*)
- FROM issue
- WHERE project = 'project-a' AND plan_id BETWEEN 1000 AND 1100`).Scan(&pagedIssueCount))
- require.Equal(t, 101, pagedIssueCount)
-
- _, err = container.GetDB().ExecContext(ctx, `
- INSERT INTO plan (id, creator, created_at, project, name, description, config)
- VALUES (12, '[email protected]', CURRENT_TIMESTAMP, 'project-a', 'racing rollout', '',
- '{"specs":[{"changeDatabaseConfig":{}}]}')`)
- require.NoError(t, err)
-
- rolloutTx, err := container.GetDB().BeginTx(ctx, nil)
- require.NoError(t, err)
- defer rolloutTx.Rollback()
- require.NoError(t, store.AcquireAdvisoryXactLockWithStringKey(
- ctx,
- rolloutTx,
- store.AdvisoryLockKeyPlanIssueRollout,
- "project-a/12",
- ))
-
- raceConn, err := container.GetDB().Conn(ctx)
- require.NoError(t, err)
- defer raceConn.Close()
- migrationDone := make(chan error, 1)
- go func() {
- migrationDone <- migrate3_21_1At(ctx, raceConn, migrationTime)
- }()
- require.Eventually(t, func() bool {
- var waiting bool
- err := container.GetDB().QueryRowContext(ctx, `
- SELECT EXISTS (
- SELECT 1
- FROM pg_locks
- WHERE locktype = 'advisory'
- AND classid = $1
- AND NOT granted
- )`, store.AdvisoryLockKeyPlanIssueRollout).Scan(&waiting)
- return err == nil && waiting
- }, 5*time.Second, 10*time.Millisecond)
- _, err = rolloutTx.ExecContext(ctx, `
- UPDATE plan
- SET config = jsonb_set(config, '{hasRollout}', 'true'::jsonb, true)
- WHERE project = 'project-a' AND id = 12`)
- require.NoError(t, err)
- require.NoError(t, rolloutTx.Commit())
- require.NoError(t, <-migrationDone)
-
- var racingIssueCount int
- require.NoError(t, container.GetDB().QueryRowContext(ctx, `
- SELECT COUNT(*) FROM issue WHERE project = 'project-a' AND plan_id = 12`).Scan(&racingIssueCount))
- require.Zero(t, racingIssueCount)
-}
-
-func TestMigration3_17_15_DedupeReadOnlyDataSources(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- setup := `
- CREATE TABLE instance (
- resource_id TEXT PRIMARY KEY,
- metadata JSONB NOT NULL DEFAULT '{}'
- );
-
- INSERT INTO instance (resource_id, metadata) VALUES
- (
- 'instance-with-duplicate-read-only',
- '{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"},{"id":"read-only-2","type":"READ_ONLY","username":"readonly-2"}]}'
- ),
- (
- 'instance-with-single-read-only',
- '{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}'
- ),
- (
- 'instance-with-non-array-data-sources',
- '{"dataSources":{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}}'
- ),
- (
- 'instance-without-data-sources',
- '{"engine":"POSTGRES"}'
- );
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- statement, err := migrationFS.ReadFile("migration/3.17/0015##dedupe_read_only_data_sources.sql")
- require.NoError(t, err)
-
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- getMetadata := func(resourceID string) string {
- t.Helper()
- var metadata string
- err := db.QueryRowContext(ctx, `SELECT metadata::text FROM instance WHERE resource_id = $1`, resourceID).Scan(&metadata)
- require.NoError(t, err)
- return metadata
- }
-
- require.JSONEq(t, `{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}`, getMetadata("instance-with-duplicate-read-only"))
- require.JSONEq(t, `{"dataSources":[{"id":"admin","type":"ADMIN","username":"admin"},{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}]}`, getMetadata("instance-with-single-read-only"))
- require.JSONEq(t, `{"dataSources":{"id":"read-only-1","type":"READ_ONLY","username":"readonly-1"}}`, getMetadata("instance-with-non-array-data-sources"))
- require.JSONEq(t, `{"engine":"POSTGRES"}`, getMetadata("instance-without-data-sources"))
-}
-
-// TestMigration3_7_20_ScalarTaskUpdateTasks verifies that the migration 3.7.20
-// UPDATE on issue_comment handles scalar (non-array) taskUpdate.tasks values
-// without error. Regression test for "cannot get array length of a scalar".
-func TestMigration3_7_20_ScalarTaskUpdateTasks(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
-
- // Create minimal schema.
- setup := `
- CREATE TABLE stage (
- id INT PRIMARY KEY,
- environment TEXT NOT NULL
- );
- INSERT INTO stage (id, environment) VALUES (101, 'environments/prod');
-
- CREATE OR REPLACE FUNCTION update_stage_reference(resource_path text) RETURNS text AS $$
- DECLARE
- stage_match text;
- stage_id int;
- environment_id text;
- BEGIN
- IF resource_path !~ '/stages/[0-9]+' THEN
- RETURN resource_path;
- END IF;
- stage_match := substring(resource_path from '/stages/([0-9]+)');
- IF stage_match IS NULL THEN
- RETURN resource_path;
- END IF;
- stage_id := stage_match::int;
- SELECT s.environment INTO environment_id FROM stage s WHERE s.id = stage_id;
- IF environment_id IS NULL THEN
- RETURN resource_path;
- END IF;
- RETURN regexp_replace(resource_path, '/stages/' || stage_id, '/stages/' || environment_id);
- END;
- $$ LANGUAGE plpgsql;
-
- CREATE TABLE issue_comment (
- id SERIAL PRIMARY KEY,
- payload JSONB NOT NULL
- );
-
- INSERT INTO issue_comment (payload) VALUES
- ('{"taskUpdate":{"tasks":["projects/p1/rollouts/1/stages/101/tasks/1"]}}'),
- ('{"taskUpdate":{"tasks":"projects/p1/rollouts/1/stages/101/tasks/1"}}'),
- ('{"taskUpdate":{"tasks":null}}');
- `
- _, err := db.ExecContext(ctx, setup)
- require.NoError(t, err)
-
- // Run the exact UPDATE from migration 3.7.20 with the fixed WHERE clause.
- migrate := `
- UPDATE issue_comment
- SET payload = jsonb_set(
- payload,
- '{taskUpdate,tasks}',
- (
- SELECT jsonb_agg(update_stage_reference(task_ref))
- FROM jsonb_array_elements_text(payload->'taskUpdate'->'tasks') AS task_ref
- )
- )
- WHERE payload->'taskUpdate' IS NOT NULL
- AND jsonb_typeof(payload->'taskUpdate'->'tasks') = 'array'
- AND CASE WHEN jsonb_typeof(payload->'taskUpdate'->'tasks') = 'array'
- THEN jsonb_array_length(payload->'taskUpdate'->'tasks') > 0
- ELSE false END;
- `
- _, err = db.ExecContext(ctx, migrate)
- require.NoError(t, err, "migration UPDATE must not fail on scalar tasks values")
-
- // Verify: valid array row was rewritten with environment ID.
- var arrayPayload string
- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 1`).Scan(&arrayPayload)
- require.NoError(t, err)
- assert.Contains(t, arrayPayload, "environments/prod", "array row should have rewritten stage reference")
- assert.NotContains(t, arrayPayload, "stages/101", "array row should no longer have numeric stage ID")
-
- // Verify: scalar row was NOT modified.
- var scalarPayload string
- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 2`).Scan(&scalarPayload)
- require.NoError(t, err)
- assert.Contains(t, scalarPayload, "stages/101", "scalar row should be unchanged")
-
- // Verify: null row was NOT modified.
- var nullPayload string
- err = db.QueryRowContext(ctx, `SELECT payload::text FROM issue_comment WHERE id = 3`).Scan(&nullPayload)
- require.NoError(t, err)
- assert.Contains(t, nullPayload, "null", "null row should be unchanged")
-}
-
-// TestMigration3_22_5_ScopeSheetBlob verifies the sheet_blob_ref backfill.
-//
-// The four project-scoped sources (plan, task, release, plan_check_run) each
-// yield one (project, hash) ref; references naming a hash with no blob are
-// skipped; unreferenced blobs end with zero refs; and a hash claimed by more
-// than one project is recorded per project so the multi-project audit reports
-// it — detected, not silently merged.
-//
-// The revision branch derives the authoring project from corroborated
-// provenance (never db.project). A corroborating row is almost always itself
-// a source granting the same (project, hash), so the observable probe for
-// each corroboration rule is the one additive shape: task-run provenance
-// whose task routes its hash through a release in a DIFFERENT project. There
-// the revision branch grants (task-run project, hash), which no source scan
-// produces, so a rule failure is visible as that ref's absence.
-func TestMigration3_22_5_ScopeSheetBlob(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
- db := container.GetDB()
-
- _, err := db.ExecContext(ctx, `
- CREATE TABLE project (resource_id TEXT PRIMARY KEY);
- CREATE TABLE sheet_blob (sha256 BYTEA NOT NULL PRIMARY KEY, content TEXT NOT NULL);
- CREATE TABLE plan (project TEXT NOT NULL, config JSONB NOT NULL DEFAULT '{}');
- CREATE TABLE task (project TEXT NOT NULL, id BIGINT NOT NULL, payload JSONB NOT NULL DEFAULT '{}');
- CREATE TABLE task_run (project TEXT NOT NULL, id BIGINT NOT NULL, task_id INTEGER NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now());
- CREATE TABLE release (project TEXT NOT NULL, release_id TEXT NOT NULL DEFAULT '', created_at TIMESTAMPTZ NOT NULL DEFAULT now(), payload JSONB NOT NULL DEFAULT '{}');
- CREATE TABLE plan_check_run (project TEXT NOT NULL, result JSONB NOT NULL DEFAULT '{}');
- CREATE TABLE revision (resource_id TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text, instance TEXT NOT NULL, db_name TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), payload JSONB NOT NULL DEFAULT '{}');
- INSERT INTO project VALUES
- ('p1'), ('p2'), ('p3'), ('p4'),
- ('rel-ok'), ('rel-upper'), ('tr-ok'), ('rel-old'), ('tr-late'), ('rel-mismatch'), ('tr-mismatch'),
- ('rel-dup'), ('tr-dup'), ('rel-task'), ('rel-taskid'), ('tr-taskid'), ('tr-direct'),
- ('rel-prefer'), ('rel-nest-prefer'), ('tr-prefer'),
- ('rel-newer'), ('rel-nest-fb'), ('tr-fb');
- `)
- require.NoError(t, err)
-
- sheetHex := func(content string) string {
- h := sha256.Sum256([]byte(content))
- return hex.EncodeToString(h[:])
- }
- blobs := map[string]string{}
- for _, name := range []string{
- "plan", "task", "release", "pcr", "shared", "orphan",
- "rev-rel", "rev-upper", "rev-nest", "rev-late", "rev-mismatch", "rev-other",
- "rev-dup", "rev-taskid", "rev-direct", "rev-prefer", "rev-fb", "rev-ghost", "rev-bare",
- "rev-overflow", "rev-stale-stamp", "rev-empty-stamp",
- } {
- content := "SELECT '" + name + "';"
- blobs[name] = sheetHex(content)
- _, err := db.ExecContext(ctx,
- "INSERT INTO sheet_blob VALUES (decode($1, 'hex'), $2)", blobs[name], content)
- require.NoError(t, err)
- }
- missingBlob := strings.Repeat("f", 64)
-
- const (
- older = "2026-01-01 00:00:00+00" // predates every revision
- rev = "2026-02-01 00:00:00+00" // every revision's created_at
- newer = "2026-03-01 00:00:00+00" // postdates every revision
- )
- taskRunName := func(project string, taskID, taskRunID int) string {
- return fmt.Sprintf("projects/%s/plans/1/rollout/stages/prod/tasks/%d/taskRuns/%d", project, taskID, taskRunID)
- }
-
- type seed struct {
- query string
- args []any
- }
- seeds := []seed{
- // The four project-scoped sources.
- {"INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",
- []any{fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}},{"createDatabaseConfig":{}}]}`, blobs["plan"])}},
- // The same hash referenced from two projects: honest dedup or a
- // laundered reference — indistinguishable, so both refs are
- // backfilled and the multi-project audit reports the hash.
- {"INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",
- []any{fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}}]}`, blobs["shared"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('p2', 1, $1::jsonb)",
- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["task"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('p2', 2, $1::jsonb)",
- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["shared"])}},
- // A reference naming a hash with no blob is skipped by the EXISTS
- // guard, not a foreign-key abort.
- {"INSERT INTO task (project, id, payload) VALUES ('p2', 3, $1::jsonb)",
- []any{fmt.Sprintf(`{"sheetSha256":%q}`, missingBlob)}},
- {"INSERT INTO task (project, id, payload) VALUES ('p2', 4, '{}')", nil},
- {"INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r1', $1::jsonb)",
- []any{fmt.Sprintf(`{"files":[{"path":"a.sql","sheetSha256":%q},{"path":"no-sheet.sql"}]}`, blobs["release"])}},
- {"INSERT INTO plan_check_run (project, result) VALUES ('p4', $1::jsonb)",
- []any{fmt.Sprintf(`{"results":[{"sheetSha256":%q},{"title":"no sheet"}]}`, blobs["pcr"])}},
-
- // Null and malformed shapes: every one must expand to nothing, not
- // abort. JSON null in place of an array is the case COALESCE alone
- // would pass through to jsonb_array_elements.
- {"INSERT INTO plan (project, config) VALUES ('p1', '{}')", nil},
- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": null}')`, nil},
- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": []}')`, nil},
- {`INSERT INTO plan (project, config) VALUES ('p1', '{"specs": ["garbage", null]}')`, nil},
- {"INSERT INTO task (project, id, payload) VALUES ('p2', 5, 'null'::jsonb)", nil},
- {`INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r-null', '{"files": null}')`, nil},
- {`INSERT INTO release (project, release_id, payload) VALUES ('p3', 'r-nullelem', '{"files": [null]}')`, nil},
- {`INSERT INTO plan_check_run (project, result) VALUES ('p4', '{"results": null}')`, nil},
-
- // Revision: release provenance corroborates (exactly one row, older,
- // carries the hash) → (rel-ok, rev-rel). Redundant with the release
- // source, so presence-only.
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-ok', 'r', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-rel"])}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-ok/releases/r"}`, blobs["rev-rel"])}},
-
- // Legacy uppercase hex: the revision stores the hash in uppercase
- // while the release file carries lowercase. Text comparisons are
- // lowered, so this corroborates → (rel-upper, rev-upper).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-upper', 'r', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-upper"])}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-upper/releases/r"}`, strings.ToUpper(blobs["rev-upper"]))}},
-
- // Observable probe, all constraints satisfied: task-run provenance in
- // tr-ok whose task routes the hash through a release in rel-task.
- // Expect the additive (tr-ok, rev-nest) alongside the source-derived
- // (rel-task, rev-nest).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-task', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-nest"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-ok', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-task/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-ok', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-nest"], taskRunName("tr-ok", 10, 100))}},
-
- // Temporal violation: the task run postdates the revision (the ID-reuse
- // shape after a purge). No (tr-late, rev-late).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-old', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-late"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-late', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-old/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-late', 100, 10, $1)", []any{newer}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-late"], taskRunName("tr-late", 10, 100))}},
-
- // Hash mismatch: the nested release carries a different hash than the
- // revision claims (laundered provenance). No (tr-mismatch, rev-mismatch).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-mismatch', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-other"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-mismatch', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-mismatch/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-mismatch', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-mismatch"], taskRunName("tr-mismatch", 10, 100))}},
-
- // (project, release_id) matching two rows: not a unique key, so
- // exactly-one fails even though both rows carry the hash and predate
- // the revision. No (tr-dup, rev-dup).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-dup', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-dup"])}},
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-dup', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-dup"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-dup', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-dup/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-dup', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-dup"], taskRunName("tr-dup", 10, 100))}},
-
- // Task-ID mismatch: the task run exists on (project, id) but belongs
- // to a different task than the name claims. No (tr-taskid, rev-taskid).
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-taskid', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-taskid"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-taskid', 11, $1::jsonb)",
- []any{`{"release":"projects/rel-taskid/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-taskid', 100, 11, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-taskid"], taskRunName("tr-taskid", 10, 100))}},
-
- // Direct task hash: corroborates through task.payload.sheetSha256.
- // Redundant with the task source, so presence-only.
- {"INSERT INTO task (project, id, payload) VALUES ('tr-direct', 10, $1::jsonb)",
- []any{fmt.Sprintf(`{"sheetSha256":%q}`, blobs["rev-direct"])}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-direct', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":%q}`, blobs["rev-direct"], taskRunName("tr-direct", 10, 100))}},
-
- // Preference: both release and task-run provenance corroborate. The
- // release wins, so the task-run fallback's additive grant
- // (tr-prefer, rev-prefer) must NOT appear.
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-prefer', 'r', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-prefer"])}},
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-nest-prefer', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-prefer"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-prefer', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-nest-prefer/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-prefer', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-prefer/releases/r","taskRun":%q}`,
- blobs["rev-prefer"], taskRunName("tr-prefer", 10, 100))}},
-
- // Fallback: the release provenance fails on age (the release postdates
- // the revision), so the corroborated task run supplies the grant —
- // (tr-fb, rev-fb) must appear.
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-newer', 'r', $1, $2::jsonb)",
- []any{newer, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-fb"])}},
- {"INSERT INTO release (project, release_id, created_at, payload) VALUES ('rel-nest-fb', 'nest', $1, $2::jsonb)",
- []any{older, fmt.Sprintf(`{"files":[{"sheetSha256":%q}]}`, blobs["rev-fb"])}},
- {"INSERT INTO task (project, id, payload) VALUES ('tr-fb', 10, $1::jsonb)",
- []any{`{"release":"projects/rel-nest-fb/releases/nest"}`}},
- {"INSERT INTO task_run (project, id, task_id, created_at) VALUES ('tr-fb', 100, 10, $1)", []any{older}},
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/rel-newer/releases/r","taskRun":%q}`,
- blobs["rev-fb"], taskRunName("tr-fb", 10, 100))}},
-
- // Provenance naming rows that no longer exist (purged authoring
- // project): no ref at all, the hash stays zero-ref.
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"release":"projects/ghost/releases/r","taskRun":%q}`,
- blobs["rev-ghost"], taskRunName("ghost", 10, 100))}},
-
- // No provenance at all: no ref, the hash stays zero-ref.
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q}`, blobs["rev-bare"])}},
-
- // Abort hazards: neither row may fail the migration.
- // A taskRun name whose IDs overflow bigint fails the bounded digit
- // match and stays uncorroborated instead of erroring the cast.
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"taskRun":"projects/tr-ok/plans/1/rollout/stages/prod/tasks/99999999999999999999999/taskRuns/99999999999999999999999"}`,
- blobs["rev-overflow"])}},
- // A pre-existing payload.project naming no live project row is kept
- // as-is but granted no ref: the project EXISTS guard skips it instead
- // of aborting on the foreign key.
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"project":"stale-ghost"}`, blobs["rev-stale-stamp"])}},
- // Same for the empty string, which survives the IS NOT NULL filter.
- {"INSERT INTO revision (instance, db_name, created_at, payload) VALUES ('i', 'd', $1, $2::jsonb)",
- []any{rev, fmt.Sprintf(`{"sheetSha256":%q,"project":""}`, blobs["rev-empty-stamp"])}},
- }
- for _, s := range seeds {
- _, err := db.ExecContext(ctx, s.query, s.args...)
- require.NoError(t, err, s.query)
- }
-
- statement, err := migrationFS.ReadFile("migration/3.22/0005##scope_sheet_blob.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(statement))
- require.NoError(t, err)
-
- rows, err := db.QueryContext(ctx,
- "SELECT project, encode(sha256, 'hex') FROM sheet_blob_ref ORDER BY project, sha256")
- require.NoError(t, err)
- defer rows.Close()
- var got []string
- for rows.Next() {
- var project, sha string
- require.NoError(t, rows.Scan(&project, &sha))
- got = append(got, project+"|"+sha)
- }
- require.NoError(t, rows.Err())
- require.ElementsMatch(t, []string{
- // The four project-scoped sources.
- "p1|" + blobs["plan"],
- "p1|" + blobs["shared"],
- "p2|" + blobs["task"],
- "p2|" + blobs["shared"],
- "p3|" + blobs["release"],
- "p4|" + blobs["pcr"],
- // Release rows seeded as corroboration targets are sources themselves.
- "rel-ok|" + blobs["rev-rel"],
- "rel-upper|" + blobs["rev-upper"],
- "rel-task|" + blobs["rev-nest"],
- "rel-old|" + blobs["rev-late"],
- "rel-mismatch|" + blobs["rev-other"],
- "rel-dup|" + blobs["rev-dup"],
- "rel-taskid|" + blobs["rev-taskid"],
- "tr-direct|" + blobs["rev-direct"],
- "rel-prefer|" + blobs["rev-prefer"],
- "rel-nest-prefer|" + blobs["rev-prefer"],
- "rel-newer|" + blobs["rev-fb"],
- "rel-nest-fb|" + blobs["rev-fb"],
- // The revision branch's observable grants: corroborated task-run
- // provenance routing through another project's release, and the
- // age-based fallback. Every negative scenario is covered by this
- // list's exactness: tr-late, tr-mismatch, tr-dup, tr-taskid and
- // tr-prefer hold no refs.
- "tr-ok|" + blobs["rev-nest"],
- "tr-fb|" + blobs["rev-fb"],
- }, got)
-
- // Corroboration also stamps payload.project — the stored fact new
- // revision writers set at creation. Each scenario's revision is
- // identified by its unique hash; an empty stamp means the provenance did
- // not corroborate. rev-prefer pins the release-over-taskRun preference
- // directly: both branches corroborate and the release's project wins.
- for sha, want := range map[string]string{
- blobs["rev-rel"]: "rel-ok",
- blobs["rev-upper"]: "rel-upper",
- blobs["rev-nest"]: "tr-ok",
- blobs["rev-late"]: "",
- blobs["rev-mismatch"]: "",
- blobs["rev-dup"]: "",
- blobs["rev-taskid"]: "",
- blobs["rev-direct"]: "tr-direct",
- blobs["rev-prefer"]: "rel-prefer",
- blobs["rev-fb"]: "tr-fb",
- blobs["rev-ghost"]: "",
- blobs["rev-bare"]: "",
- blobs["rev-overflow"]: "",
- blobs["rev-stale-stamp"]: "stale-ghost",
- blobs["rev-empty-stamp"]: "",
- } {
- var got string
- require.NoError(t, db.QueryRowContext(ctx,
- "SELECT COALESCE(payload->>'project', '') FROM revision WHERE lower(payload->>'sheetSha256') = $1", sha).Scan(&got))
- require.Equal(t, want, got, "stamped project for hash %s", sha)
- }
-
- // The zero-ref audit counts the never-referenced blob plus the revisions
- // whose provenance was absent or uncorroborated with no other source.
- var zeroRefBlobs int
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT count(*) FROM sheet_blob b
- WHERE NOT EXISTS (SELECT 1 FROM sheet_blob_ref r WHERE r.sha256 = b.sha256)
- `).Scan(&zeroRefBlobs))
- require.Equal(t, 7, zeroRefBlobs, "orphan, rev-mismatch, rev-ghost, rev-bare, rev-overflow, rev-stale-stamp, and rev-empty-stamp have zero refs")
-
- // The multi-project audit reports every hash claimed by more than one
- // project, including the deliberately shared one.
- auditRows, err := db.QueryContext(ctx, `
- SELECT encode(sha256, 'hex')
- FROM sheet_blob_ref
- GROUP BY sha256
- HAVING count(DISTINCT project) > 1
- `)
- require.NoError(t, err)
- defer auditRows.Close()
- multiProject := map[string]bool{}
- for auditRows.Next() {
- var sha string
- require.NoError(t, auditRows.Scan(&sha))
- multiProject[sha] = true
- }
- require.NoError(t, auditRows.Err())
- require.True(t, multiProject[blobs["shared"]], "the cross-project reference must be reported, not blessed")
-
- // The source-level missing-ref audit from the design doc: a plan that
- // references a pre-existing foreign hash AFTER the backfill invokes no
- // CreateSheets and writes no ref, so only this query can see it.
- _, err = db.ExecContext(ctx, "INSERT INTO plan (project, config) VALUES ('p1', $1::jsonb)",
- fmt.Sprintf(`{"specs":[{"changeDatabaseConfig":{"sheetSha256":%q}}]}`, blobs["pcr"]))
- require.NoError(t, err)
- var missingProject, missingSha string
- require.NoError(t, db.QueryRowContext(ctx, `
- SELECT DISTINCT pl.project, spec->'changeDatabaseConfig'->>'sheetSha256' AS sha
- FROM plan pl
- CROSS JOIN LATERAL jsonb_array_elements(
- CASE WHEN jsonb_typeof(pl.config->'specs') = 'array' THEN pl.config->'specs' ELSE '[]'::jsonb END) AS spec
- WHERE spec->'changeDatabaseConfig'->>'sheetSha256' ~ '^[0-9a-fA-F]{64}$'
- AND NOT EXISTS (
- SELECT 1 FROM sheet_blob_ref r
- WHERE r.project = pl.project
- AND r.sha256 = decode(spec->'changeDatabaseConfig'->>'sheetSha256', 'hex')
- )
- `).Scan(&missingProject, &missingSha))
- require.Equal(t, "p1", missingProject)
- require.Equal(t, blobs["pcr"], missingSha)
-}
diff --git a/backend/migrator/retire_export_data_test.go b/backend/migrator/retire_export_data_test.go
deleted file mode 100644
index f26c524dfa..0000000000
--- a/backend/migrator/retire_export_data_test.go
+++ /dev/null
@@ -1,178 +0,0 @@
-package migrator
-
-import (
- "context"
- "io/fs"
- "testing"
-
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
-)
-
-// TestRetireExportDataMigration seeds legacy DATABASE_EXPORT rows and verifies
-// the 3.22 purge deletes the whole export family — including the draft
-// DATABASE_CHANGE issue attached to an export plan, which a type-only delete
-// would leave behind to break the plan delete on the issue→plan FK — while
-// unrelated plans, issues, tasks, and comments survive.
-//
-// Project p2 seeds a change workflow whose plan/issue/task/task_run ids all
-// collide with p1's export family: every purge predicate must carry the
-// project column of the composite PKs, so p2's rows survive untouched.
-func TestRetireExportDataMigration(t *testing.T) {
- ctx := context.Background()
- container := testcontainer.GetTestPgContainer(ctx, t)
- t.Cleanup(func() { container.Close(ctx) })
-
- db := container.GetDB()
- require.NoError(t, MigrateSchema(ctx, db))
-
- // Fresh installs no longer have export_archive; recreate the pre-3.22 table
- // so the DROP TABLE in the migration runs against a realistic schema.
- _, err := db.ExecContext(ctx, `
- CREATE TABLE export_archive (
- resource_id text PRIMARY KEY DEFAULT gen_random_uuid()::text,
- workspace text NOT NULL REFERENCES workspace(resource_id),
- created_at timestamptz NOT NULL DEFAULT now(),
- bytes bytea,
- payload jsonb NOT NULL DEFAULT '{}'
- );
-
- INSERT INTO workspace (resource_id) VALUES ('default');
- INSERT INTO principal (name, email, password_hash) VALUES ('creator', '[email protected]', 'unused');
- INSERT INTO project (resource_id, workspace, name) VALUES ('p1', 'default', 'Project 1');
- INSERT INTO instance (resource_id, workspace) VALUES ('i1', 'default');
-
- -- Family A: classic completed export workflow.
- INSERT INTO plan (id, creator, project, name, description, config) VALUES
- (100, '[email protected]', 'p1', 'export plan', '',
- '{"specs":[{"id":"s1","exportDataConfig":{"targets":["instances/i1/databases/db1"]}}]}');
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (200, '[email protected]', 'p1', 100, 'export issue', 'DONE', 'DATABASE_EXPORT');
- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES
- (300, 'p1', 100, 'i1', 'db1', 'DATABASE_EXPORT');
- INSERT INTO task_run (id, creator, project, task_id, attempt, status, result) VALUES
- (400, '[email protected]', 'p1', 300, 0, 'DONE', '{"exportArchiveId":"arch-1"}');
- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 400);
- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (500, 'p1', 100, 'DONE');
- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 200);
- INSERT INTO plan_webhook_delivery (project, plan_id, event_type) VALUES ('p1', 100, 'PIPELINE_COMPLETED');
- INSERT INTO export_archive (resource_id, workspace, bytes) VALUES ('arch-1', 'default', ''::bytea);
-
- -- Family B: export plan carrying a draft DATABASE_CHANGE issue (the FK trap).
- INSERT INTO plan (id, creator, project, name, description, config) VALUES
- (101, '[email protected]', 'p1', 'export plan with draft', '',
- '{"specs":[{"id":"s1","exportDataConfig":{}}]}');
- INSERT INTO issue (id, creator, project, plan_id, name, status, type, payload) VALUES
- (201, '[email protected]', 'p1', 101, 'draft on export plan', 'OPEN', 'DATABASE_CHANGE', '{"draft":true}');
- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 201);
-
- -- Family D: legacy mixed plan (change + export specs). The whole family
- -- is purged via the plan-membership arm, including its DATABASE_MIGRATE
- -- task and DATABASE_CHANGE issue.
- INSERT INTO plan (id, creator, project, name, description, config) VALUES
- (103, '[email protected]', 'p1', 'mixed plan', '',
- '{"specs":[{"id":"s1","changeDatabaseConfig":{}},{"id":"s2","exportDataConfig":{}}]}');
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (205, '[email protected]', 'p1', 103, 'mixed issue', 'DONE', 'DATABASE_CHANGE');
- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES
- (303, 'p1', 103, 'i1', 'db1', 'DATABASE_MIGRATE');
- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES
- (403, '[email protected]', 'p1', 303, 0, 'DONE');
- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 403);
-
- -- An export-typed issue with no plan at all.
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (204, '[email protected]', 'p1', NULL, 'orphan export issue', 'CANCELED', 'DATABASE_EXPORT');
-
- -- Family C: unrelated change workflow that must survive.
- INSERT INTO plan (id, creator, project, name, description, config) VALUES
- (102, '[email protected]', 'p1', 'change plan', '',
- '{"specs":[{"id":"s1","changeDatabaseConfig":{"targets":["instances/i1/databases/db1"]}}]}');
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (202, '[email protected]', 'p1', 102, 'change issue', 'DONE', 'DATABASE_CHANGE');
- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES
- (302, 'p1', 102, 'i1', 'db1', 'DATABASE_MIGRATE');
- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES
- (402, '[email protected]', 'p1', 302, 0, 'DONE');
- INSERT INTO task_run_log (project, task_run_id) VALUES ('p1', 402);
- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (502, 'p1', 102, 'DONE');
- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p1', 202);
-
- -- A plan-less role grant issue that must survive.
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (203, '[email protected]', 'p1', NULL, 'role grant', 'OPEN', 'ROLE_GRANT');
-
- -- Project p2: a change workflow whose ids collide with p1's export
- -- family. Survives in full — proves composite-PK project scoping.
- INSERT INTO project (resource_id, workspace, name) VALUES ('p2', 'default', 'Project 2');
- INSERT INTO plan (id, creator, project, name, description, config) VALUES
- (100, '[email protected]', 'p2', 'p2 change plan', '',
- '{"specs":[{"id":"s1","changeDatabaseConfig":{"targets":["instances/i1/databases/db2"]}}]}');
- INSERT INTO issue (id, creator, project, plan_id, name, status, type) VALUES
- (200, '[email protected]', 'p2', 100, 'p2 change issue', 'DONE', 'DATABASE_CHANGE');
- INSERT INTO task (id, project, plan_id, instance, db_name, type) VALUES
- (300, 'p2', 100, 'i1', 'db2', 'DATABASE_MIGRATE');
- INSERT INTO task_run (id, creator, project, task_id, attempt, status) VALUES
- (400, '[email protected]', 'p2', 300, 0, 'DONE');
- INSERT INTO task_run_log (project, task_run_id) VALUES ('p2', 400);
- INSERT INTO plan_check_run (id, project, plan_id, status) VALUES (500, 'p2', 100, 'DONE');
- INSERT INTO issue_comment (creator, project, issue_id) VALUES ('[email protected]', 'p2', 200);
- INSERT INTO plan_webhook_delivery (project, plan_id, event_type) VALUES ('p2', 100, 'PIPELINE_COMPLETED');
- `)
- require.NoError(t, err)
-
- // Positive preconditions: the seeds must exist before the purge, so the
- // survival assertions below cannot pass vacuously.
- var planCount, issueCount int
- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM plan`).Scan(&planCount))
- require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM issue`).Scan(&issueCount))
- require.Equal(t, 5, planCount)
- require.Equal(t, 7, issueCount)
-
- buf, err := fs.ReadFile(migrationFS, "migration/3.22/0000##retire_export_data.sql")
- require.NoError(t, err)
-
- conn, err := db.Conn(ctx)
- require.NoError(t, err)
- defer conn.Close()
- require.NoError(t, executeMigration(ctx, conn, string(buf), "test-retire-export-data"))
-
- queryIDs := func(query string) []int64 {
- rows, err := db.QueryContext(ctx, query)
- require.NoError(t, err)
- defer rows.Close()
- var ids []int64
- for rows.Next() {
- var id int64
- require.NoError(t, rows.Scan(&id))
- ids = append(ids, id)
- }
- require.NoError(t, rows.Err())
- return ids
- }
-
- require.Equal(t, []int64{102}, queryIDs(`SELECT id FROM plan WHERE project = 'p1' ORDER BY id`))
- require.Equal(t, []int64{202, 203}, queryIDs(`SELECT id FROM issue WHERE project = 'p1' ORDER BY id`))
- require.Equal(t, []int64{302}, queryIDs(`SELECT id FROM task WHERE project = 'p1' ORDER BY id`))
- require.Equal(t, []int64{402}, queryIDs(`SELECT id FROM task_run WHERE project = 'p1' ORDER BY id`))
- require.Equal(t, []int64{402}, queryIDs(`SELECT task_run_id FROM task_run_log WHERE project = 'p1' ORDER BY task_run_id`))
- require.Equal(t, []int64{502}, queryIDs(`SELECT id FROM plan_check_run WHERE project = 'p1' ORDER BY id`))
- require.Equal(t, []int64{202}, queryIDs(`SELECT issue_id FROM issue_comment WHERE project = 'p1' ORDER BY issue_id`))
- require.Empty(t, queryIDs(`SELECT plan_id FROM plan_webhook_delivery WHERE project = 'p1'`))
-
- // p2's colliding-id change workflow survives in full.
- require.Equal(t, []int64{100}, queryIDs(`SELECT id FROM plan WHERE project = 'p2' ORDER BY id`))
- require.Equal(t, []int64{200}, queryIDs(`SELECT id FROM issue WHERE project = 'p2' ORDER BY id`))
- require.Equal(t, []int64{300}, queryIDs(`SELECT id FROM task WHERE project = 'p2' ORDER BY id`))
- require.Equal(t, []int64{400}, queryIDs(`SELECT id FROM task_run WHERE project = 'p2' ORDER BY id`))
- require.Equal(t, []int64{400}, queryIDs(`SELECT task_run_id FROM task_run_log WHERE project = 'p2' ORDER BY task_run_id`))
- require.Equal(t, []int64{500}, queryIDs(`SELECT id FROM plan_check_run WHERE project = 'p2' ORDER BY id`))
- require.Equal(t, []int64{200}, queryIDs(`SELECT issue_id FROM issue_comment WHERE project = 'p2' ORDER BY issue_id`))
- require.Equal(t, []int64{100}, queryIDs(`SELECT plan_id FROM plan_webhook_delivery WHERE project = 'p2'`))
-
- var exportArchiveExists bool
- require.NoError(t, db.QueryRowContext(ctx,
- `SELECT to_regclass('export_archive') IS NOT NULL`).Scan(&exportArchiveExists))
- require.False(t, exportArchiveExists)
-}
diff --git a/backend/migrator/split_saved_query_manage_test.go b/backend/migrator/split_saved_query_manage_test.go
deleted file mode 100644
index db159f8cf7..0000000000
--- a/backend/migrator/split_saved_query_manage_test.go
+++ /dev/null
@@ -1,76 +0,0 @@
-package migrator
-
-import (
- "context"
- "encoding/json"
- "testing"
-
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
-)
-
-// TestSplitSavedQueryManageMigration exercises 3.22/0009's jsonb transform:
-// bb.savedQueries.manage expands to the four per-verb permissions —
-// deliberately without setIamPolicy — deduplicated, while rows without
-// manage and rows with a non-array permissions payload stay untouched.
-func TestSplitSavedQueryManageMigration(t *testing.T) {
- if testing.Short() {
- t.Skip("skipping testcontainer test in short mode")
- }
- ctx := context.Background()
- pg := testcontainer.GetTestPgContainer(ctx, t)
- defer pg.Close(ctx)
- db := pg.GetDB()
-
- _, err := db.ExecContext(ctx, `
- CREATE TABLE role (
- resource_id text NOT NULL PRIMARY KEY,
- permissions jsonb NOT NULL DEFAULT '{}'
- );
- INSERT INTO role (resource_id, permissions) VALUES
- ('manage-holder', '{"permissions": ["bb.savedQueries.manage", "bb.savedQueries.get", "bb.projects.get"]}'),
- ('no-manage', '{"permissions": ["bb.savedQueries.search", "bb.projects.get"]}'),
- ('non-array', '{"permissions": {"oops": true}}'),
- ('empty', '{}');
- `)
- require.NoError(t, err)
-
- migration, err := migrationFS.ReadFile("migration/3.22/0009##split_saved_query_manage.sql")
- require.NoError(t, err)
- _, err = db.ExecContext(ctx, string(migration))
- require.NoError(t, err)
-
- permissionsOf := func(roleID string) []string {
- var raw []byte
- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions->'permissions' FROM role WHERE resource_id = $1`, roleID).Scan(&raw))
- if string(raw) == "null" {
- return nil
- }
- var list []string
- require.NoError(t, json.Unmarshal(raw, &list))
- return list
- }
-
- // manage expands to the four verbs, deduplicated against the get the
- // role already held; setIamPolicy is deliberately absent.
- require.ElementsMatch(t, []string{
- "bb.projects.get",
- "bb.savedQueries.delete",
- "bb.savedQueries.get",
- "bb.savedQueries.getIamPolicy",
- "bb.savedQueries.update",
- }, permissionsOf("manage-holder"))
- require.NotContains(t, permissionsOf("manage-holder"), "bb.savedQueries.setIamPolicy")
-
- // Rows without manage keep their permissions verbatim.
- require.Equal(t, []string{"bb.savedQueries.search", "bb.projects.get"}, permissionsOf("no-manage"))
-
- // Non-array and empty payloads are untouched rather than mangled.
- var nonArray string
- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions::text FROM role WHERE resource_id = 'non-array'`).Scan(&nonArray))
- require.JSONEq(t, `{"permissions": {"oops": true}}`, nonArray)
- var empty string
- require.NoError(t, db.QueryRowContext(ctx, `SELECT permissions::text FROM role WHERE resource_id = 'empty'`).Scan(&empty))
- require.JSONEq(t, `{}`, empty)
-}
diff --git a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go
deleted file mode 100644
index 1cf3091538..0000000000
--- a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go
+++ /dev/null
@@ -1,89 +0,0 @@
-package starrocks
-
-import (
- "bytes"
- "context"
- "database/sql"
- "fmt"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
-)
-
-// TestDump_StarRocksMaterializedViewRoundTrip locks the two dump fixes (BYT-9689) against a
-// real StarRocks (amd64-only; skipped in -short). A materialized view defined on a regular
-// view is dumped, then replayed into a fresh database of the same name, and must come back:
-// - without a name collision (P1: the temporary placeholder is dropped as a view, so the
-// real CREATE MATERIALIZED VIEW does not collide with a leftover placeholder), and
-// - IS_ACTIVE=true (P2: the MV is emitted after its source view, so it is created against
-// the real view, not inactivated when a placeholder is dropped under it).
-//
-// The dump is schema-only (--no-data), so IS_ACTIVE — not row data — is the P2 signal.
-func TestDump_StarRocksMaterializedViewRoundTrip(t *testing.T) {
- if testing.Short() {
- t.Skip("skipping StarRocks testcontainer test in short mode")
- }
- ctx := context.Background()
- c := testcontainer.GetTestStarRocksContainer(ctx, t)
- defer c.Close(ctx)
- db := c.GetDB()
-
- for _, stmt := range []string{
- "CREATE DATABASE rt",
- "CREATE TABLE rt.t (id INT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",
- "INSERT INTO rt.t VALUES (1,10),(2,20)",
- "CREATE VIEW rt.vw AS SELECT id, amt FROM rt.t",
- "CREATE MATERIALIZED VIEW rt.mv REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM rt.vw GROUP BY id",
- } {
- if _, err := db.ExecContext(ctx, stmt); err != nil {
- t.Fatalf("setup failed: %s: %v", stmt, err)
- }
- }
- requireMVActive(ctx, t, db, "rt", "mv")
-
- // Produce the dump through the real driver.
- d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "rt"}
- var buf bytes.Buffer
- require.NoError(t, d.Dump(ctx, &buf, nil))
- dump := buf.String()
- require.Contains(t, dump, "CREATE MATERIALIZED VIEW", "dump should emit the MV")
-
- // Restore into a fresh same-name database. StarRocks qualifies view/MV references with
- // the original db, so the restore target must reuse the name. DROP+CREATE+USE+replay run
- // in one multi-statement Exec so USE applies to the unqualified statements on the same
- // pooled connection.
- restore := "DROP DATABASE IF EXISTS rt;\nCREATE DATABASE rt;\nUSE rt;\n" + dump
- if _, err := db.ExecContext(ctx, restore); err != nil {
- t.Fatalf("dump replay failed (P1 collision or unreplayable DDL): %v", err)
- }
-
- // P2: the restored MV must be active, not inactivated by the placeholder dance.
- requireMVActive(ctx, t, db, "rt", "mv")
-
- // And it re-syncs cleanly: mv as a materialized view, no stale placeholder.
- meta, err := d.SyncDBSchema(ctx)
- require.NoError(t, err)
- require.Len(t, meta.Schemas, 1)
- require.Len(t, meta.Schemas[0].MaterializedViews, 1)
- require.Equal(t, "mv", meta.Schemas[0].MaterializedViews[0].Name)
-}
-
-// requireMVActive waits until the named materialized view reports IS_ACTIVE=true.
-func requireMVActive(ctx context.Context, t *testing.T, db *sql.DB, schema, name string) {
- t.Helper()
- q := fmt.Sprintf("SELECT IS_ACTIVE FROM information_schema.materialized_views WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'", schema, name)
- require.Eventually(t, func() bool {
- var active string
- if err := db.QueryRowContext(ctx, q).Scan(&active); err != nil {
- return false
- }
- return active == "true"
- // information_schema.materialized_views (listMaterializedViewStatus, an FE->BE RPC)
- // is unavailable for ~tens of seconds after a fresh allin1 boot ("BE host unknown")
- // before it settles, so allow a generous window.
- }, 180*time.Second, 3*time.Second, "MV %s.%s did not become active", schema, name)
-}
diff --git a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go
deleted file mode 100644
index ae004487d9..0000000000
--- a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package starrocks
-
-import (
- "context"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
-
- "github.com/bytebase/bytebase/backend/common/testcontainer"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
-)
-
-// TestSyncDBSchema_StarRocksMaterializedView_E2E brings up a real StarRocks (amd64-only;
-// runs in CI, skipped in -short) and verifies BYT-9689 end to end: an async materialized
-// view is synced under MaterializedViews with its definition, while a synchronous rollup
-// (REFRESH_TYPE='ROLLUP', which has no information_schema.tables row) is excluded.
-func TestSyncDBSchema_StarRocksMaterializedView_E2E(t *testing.T) {
- if testing.Short() {
- t.Skip("skipping StarRocks testcontainer test in short mode")
- }
- ctx := context.Background()
- c := testcontainer.GetTestStarRocksContainer(ctx, t)
- defer c.Close(ctx)
- db := c.GetDB()
-
- for _, stmt := range []string{
- "CREATE DATABASE mvtest",
- "CREATE TABLE mvtest.sales (id BIGINT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",
- "INSERT INTO mvtest.sales VALUES (1,10),(2,20)",
- "CREATE MATERIALIZED VIEW mvtest.mv_async REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM mvtest.sales GROUP BY id",
- "CREATE MATERIALIZED VIEW mvtest.mv_rollup AS SELECT id, sum(amt) FROM mvtest.sales GROUP BY id",
- } {
- if _, err := db.ExecContext(ctx, stmt); err != nil {
- t.Fatalf("setup failed: %s: %v", stmt, err)
- }
- }
-
- // The sync rollup builds asynchronously; wait until it registers in
- // information_schema.materialized_views so the test deterministically exercises the
- // rollup-exclusion path (it appears there as REFRESH_TYPE='ROLLUP' but never as a
- // information_schema.tables row).
- require.Eventually(t, func() bool {
- var n int
- if err := db.QueryRowContext(ctx,
- "SELECT COUNT(*) FROM information_schema.materialized_views WHERE TABLE_SCHEMA='mvtest' AND REFRESH_TYPE='ROLLUP'").Scan(&n); err != nil {
- return false
- }
- return n == 1
- // information_schema.materialized_views (an FE->BE RPC) is unavailable for tens of
- // seconds after a fresh allin1 boot before it settles; allow a generous window.
- }, 150*time.Second, 3*time.Second, "sync rollup did not register in materialized_views")
-
- d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "mvtest"}
- meta, err := d.SyncDBSchema(ctx)
- require.NoError(t, err)
- require.Len(t, meta.Schemas, 1)
- schema := meta.Schemas[0]
-
- // The async MV is synced under MaterializedViews with its definition.
- require.Len(t, schema.MaterializedViews, 1, "exactly the async MV must be synced")
- require.Equal(t, "mv_async", schema.MaterializedViews[0].Name)
- require.Contains(t, schema.MaterializedViews[0].Definition, "CREATE MATERIALIZED VIEW")
-
- // The base table is synced as a table; the sync rollup leaks nowhere; the MV is not
- // misfiled as a regular view.
- require.Len(t, schema.Tables, 1)
- require.Equal(t, "sales", schema.Tables[0].Name)
- for _, mv := range schema.MaterializedViews {
- require.NotEqual(t, "mv_rollup", mv.Name, "sync rollup must be excluded from MaterializedViews")
- }
- for _, v := range schema.Views {
- require.NotEqual(t, "mv_async", v.Name, "MV must not be misfiled as a regular view")
- }
-}
diff --git a/docs/design/backend-test-execution-time.md b/docs/design/backend-test-execution-time.md
index a7bc02d056..9ad63ea374 100644
--- a/docs/design/backend-test-execution-time.md
+++ b/docs/design/backend-test-execution-time.md
@@ -92,6 +92,9 @@ cost inside a package that runs 3.7× parallel, and effort 1's saving sits insid
effort 4's, so they do not simply add. Together they should take it from 635 s
of wall clock to roughly 330 s.
+Effort 6 was rescoped and now runs ahead of effort 2 for `api/v1`; the numbering
+is kept so existing references resolve.
+
### [✓] 1. Close idle connections before shutting the server down
**Three lines.** `httpServer.Shutdown` burns 1036 ms on every one of the 207
@@ -124,10 +127,13 @@ below against 611 s.
### 2. One shared container per package, then turn parallelism on
-**1371 s of wall clock across four packages, about 1070 s of it container
-starts.** `store`, `api/v1`, `component/review` and `migrator` start 247
-containers between them, and because they run serially that cost is their wall
-clock almost exactly.
+**459 s of wall clock, 390 s of it container starts.** `store` starts 90
+containers, and because it runs serially that cost is its wall clock almost
+exactly.
+
+`api/v1`, `component/review` and `migrator` were in this effort's original
+scope. AGENTS.md now confines a real metadata Postgres to `store` and `tests`,
+so all three move to effort 6 instead.
Two changes, in this order:
@@ -143,18 +149,18 @@ Measured on the migrated Bytebase schema (9.5 MB): migrating the template costs
`CREATE DATABASE … TEMPLATE` is **44 ms** — 98× cheaper than the 4328 ms
container it replaces.
-Two details. `api/v1` shares a Postgres rather than using a fake store: it is
-the smaller change, and a fake risks testing the fake. And close the template
-connection after migrating, because `TEMPLATE` refuses to copy a database that
-still has a session attached.
+One detail. Close the template connection after migrating, because `TEMPLATE`
+refuses to copy a database that still has a session attached.
+
+This section originally called for `api/v1` to share a Postgres "rather than
+using a fake store". Effort 6 reverses that.
Expect tests that silently assumed a virgin database to fail. That is the point.
Fix them with per-test databases, not with cleanup hooks — those reintroduce
ordering coupling.
-Step 1 alone takes those four packages from 1371 s to roughly 320 s. Step 2
-should take them well below that: what remains is about 300 s of real work
-spread across 550 tests.
+Step 1 alone takes `store` from 459 s to roughly 75 s. Step 2 should take it
+below that; what remains is real work across 128 tests.
### 3. Pool the provisioned Postgres instances
@@ -250,24 +256,61 @@ Three keep their engine. Each needs a comment saying why:
under test. It covers MySQL, Postgres, Oracle and MSSQL in one table, and
boots no server.
-### 6. Push tests down a level
-
-**Mostly architectural.** A test earns a server boot only if it needs a
-background runner, a real rollout, or the audit trail. `backend/api/mcp` shows
-the payoff: 168 tests in 39.5 s, no containers.
-
-Candidates: `mcp_capability_setting_test.go` (6 boots to read and write one
-settings row), the saved-query list and filter tests (12), and GitOps
-`CheckRelease` validation (11 of 17 — one stateless call against different
-releases).
-
-Effort 4 makes a boot cheap, which takes most of the money out of this. Treat it
-as a rule to hold going forward, not a saving to bank.
-
-Keep the 27 `TestCollision*` and `TestClaim*` tests where they are.
+### 6. Seams instead of containers
+
+**912 s of wall clock, 680 s of it container starts.** AGENTS.md confines a real
+metadata Postgres to `store` and `tests`. `api/v1`, `component/review` and
+`migrator` are not on that list and start 157 containers between them.
+`backend/api/mcp` is the proof this is livable: 168 tests in 39.5 s, 16 of its
+18 files containerless.
+
+Effort 2 declined a fake store on two grounds, and both fail here. `api/v1` is
+not testing its API — `mcp_info_test.go` calls `service.GetMCPInfo` as a Go
+method, so the container buys a `*store.Store` and nothing the test asserts. And
+"a fake store" was the wrong unit: `store.Store` has 305 methods and `api/v1`
+calls 181, but `api/mcp` needed five (`serverStore`, faked in 62 lines) and
+`mcp_gate.go` needed one (`mcpSettingsReader`). Interfaces go beside the handler
+that reads them, never over the store.
+
+Fourteen files in `api/v1` start containers, ~90 tests. Read from names, not
+bodies — verify the back two rows:
+
+| Disposition | Tests | Shape |
+| --- | ---: | --- |
+| Move to `backend/store` | ~20 | `Test*Claims`, concurrent-update and serialization, `TestMixedIssuePatchRollsBackWhenLabelsFail`, retention filter boundaries, `TestIssueApprovalFiltersRunBeforePaging` |
+| Pure function over plain data | ~40 | canonical-name tests, the four `TestCheckRelease*`, converters |
+| Narrow interface and a fake | ~30 | `TestGetMCPInfoHandler`, the list-and-hide tests, `TestApproveIssueFailsClosedWhenIAMLookupFails` |
+
+`component/review` splits the same way:
+`TestApplyApprovalTemplateAndCreatePlanCheckRunDoNotDeadlock` is lock ordering
+and moves to `store`; the CEL, template-matching and target-unfolding tests are
+pure functions already, wearing a container for the package's sake.
+
+Prefer the middle row: a handler is fetch, decide, convert, and only the fetch
+needs a store.
+
+`migrator` is done, by deletion rather than by moving. `TestLatestVersion` and
+`TestVersionUnique` need no database and stay; the other 16 booted a container
+each to apply one migration to a hand-built fixture and assert the transform.
+Moving them to `backend/store` would have relocated the 90 s, not removed it, so
+they were deleted outright. The package now runs in under 2 s.
+
+That is 90 s bought at a real price, and the price should be stated plainly: the
+migration SQL — including irreversible data transforms over customer metadata —
+now has no test coverage, and no fake can restore it. Anyone reintroducing
+coverage here should bring back the fixtures against a shared container rather
+than one per test.
+
+**Every fake needs a contract test** — one table run against both the fake and
+the real store — or effort 2's objection is right. Since none of these packages
+may hold a container, that suite lives in `backend/store`. No `mockgen`, no SQLite.
+
+Convert `mcp_info_test.go` first — `mcpSettingsReader` exists and the test is 60
+lines — and measure before doing the rest. Server boots go the same way:
+`mcp_capability_setting_test.go` spends 6 to read and write one settings row,
+the saved-query list and filter tests 12, GitOps `CheckRelease` 11 of 17. Keep
+the 27 `TestCollision*` and `TestClaim*` tests where they are —
`backend/store/AGENTS.md` requires them for a bug class unit tests cannot catch.
-25 of them already build on `setupCollidingProjects`, so effort 4 gives them a
-shared workspace almost for free.
### 7. Engine conformance to omni
@@ -305,9 +348,12 @@ deleted rather than moved. That is 3 s of the 403 s this effort is about, which
is the measure of how much of the cost sits behind the port rather than beside
it.
-This plan does not address `plugin/db/starrocks` (158 s). It tests our own
-driver against a heavyweight engine, every test runs on every PR, and skipping
-is not an option — so it stays as accepted cost.
+`plugin/db/starrocks` (158 s) is gone. Nearly all of it was two testcontainer
+tests booting the `allin1` image — an FE and a BE in one container, with a
+readiness wait measured in minutes — to assert materialized-view sync and dump
+round-trip. That is engine fidelity, which belongs in omni, paid for on every PR
+by every engineer. The two tests and the `GetStarRocksContainer` helpers were
+deleted; the package now runs in under 2 s on its remaining unit tests.
### One loose end