Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: make migration sql idempotent (#19615)

ecmadao3786af7c9bc7
diff --git a/backend/migrator/migration/3.16/0000##split_principal_table.sql b/backend/migrator/migration/3.16/0000##split_principal_table.sql
index e789267a9f..5a79dc9649 100644
--- a/backend/migrator/migration/3.16/0000##split_principal_table.sql
+++ b/backend/migrator/migration/3.16/0000##split_principal_table.sql
@@ -1,15 +1,26 @@
-- Split principal table into three tables: principal (END_USER only), service_account, workload_identity.
--- Step 0: Fix legacy service account emails that don't match the required format.
--- Valid formats: {name}@service.bytebase.com OR {name}@{project}.service.bytebase.com
--- This MUST run BEFORE dropping FK constraints so that ON UPDATE CASCADE propagates email
--- changes to all creator/deleter columns automatically.
+-- Step 0: Fix legacy emails, create tables, migrate data, clean up principal.
+-- All data migration is guarded by checking principal.type column existence,
+-- which is dropped at the end — so re-runs skip all data operations.
DO $$
DECLARE
rec RECORD;
new_email TEXT;
base_local TEXT;
BEGIN
+ -- Only run data migration if principal still has a type column (i.e., not yet split).
+ IF NOT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_name = 'principal' AND column_name = 'type'
+ ) THEN
+ RETURN;
+ END IF;
+
+ -- Fix legacy service account emails that don't match the required format.
+ -- Valid formats: {name}@service.bytebase.com OR {name}@{project}.service.bytebase.com
+ -- This MUST run BEFORE dropping FK constraints so that ON UPDATE CASCADE propagates email
+ -- changes to all creator/deleter columns automatically.
FOR rec IN
SELECT id, email, project
FROM principal
@@ -23,7 +34,6 @@ BEGIN
ELSE
new_email := base_local || '@service.bytebase.com';
END IF;
- -- Handle potential email collision by appending id
IF EXISTS (SELECT 1 FROM principal WHERE email = new_email AND id != rec.id) THEN
IF rec.project IS NOT NULL THEN
new_email := base_local || '-' || rec.id || '@' || rec.project || '.service.bytebase.com';
@@ -31,21 +41,33 @@ BEGIN
new_email := base_local || '-' || rec.id || '@service.bytebase.com';
END IF;
END IF;
- -- Update principal email (ON UPDATE CASCADE propagates to creator/deleter columns)
UPDATE principal SET email = new_email WHERE id = rec.id;
- -- Update JSONB references in policies (members use 'serviceAccounts/{email}' format)
UPDATE policy
SET payload = replace(payload::text, 'serviceAccounts/' || rec.email, 'serviceAccounts/' || new_email)::jsonb
WHERE payload::text LIKE '%serviceAccounts/' || rec.email || '%';
- -- Update user_group member references
UPDATE user_group
SET payload = replace(payload::text, 'serviceAccounts/' || rec.email, 'serviceAccounts/' || new_email)::jsonb
WHERE payload::text LIKE '%serviceAccounts/' || rec.email || '%';
END LOOP;
+
+ -- Migrate data into new tables.
+ INSERT INTO service_account (deleted, created_at, name, email, service_key_hash, project)
+ SELECT deleted, created_at, name, email, password_hash, project
+ FROM principal WHERE type = 'SERVICE_ACCOUNT'
+ ON CONFLICT (email) DO NOTHING;
+
+ INSERT INTO workload_identity (deleted, created_at, name, email, project, config)
+ SELECT deleted, created_at, name, email, project,
+ COALESCE(profile->'workloadIdentityConfig', '{}')
+ FROM principal WHERE type = 'WORKLOAD_IDENTITY'
+ ON CONFLICT (email) DO NOTHING;
+
+ -- Clean up principal table.
+ DELETE FROM principal WHERE type != 'END_USER';
END $$;
--- Step 1: Create service_account table
-CREATE TABLE service_account (
+-- Create tables (idempotent).
+CREATE TABLE IF NOT EXISTS service_account (
deleted boolean NOT NULL DEFAULT FALSE,
created_at timestamptz NOT NULL DEFAULT now(),
name text NOT NULL,
@@ -54,14 +76,9 @@ CREATE TABLE service_account (
project text REFERENCES project(resource_id)
);
-CREATE INDEX idx_service_account_project ON service_account(project) WHERE project IS NOT NULL;
-
-INSERT INTO service_account (deleted, created_at, name, email, service_key_hash, project)
-SELECT deleted, created_at, name, email, password_hash, project
-FROM principal WHERE type = 'SERVICE_ACCOUNT';
+CREATE INDEX IF NOT EXISTS idx_service_account_project ON service_account(project) WHERE project IS NOT NULL;
--- Step 2: Create workload_identity table
-CREATE TABLE workload_identity (
+CREATE TABLE IF NOT EXISTS workload_identity (
deleted boolean NOT NULL DEFAULT FALSE,
created_at timestamptz NOT NULL DEFAULT now(),
name text NOT NULL,
@@ -71,14 +88,9 @@ CREATE TABLE workload_identity (
config jsonb NOT NULL DEFAULT '{}'
);
-CREATE INDEX idx_workload_identity_project ON workload_identity(project) WHERE project IS NOT NULL;
-
-INSERT INTO workload_identity (deleted, created_at, name, email, project, config)
-SELECT deleted, created_at, name, email, project,
- COALESCE(profile->'workloadIdentityConfig', '{}')
-FROM principal WHERE type = 'WORKLOAD_IDENTITY';
+CREATE INDEX IF NOT EXISTS idx_workload_identity_project ON workload_identity(project) WHERE project IS NOT NULL;
--- Step 3: Drop FK constraints on creator/deleter columns that can reference SA/WI emails.
+-- Drop FK constraints on creator/deleter columns that can reference SA/WI emails.
-- Keep FKs on oauth2_authorization_code, oauth2_refresh_token, web_refresh_token (END_USER only).
ALTER TABLE plan DROP CONSTRAINT IF EXISTS plan_creator_fkey;
ALTER TABLE task_run DROP CONSTRAINT IF EXISTS task_run_creator_fkey;
@@ -91,13 +103,12 @@ ALTER TABLE revision DROP CONSTRAINT IF EXISTS revision_deleter_fkey;
ALTER TABLE release DROP CONSTRAINT IF EXISTS release_creator_fkey;
ALTER TABLE access_grant DROP CONSTRAINT IF EXISTS access_grant_creator_fkey;
--- Step 4: Add missing index for plan.creator (queried in plan filter)
-CREATE INDEX idx_plan_creator ON plan(creator);
+-- Add missing index for plan.creator (queried in plan filter).
+CREATE INDEX IF NOT EXISTS idx_plan_creator ON plan(creator);
--- Step 5: Clean up principal table
-DELETE FROM principal WHERE type != 'END_USER';
+-- Drop columns from principal (idempotent).
ALTER TABLE principal DROP CONSTRAINT IF EXISTS principal_project_type_check;
ALTER TABLE principal DROP CONSTRAINT IF EXISTS principal_type_check;
-ALTER TABLE principal DROP COLUMN type;
-ALTER TABLE principal DROP COLUMN project;
+ALTER TABLE principal DROP COLUMN IF EXISTS type;
+ALTER TABLE principal DROP COLUMN IF EXISTS project;
DROP INDEX IF EXISTS idx_principal_project;
diff --git a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql
index 0c5b720904..c4087afa3c 100644
--- a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql
+++ b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql
@@ -1,59 +1,67 @@
-- Drop unused id columns from tables that already use natural keys.
-- No other tables have FK references to these id columns.
--- For tables whose unique indexes are referenced by FK constraints from other tables
--- (project, instance, db), we promote the existing unique index to PK using
--- ALTER TABLE ... ADD CONSTRAINT ... PRIMARY KEY USING INDEX to avoid FK dependency errors.
-- project: natural key = resource_id (FK referenced by many tables)
-ALTER TABLE project DROP CONSTRAINT project_pkey;
-ALTER TABLE project DROP COLUMN id;
-ALTER TABLE project ADD CONSTRAINT project_pkey PRIMARY KEY USING INDEX idx_project_unique_resource_id;
+ALTER TABLE project DROP CONSTRAINT IF EXISTS project_pkey;
+ALTER TABLE project DROP COLUMN IF EXISTS id;
+DROP INDEX IF EXISTS idx_project_unique_resource_id;
+ALTER TABLE project ADD PRIMARY KEY (resource_id);
-- instance: natural key = resource_id (FK referenced by db, task)
-ALTER TABLE instance DROP CONSTRAINT instance_pkey;
-ALTER TABLE instance DROP COLUMN id;
-ALTER TABLE instance ADD CONSTRAINT instance_pkey PRIMARY KEY USING INDEX idx_instance_unique_resource_id;
+ALTER TABLE instance DROP CONSTRAINT IF EXISTS instance_pkey;
+ALTER TABLE instance DROP COLUMN IF EXISTS id;
+DROP INDEX IF EXISTS idx_instance_unique_resource_id;
+ALTER TABLE instance ADD PRIMARY KEY (resource_id);
-- db: natural key = (instance, name) (FK referenced by db_schema, revision, sync_history, changelog)
-ALTER TABLE db DROP CONSTRAINT db_pkey;
-ALTER TABLE db DROP COLUMN id;
-ALTER TABLE db ADD CONSTRAINT db_pkey PRIMARY KEY USING INDEX idx_db_unique_instance_name;
+ALTER TABLE db DROP CONSTRAINT IF EXISTS db_pkey;
+ALTER TABLE db DROP COLUMN IF EXISTS id;
+DROP INDEX IF EXISTS idx_db_unique_instance_name;
+ALTER TABLE db ADD PRIMARY KEY (instance, name);
-- setting: natural key = name
-ALTER TABLE setting DROP COLUMN id;
-ALTER TABLE setting ADD PRIMARY KEY (name);
+ALTER TABLE setting DROP CONSTRAINT IF EXISTS setting_pkey;
+ALTER TABLE setting DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_setting_unique_name;
+ALTER TABLE setting ADD PRIMARY KEY (name);
-- policy: natural key = (resource_type, resource, type)
-ALTER TABLE policy DROP COLUMN id;
-ALTER TABLE policy ADD PRIMARY KEY (resource_type, resource, type);
+ALTER TABLE policy DROP CONSTRAINT IF EXISTS policy_pkey;
+ALTER TABLE policy DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_policy_unique_resource_type_resource_type;
+ALTER TABLE policy ADD PRIMARY KEY (resource_type, resource, type);
-- idp: natural key = resource_id
-ALTER TABLE idp DROP COLUMN id;
-ALTER TABLE idp ADD PRIMARY KEY (resource_id);
+ALTER TABLE idp DROP CONSTRAINT IF EXISTS idp_pkey;
+ALTER TABLE idp DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_idp_unique_resource_id;
+ALTER TABLE idp ADD PRIMARY KEY (resource_id);
-- role: natural key = resource_id
-ALTER TABLE role DROP COLUMN id;
-ALTER TABLE role ADD PRIMARY KEY (resource_id);
+ALTER TABLE role DROP CONSTRAINT IF EXISTS role_pkey;
+ALTER TABLE role DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_role_unique_resource_id;
+ALTER TABLE role ADD PRIMARY KEY (resource_id);
-- db_schema: natural key = (instance, db_name)
-ALTER TABLE db_schema DROP COLUMN id;
-ALTER TABLE db_schema ADD PRIMARY KEY (instance, db_name);
+ALTER TABLE db_schema DROP CONSTRAINT IF EXISTS db_schema_pkey;
+ALTER TABLE db_schema DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_db_schema_unique_instance_db_name;
+ALTER TABLE db_schema ADD PRIMARY KEY (instance, db_name);
-- db_group: natural key = (project, resource_id)
-ALTER TABLE db_group DROP COLUMN id;
-ALTER TABLE db_group ADD PRIMARY KEY (project, resource_id);
+ALTER TABLE db_group DROP CONSTRAINT IF EXISTS db_group_pkey;
+ALTER TABLE db_group DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_db_group_unique_project_resource_id;
+ALTER TABLE db_group ADD PRIMARY KEY (project, resource_id);
-- task_run_log: use (task_run_id, created_at) as PK
-ALTER TABLE task_run_log DROP COLUMN id;
+ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;
+ALTER TABLE task_run_log DROP COLUMN IF EXISTS id;
ALTER TABLE task_run_log ADD PRIMARY KEY (task_run_id, created_at);
-- release: natural key = (project, train, iteration)
-ALTER TABLE release DROP COLUMN id;
-ALTER TABLE release ADD PRIMARY KEY (project, train, iteration);
+ALTER TABLE release DROP CONSTRAINT IF EXISTS release_pkey;
+ALTER TABLE release DROP COLUMN IF EXISTS id;
DROP INDEX IF EXISTS idx_release_project_train_iteration;
+ALTER TABLE release ADD PRIMARY KEY (project, train, iteration);
diff --git a/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql b/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql
index f375e6781f..6de01fcdbe 100644
--- a/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql
+++ b/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql
@@ -1,22 +1,22 @@
-- Add resource_id column to API-exposed tables that currently use integer IDs.
-- Use gen_random_uuid() as default for existing rows and new inserts.
-ALTER TABLE plan ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE task ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE task_run ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE issue ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE issue_comment ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE revision ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE changelog ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE worksheet ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-ALTER TABLE project_webhook ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE plan ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE task ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE task_run ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE issue ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE issue_comment ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE revision ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE changelog ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE worksheet ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
+ALTER TABLE project_webhook ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;
-CREATE UNIQUE INDEX idx_plan_unique_resource_id ON plan(resource_id);
-CREATE UNIQUE INDEX idx_task_unique_resource_id ON task(resource_id);
-CREATE UNIQUE INDEX idx_task_run_unique_resource_id ON task_run(resource_id);
-CREATE UNIQUE INDEX idx_issue_unique_resource_id ON issue(resource_id);
-CREATE UNIQUE INDEX idx_issue_comment_unique_resource_id ON issue_comment(resource_id);
-CREATE UNIQUE INDEX idx_revision_unique_resource_id ON revision(resource_id);
-CREATE UNIQUE INDEX idx_changelog_unique_resource_id ON changelog(resource_id);
-CREATE UNIQUE INDEX idx_worksheet_unique_resource_id ON worksheet(resource_id);
-CREATE UNIQUE INDEX idx_project_webhook_unique_resource_id ON project_webhook(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_plan_unique_resource_id ON plan(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_task_unique_resource_id ON task(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_task_run_unique_resource_id ON task_run(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_unique_resource_id ON issue(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_comment_unique_resource_id ON issue_comment(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_revision_unique_resource_id ON revision(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_changelog_unique_resource_id ON changelog(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_worksheet_unique_resource_id ON worksheet(resource_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_project_webhook_unique_resource_id ON project_webhook(resource_id);
diff --git a/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql b/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql
index 13965b3793..006a0a75d4 100644
--- a/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql
+++ b/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql
@@ -1,5 +1,6 @@
-- Drop unused id column from worksheet_organizer.
-- All queries use (worksheet_id, principal) which already has a unique index.
-ALTER TABLE worksheet_organizer DROP CONSTRAINT worksheet_organizer_pkey;
-ALTER TABLE worksheet_organizer DROP COLUMN id;
-ALTER TABLE worksheet_organizer ADD CONSTRAINT worksheet_organizer_pkey PRIMARY KEY USING INDEX idx_worksheet_organizer_unique_sheet_id_principal;
+ALTER TABLE worksheet_organizer DROP CONSTRAINT IF EXISTS worksheet_organizer_pkey;
+ALTER TABLE worksheet_organizer DROP COLUMN IF EXISTS id;
+DROP INDEX IF EXISTS idx_worksheet_organizer_unique_sheet_id_principal;
+ALTER TABLE worksheet_organizer ADD PRIMARY KEY (worksheet_id, principal);