Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: make migration sql idempotent (#19615)
ecmadao3786af7c9bc7
diff --git a/backend/migrator/migration/3.16/0000##split_principal_table.sql b/backend/migrator/migration/3.16/0000##split_principal_table.sqlindex e789267a9f..5a79dc9649 100644--- a/backend/migrator/migration/3.16/0000##split_principal_table.sql+++ b/backend/migrator/migration/3.16/0000##split_principal_table.sql@@ -1,15 +1,26 @@-- Split principal table into three tables: principal (END_USER only), service_account, workload_identity.--- Step 0: Fix legacy service account emails that don't match the required format.--- Valid formats: {name}@service.bytebase.com OR {name}@{project}.service.bytebase.com--- This MUST run BEFORE dropping FK constraints so that ON UPDATE CASCADE propagates email--- changes to all creator/deleter columns automatically.+-- Step 0: Fix legacy emails, create tables, migrate data, clean up principal.+-- All data migration is guarded by checking principal.type column existence,+-- which is dropped at the end — so re-runs skip all data operations.DO $$DECLARErec RECORD;new_email TEXT;base_local TEXT;BEGIN+ -- Only run data migration if principal still has a type column (i.e., not yet split).+ IF NOT EXISTS (+ SELECT 1 FROM information_schema.columns+ WHERE table_name = 'principal' AND column_name = 'type'+ ) THEN+ RETURN;+ END IF;++ -- Fix legacy service account emails that don't match the required format.+ -- Valid formats: {name}@service.bytebase.com OR {name}@{project}.service.bytebase.com+ -- This MUST run BEFORE dropping FK constraints so that ON UPDATE CASCADE propagates email+ -- changes to all creator/deleter columns automatically.FOR rec INSELECT id, email, projectFROM principal@@ -23,7 +34,6 @@ BEGINELSEnew_email := base_local || '@service.bytebase.com';END IF;- -- Handle potential email collision by appending idIF EXISTS (SELECT 1 FROM principal WHERE email = new_email AND id != rec.id) THENIF rec.project IS NOT NULL THENnew_email := base_local || '-' || rec.id || '@' || rec.project || '.service.bytebase.com';@@ -31,21 +41,33 @@ BEGINnew_email := base_local || '-' || rec.id || '@service.bytebase.com';END IF;END IF;- -- Update principal email (ON UPDATE CASCADE propagates to creator/deleter columns)UPDATE principal SET email = new_email WHERE id = rec.id;- -- Update JSONB references in policies (members use 'serviceAccounts/{email}' format)UPDATE policySET payload = replace(payload::text, 'serviceAccounts/' || rec.email, 'serviceAccounts/' || new_email)::jsonbWHERE payload::text LIKE '%serviceAccounts/' || rec.email || '%';- -- Update user_group member referencesUPDATE user_groupSET payload = replace(payload::text, 'serviceAccounts/' || rec.email, 'serviceAccounts/' || new_email)::jsonbWHERE payload::text LIKE '%serviceAccounts/' || rec.email || '%';END LOOP;++ -- Migrate data into new tables.+ INSERT INTO service_account (deleted, created_at, name, email, service_key_hash, project)+ SELECT deleted, created_at, name, email, password_hash, project+ FROM principal WHERE type = 'SERVICE_ACCOUNT'+ ON CONFLICT (email) DO NOTHING;++ INSERT INTO workload_identity (deleted, created_at, name, email, project, config)+ SELECT deleted, created_at, name, email, project,+ COALESCE(profile->'workloadIdentityConfig', '{}')+ FROM principal WHERE type = 'WORKLOAD_IDENTITY'+ ON CONFLICT (email) DO NOTHING;++ -- Clean up principal table.+ DELETE FROM principal WHERE type != 'END_USER';END $$;--- Step 1: Create service_account table-CREATE TABLE service_account (+-- Create tables (idempotent).+CREATE TABLE IF NOT EXISTS service_account (deleted boolean NOT NULL DEFAULT FALSE,created_at timestamptz NOT NULL DEFAULT now(),name text NOT NULL,@@ -54,14 +76,9 @@ CREATE TABLE service_account (project text REFERENCES project(resource_id));-CREATE INDEX idx_service_account_project ON service_account(project) WHERE project IS NOT NULL;--INSERT INTO service_account (deleted, created_at, name, email, service_key_hash, project)-SELECT deleted, created_at, name, email, password_hash, project-FROM principal WHERE type = 'SERVICE_ACCOUNT';+CREATE INDEX IF NOT EXISTS idx_service_account_project ON service_account(project) WHERE project IS NOT NULL;--- Step 2: Create workload_identity table-CREATE TABLE workload_identity (+CREATE TABLE IF NOT EXISTS workload_identity (deleted boolean NOT NULL DEFAULT FALSE,created_at timestamptz NOT NULL DEFAULT now(),name text NOT NULL,@@ -71,14 +88,9 @@ CREATE TABLE workload_identity (config jsonb NOT NULL DEFAULT '{}');-CREATE INDEX idx_workload_identity_project ON workload_identity(project) WHERE project IS NOT NULL;--INSERT INTO workload_identity (deleted, created_at, name, email, project, config)-SELECT deleted, created_at, name, email, project,- COALESCE(profile->'workloadIdentityConfig', '{}')-FROM principal WHERE type = 'WORKLOAD_IDENTITY';+CREATE INDEX IF NOT EXISTS idx_workload_identity_project ON workload_identity(project) WHERE project IS NOT NULL;--- Step 3: Drop FK constraints on creator/deleter columns that can reference SA/WI emails.+-- Drop FK constraints on creator/deleter columns that can reference SA/WI emails.-- Keep FKs on oauth2_authorization_code, oauth2_refresh_token, web_refresh_token (END_USER only).ALTER TABLE plan DROP CONSTRAINT IF EXISTS plan_creator_fkey;ALTER TABLE task_run DROP CONSTRAINT IF EXISTS task_run_creator_fkey;@@ -91,13 +103,12 @@ ALTER TABLE revision DROP CONSTRAINT IF EXISTS revision_deleter_fkey;ALTER TABLE release DROP CONSTRAINT IF EXISTS release_creator_fkey;ALTER TABLE access_grant DROP CONSTRAINT IF EXISTS access_grant_creator_fkey;--- Step 4: Add missing index for plan.creator (queried in plan filter)-CREATE INDEX idx_plan_creator ON plan(creator);+-- Add missing index for plan.creator (queried in plan filter).+CREATE INDEX IF NOT EXISTS idx_plan_creator ON plan(creator);--- Step 5: Clean up principal table-DELETE FROM principal WHERE type != 'END_USER';+-- Drop columns from principal (idempotent).ALTER TABLE principal DROP CONSTRAINT IF EXISTS principal_project_type_check;ALTER TABLE principal DROP CONSTRAINT IF EXISTS principal_type_check;-ALTER TABLE principal DROP COLUMN type;-ALTER TABLE principal DROP COLUMN project;+ALTER TABLE principal DROP COLUMN IF EXISTS type;+ALTER TABLE principal DROP COLUMN IF EXISTS project;DROP INDEX IF EXISTS idx_principal_project;diff --git a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sqlindex 0c5b720904..c4087afa3c 100644--- a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql+++ b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql@@ -1,59 +1,67 @@-- Drop unused id columns from tables that already use natural keys.-- No other tables have FK references to these id columns.--- For tables whose unique indexes are referenced by FK constraints from other tables--- (project, instance, db), we promote the existing unique index to PK using--- ALTER TABLE ... ADD CONSTRAINT ... PRIMARY KEY USING INDEX to avoid FK dependency errors.-- project: natural key = resource_id (FK referenced by many tables)-ALTER TABLE project DROP CONSTRAINT project_pkey;-ALTER TABLE project DROP COLUMN id;-ALTER TABLE project ADD CONSTRAINT project_pkey PRIMARY KEY USING INDEX idx_project_unique_resource_id;+ALTER TABLE project DROP CONSTRAINT IF EXISTS project_pkey;+ALTER TABLE project DROP COLUMN IF EXISTS id;+DROP INDEX IF EXISTS idx_project_unique_resource_id;+ALTER TABLE project ADD PRIMARY KEY (resource_id);-- instance: natural key = resource_id (FK referenced by db, task)-ALTER TABLE instance DROP CONSTRAINT instance_pkey;-ALTER TABLE instance DROP COLUMN id;-ALTER TABLE instance ADD CONSTRAINT instance_pkey PRIMARY KEY USING INDEX idx_instance_unique_resource_id;+ALTER TABLE instance DROP CONSTRAINT IF EXISTS instance_pkey;+ALTER TABLE instance DROP COLUMN IF EXISTS id;+DROP INDEX IF EXISTS idx_instance_unique_resource_id;+ALTER TABLE instance ADD PRIMARY KEY (resource_id);-- db: natural key = (instance, name) (FK referenced by db_schema, revision, sync_history, changelog)-ALTER TABLE db DROP CONSTRAINT db_pkey;-ALTER TABLE db DROP COLUMN id;-ALTER TABLE db ADD CONSTRAINT db_pkey PRIMARY KEY USING INDEX idx_db_unique_instance_name;+ALTER TABLE db DROP CONSTRAINT IF EXISTS db_pkey;+ALTER TABLE db DROP COLUMN IF EXISTS id;+DROP INDEX IF EXISTS idx_db_unique_instance_name;+ALTER TABLE db ADD PRIMARY KEY (instance, name);-- setting: natural key = name-ALTER TABLE setting DROP COLUMN id;-ALTER TABLE setting ADD PRIMARY KEY (name);+ALTER TABLE setting DROP CONSTRAINT IF EXISTS setting_pkey;+ALTER TABLE setting DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_setting_unique_name;+ALTER TABLE setting ADD PRIMARY KEY (name);-- policy: natural key = (resource_type, resource, type)-ALTER TABLE policy DROP COLUMN id;-ALTER TABLE policy ADD PRIMARY KEY (resource_type, resource, type);+ALTER TABLE policy DROP CONSTRAINT IF EXISTS policy_pkey;+ALTER TABLE policy DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_policy_unique_resource_type_resource_type;+ALTER TABLE policy ADD PRIMARY KEY (resource_type, resource, type);-- idp: natural key = resource_id-ALTER TABLE idp DROP COLUMN id;-ALTER TABLE idp ADD PRIMARY KEY (resource_id);+ALTER TABLE idp DROP CONSTRAINT IF EXISTS idp_pkey;+ALTER TABLE idp DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_idp_unique_resource_id;+ALTER TABLE idp ADD PRIMARY KEY (resource_id);-- role: natural key = resource_id-ALTER TABLE role DROP COLUMN id;-ALTER TABLE role ADD PRIMARY KEY (resource_id);+ALTER TABLE role DROP CONSTRAINT IF EXISTS role_pkey;+ALTER TABLE role DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_role_unique_resource_id;+ALTER TABLE role ADD PRIMARY KEY (resource_id);-- db_schema: natural key = (instance, db_name)-ALTER TABLE db_schema DROP COLUMN id;-ALTER TABLE db_schema ADD PRIMARY KEY (instance, db_name);+ALTER TABLE db_schema DROP CONSTRAINT IF EXISTS db_schema_pkey;+ALTER TABLE db_schema DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_db_schema_unique_instance_db_name;+ALTER TABLE db_schema ADD PRIMARY KEY (instance, db_name);-- db_group: natural key = (project, resource_id)-ALTER TABLE db_group DROP COLUMN id;-ALTER TABLE db_group ADD PRIMARY KEY (project, resource_id);+ALTER TABLE db_group DROP CONSTRAINT IF EXISTS db_group_pkey;+ALTER TABLE db_group DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_db_group_unique_project_resource_id;+ALTER TABLE db_group ADD PRIMARY KEY (project, resource_id);-- task_run_log: use (task_run_id, created_at) as PK-ALTER TABLE task_run_log DROP COLUMN id;+ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;+ALTER TABLE task_run_log DROP COLUMN IF EXISTS id;ALTER TABLE task_run_log ADD PRIMARY KEY (task_run_id, created_at);-- release: natural key = (project, train, iteration)-ALTER TABLE release DROP COLUMN id;-ALTER TABLE release ADD PRIMARY KEY (project, train, iteration);+ALTER TABLE release DROP CONSTRAINT IF EXISTS release_pkey;+ALTER TABLE release DROP COLUMN IF EXISTS id;DROP INDEX IF EXISTS idx_release_project_train_iteration;+ALTER TABLE release ADD PRIMARY KEY (project, train, iteration);diff --git a/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql b/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sqlindex f375e6781f..6de01fcdbe 100644--- a/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql+++ b/backend/migrator/migration/3.16/0003##add_resource_id_to_api_tables.sql@@ -1,22 +1,22 @@-- Add resource_id column to API-exposed tables that currently use integer IDs.-- Use gen_random_uuid() as default for existing rows and new inserts.-ALTER TABLE plan ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE task ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE task_run ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE issue ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE issue_comment ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE revision ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE changelog ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE worksheet ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-ALTER TABLE project_webhook ADD COLUMN resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE plan ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE task ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE task_run ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE issue ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE issue_comment ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE revision ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE changelog ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE worksheet ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;+ALTER TABLE project_webhook ADD COLUMN IF NOT EXISTS resource_id text NOT NULL DEFAULT gen_random_uuid()::text;-CREATE UNIQUE INDEX idx_plan_unique_resource_id ON plan(resource_id);-CREATE UNIQUE INDEX idx_task_unique_resource_id ON task(resource_id);-CREATE UNIQUE INDEX idx_task_run_unique_resource_id ON task_run(resource_id);-CREATE UNIQUE INDEX idx_issue_unique_resource_id ON issue(resource_id);-CREATE UNIQUE INDEX idx_issue_comment_unique_resource_id ON issue_comment(resource_id);-CREATE UNIQUE INDEX idx_revision_unique_resource_id ON revision(resource_id);-CREATE UNIQUE INDEX idx_changelog_unique_resource_id ON changelog(resource_id);-CREATE UNIQUE INDEX idx_worksheet_unique_resource_id ON worksheet(resource_id);-CREATE UNIQUE INDEX idx_project_webhook_unique_resource_id ON project_webhook(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_plan_unique_resource_id ON plan(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_task_unique_resource_id ON task(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_task_run_unique_resource_id ON task_run(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_unique_resource_id ON issue(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_issue_comment_unique_resource_id ON issue_comment(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_revision_unique_resource_id ON revision(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_changelog_unique_resource_id ON changelog(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_worksheet_unique_resource_id ON worksheet(resource_id);+CREATE UNIQUE INDEX IF NOT EXISTS idx_project_webhook_unique_resource_id ON project_webhook(resource_id);diff --git a/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql b/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sqlindex 13965b3793..006a0a75d4 100644--- a/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql+++ b/backend/migrator/migration/3.16/0004##drop_worksheet_organizer_id.sql@@ -1,5 +1,6 @@-- Drop unused id column from worksheet_organizer.-- All queries use (worksheet_id, principal) which already has a unique index.-ALTER TABLE worksheet_organizer DROP CONSTRAINT worksheet_organizer_pkey;-ALTER TABLE worksheet_organizer DROP COLUMN id;-ALTER TABLE worksheet_organizer ADD CONSTRAINT worksheet_organizer_pkey PRIMARY KEY USING INDEX idx_worksheet_organizer_unique_sheet_id_principal;+ALTER TABLE worksheet_organizer DROP CONSTRAINT IF EXISTS worksheet_organizer_pkey;+ALTER TABLE worksheet_organizer DROP COLUMN IF EXISTS id;+DROP INDEX IF EXISTS idx_worksheet_organizer_unique_sheet_id_principal;+ALTER TABLE worksheet_organizer ADD PRIMARY KEY (worksheet_id, principal);