Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: drop task_run_log primary key to unblock 3.16.2 upgrades
Danny14bc19b6f46b
diff --git a/AGENTS.md b/AGENTS.mdindex 5084f1adcc..b357d88b03 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -198,7 +198,10 @@ The canonical frontend ownership map is in `./frontend/AGENTS.md`. In summary:Several tables use composite primary keys (e.g., `(project, id)`). Check`backend/migrator/migration/LATEST.sql` for the full list — any table with a-multi-column PRIMARY KEY.+multi-column PRIMARY KEY. `task_run_log` deliberately has no primary key (it is+an append-only log whose entries can share a `created_at` microsecond,+BYT-10035) but is equally project-scoped, so the same predicate rules apply to+it.When writing or modifying queries on these tables:- Every WHERE, JOIN, USING, DELETE, and UPDATE predicate must include everydiff --git a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sqlindex 5808b8b1d0..16f22b2eba 100644--- a/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql+++ b/backend/migrator/migration/3.16/0002##drop_unused_id_columns.sql@@ -109,10 +109,11 @@ BEGINEND IF;END $$;--- task_run_log: use (task_run_id, created_at) as PK+-- task_run_log: append-only log with no natural key. Entries for one task run+-- can share a created_at microsecond (BYT-10035), so no primary key is added;+-- the pre-existing idx_task_run_log_task_run_id index keeps lookups fast.ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;ALTER TABLE task_run_log DROP COLUMN IF EXISTS id;-ALTER TABLE task_run_log ADD PRIMARY KEY (task_run_id, created_at);-- release: natural key = (project, train, iteration)ALTER TABLE release DROP CONSTRAINT IF EXISTS release_pkey;diff --git a/backend/migrator/migration/3.17/0001##add_project_to_plan_chain.sql b/backend/migrator/migration/3.17/0001##add_project_to_plan_chain.sqlindex d5ad44828f..87666a9276 100644--- a/backend/migrator/migration/3.17/0001##add_project_to_plan_chain.sql+++ b/backend/migrator/migration/3.17/0001##add_project_to_plan_chain.sql@@ -57,17 +57,12 @@ END $$;ALTER TABLE task_run_log ADD COLUMN IF NOT EXISTS project TEXT;UPDATE task_run_log SET project = task_run.project FROM task_run WHERE task_run_log.project IS NULL AND task_run_log.task_run_id = task_run.id;ALTER TABLE task_run_log ALTER COLUMN project SET NOT NULL;-DO $$ BEGIN- IF NOT EXISTS (- SELECT 1 FROM pg_constraint- WHERE conname = 'task_run_log_pkey'- AND conrelid = 'task_run_log'::regclass- AND array_length(conkey, 1) > 1- ) THEN- ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;- ALTER TABLE task_run_log ADD PRIMARY KEY (project, task_run_id, created_at);- END IF;-END $$;+-- Append-only log with no natural key: entries for one task run can share a+-- created_at microsecond (BYT-10035), so index the read path instead of adding+-- a primary key.+ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;+DROP INDEX IF EXISTS idx_task_run_log_task_run_id;+CREATE INDEX IF NOT EXISTS idx_task_run_log_project_task_run_id_created_at ON task_run_log(project, task_run_id, created_at);-- plan_check_runALTER TABLE plan_check_run ADD COLUMN IF NOT EXISTS project TEXT;diff --git a/backend/migrator/migration/3.22/0004##drop_task_run_log_pkey.sql b/backend/migrator/migration/3.22/0004##drop_task_run_log_pkey.sqlnew file mode 100644index 0000000000..a7d74a32d4--- /dev/null+++ b/backend/migrator/migration/3.22/0004##drop_task_run_log_pkey.sql@@ -0,0 +1,9 @@+-- task_run_log is an append-only log with no natural key: entries for one task+-- run can legitimately share a created_at microsecond, and legacy data already+-- does (BYT-10035). Databases that upgraded through the original 3.16.2 carry+-- PRIMARY KEY (task_run_id, created_at) while fresh installs carry+-- PRIMARY KEY (project, task_run_id, created_at); drop whichever is present and+-- index the read path instead.+ALTER TABLE task_run_log DROP CONSTRAINT IF EXISTS task_run_log_pkey;+DROP INDEX IF EXISTS idx_task_run_log_task_run_id;+CREATE INDEX IF NOT EXISTS idx_task_run_log_project_task_run_id_created_at ON task_run_log(project, task_run_id, created_at);diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex 92250d9eec..51ce6009e2 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -596,16 +596,19 @@ CREATE TABLE replica_heartbeat (last_heartbeat TIMESTAMPTZ NOT NULL);+-- Append-only log with no primary key on purpose: entries for one task run can+-- legitimately share a created_at microsecond (BYT-10035).CREATE TABLE task_run_log (project text NOT NULL REFERENCES project(resource_id),task_run_id integer NOT NULL,created_at timestamptz NOT NULL DEFAULT now(),-- Stored as TaskRunLog (proto/store/store/task_run_log.proto)payload jsonb NOT NULL DEFAULT '{}',- PRIMARY KEY (project, task_run_id, created_at),FOREIGN KEY (project, task_run_id) REFERENCES task_run(project, id));+CREATE INDEX idx_task_run_log_project_task_run_id_created_at ON task_run_log(project, task_run_id, created_at);+------------------------- OAuth2 and auth-----------------------diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 09a5cb9f66..3fff38297e 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -2,6 +2,7 @@ package migratorimport ("context"+ "fmt""testing""time"@@ -17,8 +18,8 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.22.3"), *files[len(files)-1].version)- require.Equal(t, "migration/3.22/0003##hash_directory_sync_token.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.22.4"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.22/0004##drop_task_run_log_pkey.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {@@ -36,6 +37,190 @@ func TestVersionUnique(t *testing.T) {}}+// TestMigration3_16_2_TaskRunLogDuplicateTimestamps verifies that the 3.16.2+// id-column cleanup succeeds on legacy task_run_log data holding duplicate+// (task_run_id, created_at) pairs, which were legal under the old id primary+// key. Regression test for BYT-10035.+func TestMigration3_16_2_TaskRunLogDuplicateTimestamps(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(ctx) })++ db := container.GetDB()++ // Minimal pre-3.16.2 shapes for every table the migration touches.+ setup := `+ CREATE TABLE project (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);+ CREATE TABLE instance (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);+ CREATE TABLE db (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, name TEXT NOT NULL);+ CREATE TABLE setting (id BIGSERIAL PRIMARY KEY, name TEXT NOT NULL);+ CREATE TABLE policy (id BIGSERIAL PRIMARY KEY, resource_type TEXT NOT NULL, resource TEXT NOT NULL, type TEXT NOT NULL);+ CREATE TABLE idp (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);+ CREATE TABLE role (id BIGSERIAL PRIMARY KEY, resource_id TEXT NOT NULL);+ CREATE TABLE db_schema (id BIGSERIAL PRIMARY KEY, instance TEXT NOT NULL, db_name TEXT NOT NULL);+ CREATE TABLE db_group (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, resource_id TEXT NOT NULL);+ CREATE TABLE release (id BIGSERIAL PRIMARY KEY, project TEXT NOT NULL, train TEXT NOT NULL, iteration INT NOT NULL);+ CREATE TABLE task_run_log (+ id BIGSERIAL PRIMARY KEY,+ task_run_id INTEGER NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ payload JSONB NOT NULL DEFAULT '{}'+ );+ CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);++ INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES+ (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),+ (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),+ (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');+ `+ _, err := db.ExecContext(ctx, setup)+ require.NoError(t, err)++ statement, err := migrationFS.ReadFile("migration/3.16/0002##drop_unused_id_columns.sql")+ require.NoError(t, err)+ _, err = db.ExecContext(ctx, string(statement))+ require.NoError(t, err)++ var rowCount int+ require.NoError(t, db.QueryRowContext(ctx, `SELECT COUNT(*) FROM task_run_log`).Scan(&rowCount))+ require.Equal(t, 3, rowCount)++ var hasPkey bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)+ `).Scan(&hasPkey))+ require.False(t, hasPkey)++ var hasIDColumn bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'task_run_log' AND column_name = 'id')+ `).Scan(&hasIDColumn))+ require.False(t, hasIDColumn)+}++// TestMigration3_17_1_TaskRunLogDuplicateTimestamps verifies that the 3.17.1+// project-scoping migration accepts duplicate (task_run_id, created_at) pairs:+// it must backfill project and index the table without imposing uniqueness.+// Regression test for BYT-10035.+func TestMigration3_17_1_TaskRunLogDuplicateTimestamps(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(ctx) })++ db := container.GetDB()++ // Minimal post-3.16.2 shapes for the plan-chain tables the migration touches;+ // task_run_log has no id column and no primary key at this point.+ setup := `+ CREATE TABLE project (resource_id TEXT PRIMARY KEY);+ CREATE TABLE plan (id BIGINT PRIMARY KEY, project TEXT NOT NULL);+ CREATE TABLE issue (id BIGINT PRIMARY KEY, project TEXT NOT NULL, plan_id BIGINT);+ CREATE TABLE task (id INT PRIMARY KEY, plan_id BIGINT NOT NULL, environment TEXT NOT NULL);+ CREATE TABLE task_run (id INT PRIMARY KEY, task_id INT NOT NULL, attempt INT NOT NULL);+ CREATE TABLE plan_check_run (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);+ CREATE TABLE plan_webhook_delivery (id INT PRIMARY KEY, plan_id BIGINT NOT NULL);+ CREATE TABLE task_run_log (+ task_run_id INTEGER NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ payload JSONB NOT NULL DEFAULT '{}'+ );+ CREATE INDEX idx_task_run_log_task_run_id ON task_run_log(task_run_id);++ INSERT INTO project VALUES ('proj-a');+ INSERT INTO plan VALUES (1, 'proj-a');+ INSERT INTO task VALUES (10, 1, 'environments/prod');+ INSERT INTO task_run VALUES (100, 10, 1);+ INSERT INTO task_run_log (task_run_id, created_at, payload) VALUES+ (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_EXECUTE"}'),+ (100, '2026-01-01 00:00:00.000001+00', '{"type":"COMMAND_RESPONSE"}'),+ (100, '2026-01-01 00:00:00.000002+00', '{"type":"COMMAND_EXECUTE"}');+ `+ _, err := db.ExecContext(ctx, setup)+ require.NoError(t, err)++ statement, err := migrationFS.ReadFile("migration/3.17/0001##add_project_to_plan_chain.sql")+ require.NoError(t, err)+ _, err = db.ExecContext(ctx, string(statement))+ require.NoError(t, err)++ var backfilledCount int+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT COUNT(*) FROM task_run_log WHERE project = 'proj-a'+ `).Scan(&backfilledCount))+ require.Equal(t, 3, backfilledCount)++ var hasPkey bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)+ `).Scan(&hasPkey))+ require.False(t, hasPkey)++ var hasIndex bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')+ `).Scan(&hasIndex))+ require.True(t, hasIndex)++ var hasOldIndex bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_task_run_id')+ `).Scan(&hasOldIndex))+ require.False(t, hasOldIndex)+}++// TestMigration3_22_4_DropTaskRunLogPkey verifies that the pkey drop converges+// both historical shapes — the 2-column PK left by upgrading through the+// original 3.16.2 and the 3-column PK of fresh installs — is idempotent, and+// that same-microsecond entries insert afterwards instead of colliding.+func TestMigration3_22_4_DropTaskRunLogPkey(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.GetTestPgContainer(ctx, t)+ t.Cleanup(func() { container.Close(ctx) })++ db := container.GetDB()++ statement, err := migrationFS.ReadFile("migration/3.22/0004##drop_task_run_log_pkey.sql")+ require.NoError(t, err)++ for _, pkey := range []string{"(task_run_id, created_at)", "(project, task_run_id, created_at)"} {+ _, err := db.ExecContext(ctx, fmt.Sprintf(`+ DROP TABLE IF EXISTS task_run_log;+ CREATE TABLE task_run_log (+ project TEXT NOT NULL,+ task_run_id INTEGER NOT NULL,+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ payload JSONB NOT NULL DEFAULT '{}',+ PRIMARY KEY %s+ );`, pkey))+ require.NoError(t, err)++ _, err = db.ExecContext(ctx, string(statement))+ require.NoError(t, err)+ // Re-running must be a no-op: the file also applies to databases that+ // come through the edited 3.16.2/3.17.1 path with no pkey left at all.+ _, err = db.ExecContext(ctx, string(statement))+ require.NoError(t, err)++ var hasPkey bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'task_run_log_pkey' AND conrelid = 'task_run_log'::regclass)+ `).Scan(&hasPkey))+ require.False(t, hasPkey, "pkey %s should be dropped", pkey)++ var hasIndex bool+ require.NoError(t, db.QueryRowContext(ctx, `+ SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = 'public' AND indexname = 'idx_task_run_log_project_task_run_id_created_at')+ `).Scan(&hasIndex))+ require.True(t, hasIndex)++ _, err = db.ExecContext(ctx, `+ INSERT INTO task_run_log (project, task_run_id, created_at) VALUES+ ('proj-a', 100, '2026-01-01 00:00:00.000001+00'),+ ('proj-a', 100, '2026-01-01 00:00:00.000001+00')`)+ require.NoError(t, err, "same-microsecond entries must both insert")+ }+}+func TestMigration3_21_2_MigrateInstanceSyncDatabases(t *testing.T) {ctx := context.Background()container := testcontainer.GetTestPgContainer(ctx, t)diff --git a/docs/pre-pr-checklist.md b/docs/pre-pr-checklist.mdindex 30d2ee45cc..2981888a6d 100644--- a/docs/pre-pr-checklist.md+++ b/docs/pre-pr-checklist.md@@ -58,10 +58,14 @@ Read `backend/migrator/migration/LATEST.sql` and find every table with a multi-c- `issue (project, id)`- `task (project, id)`- `task_run (project, id)`-- `task_run_log (project, task_run_id, created_at)`- `db_group (project, resource_id)`- `release (project, train, iteration)`+`task_run_log` deliberately has no primary key (append-only log; entries for one+task run can share a `created_at` microsecond — BYT-10035), but it is equally+project-scoped: every predicate on it must include `project` alongside+`task_run_id`.+Always verify against LATEST.sql — tables may have been added since this list waslast updated. Cross-reference with the tables your diff touches.