Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: unify the project column and fk (#19626)
ecmadao42b4d4669b1d
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex a491b1292a..937d79cf84 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -1226,7 +1226,7 @@ func doEncrypt(exports []*encryptContent, password string) ([]byte, error) {func (s *SQLService) createQueryHistory(database *store.DatabaseMessage, queryType store.QueryHistoryType, statement string, userEmail string, duration time.Duration, queryErr error) {qh := &store.QueryHistoryMessage{Creator: userEmail,- ProjectID: database.ProjectID,+ Project: database.ProjectID,Database: common.FormatDatabase(database.InstanceID, database.DatabaseName),Statement: statement,Type: queryType,diff --git a/backend/migrator/migration/3.17/0007##fix_project_fk_references.sql b/backend/migrator/migration/3.17/0007##fix_project_fk_references.sqlnew file mode 100644index 0000000000..3e1113b3e3--- /dev/null+++ b/backend/migrator/migration/3.17/0007##fix_project_fk_references.sql@@ -0,0 +1,23 @@+-- Add missing direct REFERENCES project(resource_id) to tables that only have composite FKs.+-- Rename query_history.project_id to project and add FK reference.++DO $$+DECLARE+ t text;+BEGIN+ -- Add direct project FK to tables that only had composite FKs.+ FOREACH t IN ARRAY ARRAY['plan_check_run', 'plan_webhook_delivery', 'task_run', 'task_run_log', 'issue_comment', 'query_history'] LOOP+ IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = t || '_project_fkey') THEN+ EXECUTE format('ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (project) REFERENCES project(resource_id)', t, t || '_project_fkey');+ END IF;+ END LOOP;++ -- Rename query_history.project_id to project.+ IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'query_history' AND column_name = 'project_id') THEN+ ALTER TABLE query_history RENAME COLUMN project_id TO project;+ END IF;+END $$;++-- Recreate the index with the new column name.+DROP INDEX IF EXISTS idx_query_history_creator_created_at_project_id;+CREATE INDEX IF NOT EXISTS idx_query_history_creator_created_at_project ON query_history(creator, created_at, project DESC);diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex d830ab03c3..2a353013c8 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -185,7 +185,7 @@ CREATE TABLE plan_check_run (id serial,created_at timestamptz NOT NULL DEFAULT now(),updated_at timestamptz NOT NULL DEFAULT now(),- project text NOT NULL,+ project text NOT NULL REFERENCES project(resource_id),plan_id bigint NOT NULL,status text NOT NULL CHECK (status IN ('AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),-- Stored as PlanCheckRunResult (proto/store/store/plan_check_run.proto)@@ -204,7 +204,7 @@ ALTER SEQUENCE plan_check_run_id_seq RESTART WITH 101;-- One row per plan at any time - mutually exclusive events.-- Row is deleted when user clicks BatchRunTasks to reset notification state.CREATE TABLE plan_webhook_delivery (- project TEXT NOT NULL,+ project TEXT NOT NULL REFERENCES project(resource_id),plan_id BIGINT NOT NULL,-- Event type: 'PIPELINE_FAILED' or 'PIPELINE_COMPLETED'event_type TEXT NOT NULL,@@ -238,7 +238,7 @@ CREATE TABLE task_run (creator text,created_at timestamptz NOT NULL DEFAULT now(),updated_at timestamptz NOT NULL DEFAULT now(),- project text NOT NULL,+ project text NOT NULL REFERENCES project(resource_id),task_id integer NOT NULL,attempt integer NOT NULL,status text NOT NULL CHECK (status IN ('PENDING', 'AVAILABLE', 'RUNNING', 'DONE', 'FAILED', 'CANCELED')),@@ -275,7 +275,7 @@ CREATE TABLE replica_heartbeat ();CREATE TABLE task_run_log (- project text NOT NULL,+ project text NOT NULL REFERENCES project(resource_id),task_run_id integer NOT NULL,created_at timestamptz NOT NULL DEFAULT now(),-- Stored as TaskRunLog (proto/store/store/task_run_log.proto)@@ -351,7 +351,7 @@ CREATE TABLE issue_comment (creator text NOT NULL,created_at timestamptz NOT NULL DEFAULT now(),updated_at timestamptz NOT NULL DEFAULT now(),- project text NOT NULL,+ project text NOT NULL REFERENCES project(resource_id),issue_id integer NOT NULL,-- Stored as IssueCommentPayload (proto/store/store/issue_comment.proto)payload jsonb NOT NULL DEFAULT '{}',@@ -366,7 +366,7 @@ CREATE TABLE query_history (id bigserial PRIMARY KEY,creator text NOT NULL,created_at timestamptz NOT NULL DEFAULT now(),- project_id text NOT NULL, -- the project resource id+ project text NOT NULL REFERENCES project(resource_id),database text NOT NULL, -- the database resource name, for example, instances/{instance}/databases/{database}statement text NOT NULL,-- type: QUERY, EXPORT@@ -376,7 +376,7 @@ CREATE TABLE query_history (payload jsonb NOT NULL DEFAULT '{}');-CREATE INDEX idx_query_history_creator_created_at_project_id ON query_history(creator, created_at, project_id DESC);+CREATE INDEX idx_query_history_creator_created_at_project ON query_history(creator, created_at, project DESC);ALTER SEQUENCE query_history_id_seq RESTART WITH 101;diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex a2451ec03e..fe778020df 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -15,8 +15,8 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.17.6"), *files[len(files)-1].version)- require.Equal(t, "migration/3.17/0006##per_project_auto_increment.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.17.7"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.17/0007##fix_project_fk_references.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {diff --git a/backend/store/project.go b/backend/store/project.goindex 789cbdd22b..49bf986903 100644--- a/backend/store/project.go+++ b/backend/store/project.go@@ -316,7 +316,7 @@ func (s *Store) DeleteProject(ctx context.Context, resourceID string) error {defer tx.Rollback()// Delete query_history entries that reference this project- q := qb.Q().Space("DELETE FROM query_history WHERE project_id = ?", resourceID)+ q := qb.Q().Space("DELETE FROM query_history WHERE project = ?", resourceID)sql, args, err := q.ToSQL()if err != nil {return errors.Wrap(err, "failed to build query_history delete query")diff --git a/backend/store/query_history.go b/backend/store/query_history.goindex cac83178e5..cdb1b053c7 100644--- a/backend/store/query_history.go+++ b/backend/store/query_history.go@@ -33,8 +33,8 @@ type QueryHistoryMessage struct {// Related fieldsCreator string- // ProjectID is the project resource id.- ProjectID string+ // Project is the project resource id.+ Project string// Domain specific fieldsType QueryHistoryType@@ -46,8 +46,8 @@ type QueryHistoryMessage struct {// FindQueryHistoryMessage is the API message for finding query histories.type FindQueryHistoryMessage struct {- Creator *string- ProjectID *string+ Creator *string+ Project *string// Instance is the instance resource name like instances/{instance}.Instance *string// Database is database resource name like instances/{instance}/databases/{database}.@@ -69,7 +69,7 @@ func (s *Store) CreateQueryHistory(ctx context.Context, create *QueryHistoryMessq := qb.Q().Space(`INSERT INTO query_history (creator,- project_id,+ project,database,statement,type,@@ -81,7 +81,7 @@ func (s *Store) CreateQueryHistory(ctx context.Context, create *QueryHistoryMesscreated_at`,create.Creator,- create.ProjectID,+ create.Project,create.Database,create.Statement,create.Type,@@ -110,7 +110,7 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMequery_history.id,query_history.creator,query_history.created_at,- query_history.project_id,+ query_history.project,query_history.database,query_history.statement,query_history.type,@@ -125,8 +125,8 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMeif v := find.Creator; v != nil {q.And("query_history.creator = ?", *v)}- if v := find.ProjectID; v != nil {- q.And("query_history.project_id = ?", *v)+ if v := find.Project; v != nil {+ q.And("query_history.project = ?", *v)}if v := find.Instance; v != nil {q.And("query_history.database LIKE ?", *v)@@ -164,7 +164,7 @@ func (s *Store) ListQueryHistories(ctx context.Context, find *FindQueryHistoryMe&queryHistory.UID,&queryHistory.Creator,&queryHistory.CreatedAt,- &queryHistory.ProjectID,+ &queryHistory.Project,&queryHistory.Database,&queryHistory.Statement,&queryHistory.Type,@@ -211,7 +211,7 @@ func GetListQueryHistoryFilter(filter string) (*qb.Query, error) {if err != nil {return nil, errors.Errorf("invalid project filter %q", value)}- return qb.Q().Space("query_history.project_id = ?", projectID), nil+ return qb.Q().Space("query_history.project = ?", projectID), nilcase "database":return qb.Q().Space("query_history.database = ?", value.(string)), nilcase "instance":diff --git a/backend/store/query_history_filter_test.go b/backend/store/query_history_filter_test.goindex fed50dda74..be29047aa9 100644--- a/backend/store/query_history_filter_test.go+++ b/backend/store/query_history_filter_test.go@@ -25,7 +25,7 @@ func TestGetListQueryHistoryFilter(t *testing.T) {{name: "project filter",filter: `project == "projects/test-project"`,- wantSQL: "(query_history.project_id = $1)",+ wantSQL: "(query_history.project = $1)",wantArgs: []any{"test-project"},wantErr: false,},@@ -67,21 +67,21 @@ func TestGetListQueryHistoryFilter(t *testing.T) {{name: "AND condition with project and database",filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db"`,- wantSQL: "((query_history.project_id = $1 AND query_history.database = $2))",+ wantSQL: "((query_history.project = $1 AND query_history.database = $2))",wantArgs: []any{"test-project", "instances/test-instance/databases/test-db"},wantErr: false,},{name: "AND condition with three filters",filter: `project == "projects/test-project" && database == "instances/test-instance/databases/test-db" && type == "QUERY"`,- wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND query_history.type = $3))",+ wantSQL: "(((query_history.project = $1 AND query_history.database = $2) AND query_history.type = $3))",wantArgs: []any{"test-project", "instances/test-instance/databases/test-db", QueryHistoryType("QUERY")},wantErr: false,},{name: "OR condition with project and database",filter: `project == "projects/project1" || project == "projects/project2"`,- wantSQL: "((query_history.project_id = $1 OR query_history.project_id = $2))",+ wantSQL: "((query_history.project = $1 OR query_history.project = $2))",wantArgs: []any{"project1", "project2"},wantErr: false,},@@ -95,14 +95,14 @@ func TestGetListQueryHistoryFilter(t *testing.T) {{name: "complex nested AND/OR",filter: `(project == "projects/p1" || project == "projects/p2") && type == "QUERY"`,- wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) AND query_history.type = $3))",+ wantSQL: "(((query_history.project = $1 OR query_history.project = $2) AND query_history.type = $3))",wantArgs: []any{"p1", "p2", QueryHistoryType("QUERY")},wantErr: false,},{name: "complex nested with statement matches",filter: `project == "projects/test" && statement.matches("SELECT") && type == "QUERY"`,- wantSQL: "(((query_history.project_id = $1 AND query_history.statement LIKE $2) AND query_history.type = $3))",+ wantSQL: "(((query_history.project = $1 AND query_history.statement LIKE $2) AND query_history.type = $3))",wantArgs: []any{"test", "%SELECT%", QueryHistoryType("QUERY")},wantErr: false,},@@ -197,7 +197,7 @@ func TestGetListQueryHistoryFilter_EdgeCases(t *testing.T) {name: "multiple OR conditions",filter: `project == "projects/p1" || project == "projects/p2" || project == "projects/p3"`,description: "multiple OR conditions should be chained",- wantSQL: "(((query_history.project_id = $1 OR query_history.project_id = $2) OR query_history.project_id = $3))",+ wantSQL: "(((query_history.project = $1 OR query_history.project = $2) OR query_history.project = $3))",wantArgs: []any{"p1", "p2", "p3"},wantErr: false,},@@ -221,7 +221,7 @@ func TestGetListQueryHistoryFilter_EdgeCases(t *testing.T) {name: "complex filter with all supported fields",filter: `project == "projects/test" && database == "instances/i1/databases/db1" && type == "EXPORT" && statement.matches("INSERT")`,description: "combination of all supported filter types",- wantSQL: "(((query_history.project_id = $1 AND query_history.database = $2) AND (query_history.type = $3 AND query_history.statement LIKE $4)))",+ wantSQL: "(((query_history.project = $1 AND query_history.database = $2) AND (query_history.type = $3 AND query_history.statement LIKE $4)))",wantArgs: []any{"test", "instances/i1/databases/db1", QueryHistoryType("EXPORT"), "%INSERT%"},wantErr: false,},@@ -261,7 +261,7 @@ func TestGetListQueryHistoryFilter_CompareWithOriginal(t *testing.T) {{name: "simple project filter",filter: `project == "projects/test-project"`,- expectedConditions: []string{"query_history.project_id = $1"},+ expectedConditions: []string{"query_history.project = $1"},},{name: "database filter",@@ -271,7 +271,7 @@ func TestGetListQueryHistoryFilter_CompareWithOriginal(t *testing.T) {{name: "type and project combined",filter: `type == "QUERY" && project == "projects/my-project"`,- expectedConditions: []string{"query_history.type = $1", "AND", "query_history.project_id = $2"},+ expectedConditions: []string{"query_history.type = $1", "AND", "query_history.project = $2"},},{name: "statement matches",