Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(store): optimize ListProjects by batch fetching webhooks (#18946)
p0nyf201a1bca77e
diff --git a/backend/store/project.go b/backend/store/project.goindex c94da8dafb..268a86e7cb 100644--- a/backend/store/project.go+++ b/backend/store/project.go@@ -82,12 +82,6 @@ func (s *Store) GetProject(ctx context.Context, find *FindProjectMessage) (*Proj// ListProjects lists all projects.func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*ProjectMessage, error) {- tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})- if err != nil {- return nil, err- }- defer tx.Rollback()-q := qb.Q().Space("SELECT resource_id, name, setting, deleted FROM project WHERE TRUE")if filterQ := find.FilterQ; filterQ != nil {q.And("?", filterQ)@@ -118,13 +112,13 @@ func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*q.Space("OFFSET ?", *v)}- sql, args, err := q.ToSQL()+ query, args, err := q.ToSQL()if err != nil {return nil, err}var projectMessages []*ProjectMessage- rows, err := tx.QueryContext(ctx, sql, args...)+ rows, err := s.GetDB().QueryContext(ctx, query, args...)if err != nil {return nil, err}@@ -152,17 +146,24 @@ func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*return nil, err}- if err := tx.Commit(); err != nil {- return nil, err- }-- // Fetch webhooks outside the transaction to prevent deadlock when connection pool is exhausted.- for _, project := range projectMessages {- projectWebhooks, err := s.ListProjectWebhooks(ctx, &FindProjectWebhookMessage{ProjectID: &project.ResourceID})+ // Batch fetch webhooks for all projects in a single query.+ if len(projectMessages) > 0 {+ projectIDs := make([]string, 0, len(projectMessages))+ for _, p := range projectMessages {+ projectIDs = append(projectIDs, p.ResourceID)+ }+ webhooks, err := s.ListProjectWebhooks(ctx, &FindProjectWebhookMessage{ProjectIDs: projectIDs})if err != nil {return nil, err}- project.Webhooks = projectWebhooks+ // Group webhooks by project ID.+ webhooksByProject := make(map[string][]*ProjectWebhookMessage)+ for _, w := range webhooks {+ webhooksByProject[w.ProjectID] = append(webhooksByProject[w.ProjectID], w)+ }+ for _, project := range projectMessages {+ project.Webhooks = webhooksByProject[project.ResourceID]+ }}for _, project := range projectMessages {diff --git a/backend/store/project_webhook.go b/backend/store/project_webhook.goindex 55d6da463c..cb6ba3e141 100644--- a/backend/store/project_webhook.go+++ b/backend/store/project_webhook.go@@ -30,10 +30,11 @@ type UpdateProjectWebhookMessage struct {// FindProjectWebhookMessage is the message for finding project webhooks,// if all fields are nil, it will list all project webhooks.type FindProjectWebhookMessage struct {- ID *int- ProjectID *string- URL *string- EventType *storepb.Activity_Type+ ID *int+ ProjectID *string+ ProjectIDs []string+ URL *string+ EventType *storepb.Activity_Type}// CreateProjectWebhook creates an instance of ProjectWebhook.@@ -89,11 +90,6 @@ func (s *Store) CreateProjectWebhook(ctx context.Context, projectID string, crea// ListProjectWebhooks lists a list of ProjectWebhook instances.func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhookMessage) ([]*ProjectWebhookMessage, error) {- tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})- if err != nil {- return nil, errors.Wrapf(err, "failed to begin transaction")- }-q := qb.Q().Space(`SELECTid,@@ -109,6 +105,9 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhooif v := find.ProjectID; v != nil {q.And("project = ?", *v)}+ if len(find.ProjectIDs) > 0 {+ q.And("project = ANY(?)", find.ProjectIDs)+ }if v := find.URL; v != nil {q.And("payload->>'url' = ?", *v)}@@ -118,13 +117,13 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhooreturn nil, errors.Wrapf(err, "failed to build sql")}- rows, err := tx.QueryContext(ctx, query, args...)+ rows, err := s.GetDB().QueryContext(ctx, query, args...)if err != nil {return nil, err}defer rows.Close()- var projectWebhooks []*ProjectWebhookMessage // Declare it here+ var projectWebhooks []*ProjectWebhookMessagefor rows.Next() {projectWebhook := ProjectWebhookMessage{Payload: &storepb.ProjectWebhook{},@@ -161,12 +160,7 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhooif err := rows.Err(); err != nil {return nil, err}- webhooks := projectWebhooks-- if err := tx.Commit(); err != nil {- return nil, errors.Wrapf(err, "failed to commit transaction")- }- return webhooks, nil+ return projectWebhooks, nil}// GetProjectWebhook gets an instance of ProjectWebhook.