Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(store): optimize ListProjects by batch fetching webhooks (#18946)

p0nyf201a1bca77e
diff --git a/backend/store/project.go b/backend/store/project.go
index c94da8dafb..268a86e7cb 100644
--- a/backend/store/project.go
+++ b/backend/store/project.go
@@ -82,12 +82,6 @@ func (s *Store) GetProject(ctx context.Context, find *FindProjectMessage) (*Proj
// ListProjects lists all projects.
func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*ProjectMessage, error) {
- tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
- if err != nil {
- return nil, err
- }
- defer tx.Rollback()
-
q := qb.Q().Space("SELECT resource_id, name, setting, deleted FROM project WHERE TRUE")
if filterQ := find.FilterQ; filterQ != nil {
q.And("?", filterQ)
@@ -118,13 +112,13 @@ func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*
q.Space("OFFSET ?", *v)
}
- sql, args, err := q.ToSQL()
+ query, args, err := q.ToSQL()
if err != nil {
return nil, err
}
var projectMessages []*ProjectMessage
- rows, err := tx.QueryContext(ctx, sql, args...)
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
@@ -152,17 +146,24 @@ func (s *Store) ListProjects(ctx context.Context, find *FindProjectMessage) ([]*
return nil, err
}
- if err := tx.Commit(); err != nil {
- return nil, err
- }
-
- // Fetch webhooks outside the transaction to prevent deadlock when connection pool is exhausted.
- for _, project := range projectMessages {
- projectWebhooks, err := s.ListProjectWebhooks(ctx, &FindProjectWebhookMessage{ProjectID: &project.ResourceID})
+ // Batch fetch webhooks for all projects in a single query.
+ if len(projectMessages) > 0 {
+ projectIDs := make([]string, 0, len(projectMessages))
+ for _, p := range projectMessages {
+ projectIDs = append(projectIDs, p.ResourceID)
+ }
+ webhooks, err := s.ListProjectWebhooks(ctx, &FindProjectWebhookMessage{ProjectIDs: projectIDs})
if err != nil {
return nil, err
}
- project.Webhooks = projectWebhooks
+ // Group webhooks by project ID.
+ webhooksByProject := make(map[string][]*ProjectWebhookMessage)
+ for _, w := range webhooks {
+ webhooksByProject[w.ProjectID] = append(webhooksByProject[w.ProjectID], w)
+ }
+ for _, project := range projectMessages {
+ project.Webhooks = webhooksByProject[project.ResourceID]
+ }
}
for _, project := range projectMessages {
diff --git a/backend/store/project_webhook.go b/backend/store/project_webhook.go
index 55d6da463c..cb6ba3e141 100644
--- a/backend/store/project_webhook.go
+++ b/backend/store/project_webhook.go
@@ -30,10 +30,11 @@ type UpdateProjectWebhookMessage struct {
// FindProjectWebhookMessage is the message for finding project webhooks,
// if all fields are nil, it will list all project webhooks.
type FindProjectWebhookMessage struct {
- ID *int
- ProjectID *string
- URL *string
- EventType *storepb.Activity_Type
+ ID *int
+ ProjectID *string
+ ProjectIDs []string
+ URL *string
+ EventType *storepb.Activity_Type
}
// CreateProjectWebhook creates an instance of ProjectWebhook.
@@ -89,11 +90,6 @@ func (s *Store) CreateProjectWebhook(ctx context.Context, projectID string, crea
// ListProjectWebhooks lists a list of ProjectWebhook instances.
func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhookMessage) ([]*ProjectWebhookMessage, error) {
- tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
- if err != nil {
- return nil, errors.Wrapf(err, "failed to begin transaction")
- }
-
q := qb.Q().Space(`
SELECT
id,
@@ -109,6 +105,9 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhoo
if v := find.ProjectID; v != nil {
q.And("project = ?", *v)
}
+ if len(find.ProjectIDs) > 0 {
+ q.And("project = ANY(?)", find.ProjectIDs)
+ }
if v := find.URL; v != nil {
q.And("payload->>'url' = ?", *v)
}
@@ -118,13 +117,13 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhoo
return nil, errors.Wrapf(err, "failed to build sql")
}
- rows, err := tx.QueryContext(ctx, query, args...)
+ rows, err := s.GetDB().QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
- var projectWebhooks []*ProjectWebhookMessage // Declare it here
+ var projectWebhooks []*ProjectWebhookMessage
for rows.Next() {
projectWebhook := ProjectWebhookMessage{
Payload: &storepb.ProjectWebhook{},
@@ -161,12 +160,7 @@ func (s *Store) ListProjectWebhooks(ctx context.Context, find *FindProjectWebhoo
if err := rows.Err(); err != nil {
return nil, err
}
- webhooks := projectWebhooks
-
- if err := tx.Commit(); err != nil {
- return nil, errors.Wrapf(err, "failed to commit transaction")
- }
- return webhooks, nil
+ return projectWebhooks, nil
}
// GetProjectWebhook gets an instance of ProjectWebhook.