Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: introducing query builder (#17801)
p0nye07fbf47a321
diff --git a/backend/common/qb/qb.go b/backend/common/qb/qb.gonew file mode 100644index 0000000000..3299846275--- /dev/null+++ b/backend/common/qb/qb.go@@ -0,0 +1,163 @@+package qb++import (+ "strconv"+ "strings"++ "github.com/pkg/errors"+)++// QueryPart represents a segment of a SQL query with text, parameters, and errors.+type QueryPart struct {+ Text string+ Params []any+ Errs []error+}++// Query builds SQL queries safely without string concatenation of user data.+// It stays close to raw SQL - you write SQL fragments with ? placeholders,+// and it handles converting them to PostgreSQL's $1, $2, $3 format.+type Query struct {+ parts []QueryPart+}++// Q creates a new empty Query.+func Q() *Query {+ return &Query{}+}++// Join adds a query part with a custom separator.+// The separator is prepended to the text (except for the first part).+func (q *Query) Join(separator, text string, params ...any) *Query {+ if q == nil {+ q = Q()+ }+ // Prepend separator if not the first part+ if len(q.parts) > 0 {+ text = separator + text+ }+ q.parts = append(q.parts, makePart(text, params...))+ return q+}++// makePart creates a QueryPart, expanding any *Query parameters inline.+func makePart(text string, params ...any) QueryPart {+ var newParams []any+ var errs []error++ for _, param := range params {+ // Check if the parameter is a *Query+ if nestedQuery, ok := param.(*Query); ok {+ // Expand the nested query inline - use toRawSql which keeps ? placeholders+ nestedSQL, nestedParams, err := nestedQuery.toRawSQL()+ if err != nil {+ errs = append(errs, errors.Wrap(err, "failed to expand nested query"))+ continue+ }+ // Replace the first ? with the nested SQL+ text = strings.Replace(text, "?", nestedSQL, 1)+ // Add the nested params to our param list+ newParams = append(newParams, nestedParams...)+ } else {+ // Regular parameter+ newParams = append(newParams, param)+ }+ }++ return QueryPart{+ Text: text,+ Params: newParams,+ Errs: errs,+ }+}++// toRawSQL concatenates all parts and returns SQL with ? placeholders (not $1, $2, etc).+// This is used internally for query composition.+func (q *Query) toRawSQL() (string, []any, error) {+ if q == nil {+ return "", nil, errors.New("cannot generate SQL from nil Query")+ }++ var sqlBuilder strings.Builder+ params := make([]any, 0)++ for _, part := range q.parts {+ // Check for errors in this part+ if len(part.Errs) > 0 {+ // Return the first error encountered+ return "", nil, part.Errs[0]+ }+ sqlBuilder.WriteString(part.Text)+ params = append(params, part.Params...)+ }++ return sqlBuilder.String(), params, nil+}++// Space adds a query part with a space separator. Convenience wrapper for Join(" ", ...).+func (q *Query) Space(text string, params ...any) *Query {+ return q.Join(" ", text, params...)+}++// And adds an AND condition.+func (q *Query) And(text string, params ...any) *Query {+ return q.Join(" AND ", text, params...)+}++// Or adds an OR condition.+func (q *Query) Or(text string, params ...any) *Query {+ return q.Join(" OR ", text, params...)+}++// Where adds a WHERE clause.+func (q *Query) Where(text string, params ...any) *Query {+ return q.Join(" WHERE ", text, params...)+}++// Len returns the number of parts in the query.+func (q *Query) Len() int {+ if q == nil {+ return 0+ }+ return len(q.parts)+}++// ToSQL generates PostgreSQL-compatible SQL with $1, $2, ... placeholders.+// Returns the SQL string, parameters slice, and any error.+func (q *Query) ToSQL() (string, []any, error) {+ if q == nil {+ return "", nil, errors.New("cannot generate SQL from nil Query")+ }++ // First pass: concatenate all parts and collect params+ var sqlBuilder strings.Builder+ params := make([]any, 0)++ for _, part := range q.parts {+ // Check for errors in this part+ if len(part.Errs) > 0 {+ // Return the first error encountered+ return "", nil, part.Errs[0]+ }+ sqlBuilder.WriteString(part.Text)+ params = append(params, part.Params...)+ }++ sql := sqlBuilder.String()++ // Second pass: replace ? placeholders with $1, $2, etc.+ sqlParts := strings.Split(sql, "?")+ if len(sqlParts)-1 != len(params) {+ return "", nil, errors.New("mismatched parameters: ? placeholders count does not match params count")+ }++ var builder strings.Builder+ for i := range params {+ builder.WriteString(sqlParts[i])+ builder.WriteString("$")+ builder.WriteString(strconv.Itoa(i + 1))+ }+ builder.WriteString(sqlParts[len(sqlParts)-1])++ return builder.String(), params, nil+}diff --git a/backend/common/qb/qb_test.go b/backend/common/qb/qb_test.gonew file mode 100644index 0000000000..7efbf6ae3f--- /dev/null+++ b/backend/common/qb/qb_test.go@@ -0,0 +1,309 @@+package qb++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestQuery_Basic(t *testing.T) {+ q := Q().Space("SELECT * FROM users")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users", sql)+ require.Empty(t, args)+}++func TestQuery_WithParameters(t *testing.T) {+ q := Q().Space("SELECT * FROM users").Where("id = ?", 123)+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE id = $1", sql)+ require.Equal(t, []any{123}, args)+}++func TestQuery_MultipleParameters(t *testing.T) {+ q := Q().Space("SELECT * FROM users").+ Where("id = ?", 123).+ And("active = ?", true).+ And("role = ?", "admin")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE id = $1 AND active = $2 AND role = $3", sql)+ require.Equal(t, []any{123, true, "admin"}, args)+}++func TestQuery_AndOr(t *testing.T) {+ q := Q().Space("SELECT * FROM users").+ Where("status = ?", "active").+ And("(role = ?", "admin").+ Or("role = ?)", "moderator")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE status = $1 AND (role = $2 OR role = $3)", sql)+ require.Equal(t, []any{"active", "admin", "moderator"}, args)+}++func TestQuery_Join(t *testing.T) {+ q := Q().Space("SELECT").+ Join(", ", "id").+ Join(", ", "name").+ Join(", ", "email").+ Space("FROM users")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT, id, name, email FROM users", sql)+ require.Empty(t, args)+}++func TestQuery_JoinWithParams(t *testing.T) {+ q := Q().Space("INSERT INTO users (name, email) VALUES").+ Join(", ", "(?, ?)", "Alice", "[email protected]").+ Join(", ", "(?, ?)", "Bob", "[email protected]")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "INSERT INTO users (name, email) VALUES, ($1, $2), ($3, $4)", sql)+ require.Equal(t, []any{"Alice", "[email protected]", "Bob", "[email protected]"}, args)+}++func TestQuery_NoParameters(t *testing.T) {+ q := Q().Space("SELECT * FROM users").+ Where("TRUE").+ And("deleted_at IS NULL")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE TRUE AND deleted_at IS NULL", sql)+ require.Empty(t, args)+}++func TestQuery_ComplexConditions(t *testing.T) {+ q := Q().Space("SELECT * FROM revision").+ Where("TRUE").+ And("instance = ?", "prod").+ And("db_name = ?", "main").+ And("payload->>'type' = ?", "MIGRATE").+ And("deleted_at IS NULL").+ Space("ORDER BY version DESC").+ Space("LIMIT ?", 10).+ Space("OFFSET ?", 20)+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM revision WHERE TRUE AND instance = $1 AND db_name = $2 AND payload->>'type' = $3 AND deleted_at IS NULL ORDER BY version DESC LIMIT $4 OFFSET $5", sql)+ require.Equal(t, []any{"prod", "main", "MIGRATE", 10, 20}, args)+}++func TestQuery_ArrayParameter(t *testing.T) {+ versions := []string{"v1", "v2", "v3"}+ q := Q().Space("SELECT * FROM revision").+ Where("version = ANY(?)", versions)+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM revision WHERE version = ANY($1)", sql)+ require.Equal(t, []any{versions}, args)+}++func TestQuery_NilQuery(t *testing.T) {+ var q *Query+ q = q.Space("SELECT * FROM users")+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users", sql)+ require.Empty(t, args)+}++func TestQuery_EmptyQuery(t *testing.T) {+ q := Q()+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "", sql)+ require.Empty(t, args)+}++func TestQuery_MismatchedParameters(t *testing.T) {+ q := Q().Space("SELECT * FROM users WHERE id = ? AND name = ?", 123)+ _, _, err := q.ToSQL()+ require.Error(t, err)+ require.Contains(t, err.Error(), "mismatched parameters")+}++func TestQuery_MultilineSQL(t *testing.T) {+ q := Q().Space(`+ SELECT+ id,+ name,+ FROM users+ `).Where("id = ?", 123)+ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Contains(t, sql, "SELECT")+ require.Contains(t, sql, "WHERE id = $1")+ require.Equal(t, []any{123}, args)+}++func TestQuery_ChainedCalls(t *testing.T) {+ // Test that we can chain calls continuously+ sql, args, err := Q().+ Space("SELECT * FROM users").+ Where("status = ?", "active").+ And("role = ?", "admin").+ Space("ORDER BY created_at DESC").+ ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE status = $1 AND role = $2 ORDER BY created_at DESC", sql)+ require.Equal(t, []any{"active", "admin"}, args)+}++func TestQuery_ConditionalBuilding(t *testing.T) {+ // Simulate building a query with optional filters+ id := 123+ name := "Alice"+ var status *string // nil++ q := Q().Space("SELECT * FROM users").Where("TRUE")++ if id != 0 {+ q.And("id = ?", id)+ }+ if name != "" {+ q.And("name = ?", name)+ }+ if status != nil { //nolint:govet+ q.And("status = ?", *status)+ }++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE TRUE AND id = $1 AND name = $2", sql)+ require.Equal(t, []any{123, "Alice"}, args)+}++func TestQuery_Composition(t *testing.T) {+ // Build query parts separately+ sel := Q().Space("SELECT *")+ from := Q().Space("FROM users")+ where := Q().Space("WHERE id = ?", 123)++ // Compose them together+ q := Q().Space("? ? ?", sel, from, where).Space("LIMIT ?", 10)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE id = $1 LIMIT $2", sql)+ require.Equal(t, []any{123, 10}, args)+}++func TestQuery_ConditionalComposition(t *testing.T) {+ getName := true+ getID := false+ filterAdult := true+ ageCheck := true+ filterChild := false++ // Build SELECT clause+ sel := Q().Space("SELECT")+ if getName {+ sel.Join(", ", "name")+ }+ if getID {+ sel.Join(", ", "id")+ }+ if !getName && !getID {+ sel.Join(", ", "*")+ }++ from := Q().Space("FROM my_table")++ // Build WHERE clause+ where := Q()+ hasConditions := false++ if filterAdult {+ adultCond := Q().Space("name = ?", "adult")+ if ageCheck {+ adultCond.And("age > ?", 20)+ }+ where.Space("WHERE (?)", adultCond)+ hasConditions = true+ }++ if filterChild {+ if hasConditions {+ where.Or("(name = ? AND age < ?)", "youth", 21)+ } else {+ where.Space("WHERE (name = ? AND age < ?)", "youth", 21)+ }+ }++ // Compose final query+ q := Q().Space("? ? ?", sel, from, where).Space("LIMIT ?", 10)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT, name FROM my_table WHERE (name = $1 AND age > $2) LIMIT $3", sql)+ require.Equal(t, []any{"adult", 20, 10}, args)+}++func TestQuery_NestedComposition(t *testing.T) {+ // Build a subquery+ subquery := Q().Space("SELECT user_id FROM orders WHERE total > ?", 100)++ // Use it in main query+ q := Q().Space("SELECT * FROM users").+ Where("id IN (?)", subquery)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT * FROM users WHERE id IN (SELECT user_id FROM orders WHERE total > $1)", sql)+ require.Equal(t, []any{100}, args)+}++func TestQuery_PracticalComposition(t *testing.T) {+ // Realistic example: build a complex query with reusable parts+ getUser := true+ getEmail := true+ filterActive := true+ filterRole := "admin"++ // Build SELECT clause dynamically+ sel := Q().Space("SELECT id")+ if getUser {+ sel.Join(", ", "name")+ }+ if getEmail {+ sel.Join(", ", "email")+ }++ // Build WHERE clause+ where := Q()+ if filterActive {+ where.Space("WHERE active = ?", true)+ }+ if filterRole != "" {+ if where.Len() > 0 {+ where.And("role = ?", filterRole)+ } else {+ where.Space("WHERE role = ?", filterRole)+ }+ }++ // Compose final query+ q := Q().Space("? FROM users ?", sel, where).+ Space("ORDER BY created_at DESC").+ Space("LIMIT ?", 10)++ sql, args, err := q.ToSQL()+ require.NoError(t, err)+ require.Equal(t, "SELECT id, name, email FROM users WHERE active = $1 AND role = $2 ORDER BY created_at DESC LIMIT $3", sql)+ require.Equal(t, []any{true, "admin", 10}, args)+}++func TestQuery_ErrorHandling(t *testing.T) {+ // Create a query with nil nested query to trigger error+ var nilQuery *Query+ q := Q().Space("SELECT * FROM users WHERE id IN (?)", nilQuery)++ _, _, err := q.ToSQL()+ require.Error(t, err)+ require.Contains(t, err.Error(), "nil Query")+}diff --git a/backend/store/revision.go b/backend/store/revision.goindex 1b9a19a78c..5b76576458 100644--- a/backend/store/revision.go+++ b/backend/store/revision.go@@ -3,14 +3,13 @@ package storeimport ("context""database/sql"- "fmt"- "strings""time""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson""github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/qb"storepb "github.com/bytebase/bytebase/backend/generated-go/store")@@ -45,59 +44,53 @@ type FindRevisionMessage struct {// ListRevisions lists revisions.// The results are ordered by version desc.func (s *Store) ListRevisions(ctx context.Context, find *FindRevisionMessage) ([]*RevisionMessage, error) {- where, args := []string{"TRUE"}, []any{}+ q := qb.Q().Space(`+ SELECT+ id,+ instance,+ db_name,+ created_at,+ deleter_id,+ deleted_at,+ version,+ payload+ FROM revision+ WHERE TRUE+ `)if v := find.UID; v != nil {- where = append(where, fmt.Sprintf("id = $%d", len(args)+1))- args = append(args, *v)+ q.And("id = ?", *v)}if v := find.InstanceID; v != nil {- where = append(where, fmt.Sprintf("instance = $%d", len(args)+1))- args = append(args, *v)+ q.And("instance = ?", *v)}if v := find.DatabaseName; v != nil {- where = append(where, fmt.Sprintf("db_name = $%d", len(args)+1))- args = append(args, *v)+ q.And("db_name = ?", *v)}if v := find.Type; v != nil {- where = append(where, fmt.Sprintf("payload->>'type' = $%d", len(args)+1))- args = append(args, v.String())+ q.And("payload->>'type' = ?", v.String())}if v := find.Version; v != nil {- where = append(where, fmt.Sprintf("version = $%d", len(args)+1))- args = append(args, *v)+ q.And("version = ?", *v)}if v := find.Versions; v != nil {- where = append(where, fmt.Sprintf("version = ANY($%d)", len(args)+1))- args = append(args, *v)+ q.And("version = ANY(?)", *v)}if !find.ShowDeleted {- where = append(where, "deleted_at IS NULL")+ q.And("deleted_at IS NULL")}-- limitOffsetClause := ""+ q.Space("ORDER BY version DESC")if v := find.Limit; v != nil {- limitOffsetClause += fmt.Sprintf(" LIMIT %d", *v)+ q.Space("LIMIT ?", *v)}if v := find.Offset; v != nil {- limitOffsetClause += fmt.Sprintf(" OFFSET %d", *v)+ q.Space("OFFSET ?", *v)}- query := fmt.Sprintf(`- SELECT- id,- instance,- db_name,- created_at,- deleter_id,- deleted_at,- version,- payload- FROM revision- WHERE %s- ORDER BY version DESC- %s- `, strings.Join(where, " AND "), limitOffsetClause)+ query, args, err := q.ToSQL()+ if err != nil {+ return nil, errors.Wrapf(err, "failed to build sql")+ }tx, err := s.GetDB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true})if err != nil {