Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

9.7 KB
Raw Blame
package hive

import (
	"context"
	"fmt"
	"io"
	"strings"

	// Import gohive v2 driver for database/sql registration.
	_ "github.com/beltran/gohive/v2"
	metadatapb "github.com/bytebase/omni/metadata"
	"github.com/pkg/errors"
)

const (
	schemaStmtFmt = "" +
		"--\n" +
		"-- %s structure for %s\n" +
		"--\n" +
		"%s;\n"
)

type IndexDDLOptions struct {
	databaseName          string
	indexName             string
	tableName             string
	indexType             string
	colNames              []string
	isWithDeferredRebuild bool
	idxProperties         map[string]string
	indexTableName        string
	rowFmt                string
	storedAs              string
	storedBy              string
	location              string
	tableProperties       map[string]string
	comment               string
}

type MaterializedViewDDLOptions struct {
	databaseName   string
	mtViewName     string
	disableRewrite bool
	comment        string
	partitionedOn  []string
	clusteredOn    []string
	distributedOn  []string
	sortedOn       []string
	rowFmt         string
	storedAs       string
	storedBy       string
	serdProperties map[string]string
	location       string
	tblProperties  map[string]string
	as             string
}

func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
	if d.db == nil {
		return errors.New("connection not initialized")
	}

	var builder strings.Builder

	databaseName := d.config.ConnectionContext.DatabaseName
	database, err := d.SyncDBSchema(ctx)
	if err != nil {
		return err
	}
	if len(database.Schemas) == 0 {
		return errors.New("database schemas is empty")
	}
	schema := database.Schemas[0]

	// Set current database
	if _, err := d.db.ExecContext(ctx, fmt.Sprintf("USE %s", databaseName)); err != nil {
		return errors.Wrap(err, "failed to set database")
	}

	// Dump managed tables
	for _, table := range schema.GetTables() {
		tableDDL, err := d.showCreateDDL(ctx, "TABLE", table.Name)
		if err != nil {
			return errors.Wrapf(err, "failed to dump table %s", table.Name)
		}

		// Dump indexes
		for _, index := range table.GetIndexes() {
			indexDDL, err := genIndexDDL(&IndexDDLOptions{
				databaseName:          databaseName,
				indexName:             index.Name,
				tableName:             table.Name,
				indexType:             index.Type,
				colNames:              index.Expressions,
				isWithDeferredRebuild: true,
				idxProperties:         nil,
				indexTableName:        "",
				rowFmt:                "",
				storedAs:              "",
				storedBy:              "",
				location:              "",
				tableProperties:       nil,
				comment:               index.Comment,
			})
			if err != nil {
				return errors.Wrapf(err, "failed to generate DDL for index %s", index.Name)
			}

			_, _ = fmt.Fprintf(&builder, schemaStmtFmt, "INDEX", fmt.Sprintf("`%s`", index.Name), indexDDL)
		}
		_, _ = builder.WriteString(tableDDL)
	}

	// Dump external tables
	for _, extTable := range schema.GetExternalTables() {
		tabDDL, err := d.showCreateDDL(ctx, "TABLE", extTable.Name)
		if err != nil {
			return errors.Wrapf(err, "failed to dump table %s", extTable.Name)
		}
		_, _ = builder.WriteString(tabDDL)
	}

	// Dump views
	for _, view := range schema.GetViews() {
		viewDDL, err := d.showCreateDDL(ctx, "VIEW", view.Name)
		if err != nil {
			return errors.Wrapf(err, "failed to dump view %s", view.Name)
		}
		_, _ = builder.WriteString(viewDDL)
	}

	// Dump materialized views
	for _, mtView := range schema.GetMaterializedViews() {
		mtViewDDL, err := genMaterializedViewDDL(&MaterializedViewDDLOptions{
			databaseName:   databaseName,
			mtViewName:     mtView.Name,
			disableRewrite: false,
			comment:        mtView.Comment,
			partitionedOn:  nil,
			clusteredOn:    nil,
			distributedOn:  nil,
			sortedOn:       nil,
			rowFmt:         "",
			storedAs:       "",
			storedBy:       "",
			serdProperties: nil,
			location:       "",
			tblProperties:  nil,
			as:             mtView.Definition,
		})
		if err != nil {
			return errors.Wrapf(err, "failed to generate DDL for materialized view %s", mtView.Name)
		}

		_, _ = fmt.Fprintf(&builder, schemaStmtFmt, "MATERIALIZED VIEW", fmt.Sprintf("`%s`.`%s`", databaseName, mtView.Name), mtViewDDL)
	}

	if _, err = io.WriteString(out, builder.String()); err != nil {
		return err
	}
	return nil
}

func (d *Driver) showCreateDDL(ctx context.Context, objectType string, objectName string) (string, error) {
	objectName = fmt.Sprintf("`%s`", objectName)

	// 'SHOW CREATE TABLE' can also be used for dumping views
	stmt := fmt.Sprintf("SHOW CREATE %s %s", objectType, objectName)
	if objectType == "VIEW" {
		stmt = fmt.Sprintf("SHOW CREATE TABLE %s", objectName)
	}

	rows, err := d.db.QueryContext(ctx, stmt)
	if err != nil {
		return "", errors.Wrapf(err, "failed to execute %s", stmt)
	}
	defer rows.Close()

	var schemaDDL strings.Builder
	for rows.Next() {
		var ddlLine string
		if err := rows.Scan(&ddlLine); err != nil {
			return "", errors.Wrap(err, "failed to scan DDL row")
		}
		schemaDDL.WriteString(ddlLine)
		schemaDDL.WriteString("\n")
	}

	if err := rows.Err(); err != nil {
		return "", errors.Wrap(err, "error reading DDL rows")
	}

	if schemaDDL.Len() == 0 {
		return "", errors.New("empty DDL result")
	}

	return fmt.Sprintf(schemaStmtFmt, objectType, objectName, schemaDDL.String()), nil
}

func genIndexDDL(opts *IndexDDLOptions) (string, error) {
	var builder strings.Builder

	// index name, table name.
	_, _ = fmt.Fprintf(&builder, "CREATE INDEX `%s`\nON TABLE %s ", opts.indexName, fmt.Sprintf("`%s`.`%s`", opts.databaseName, opts.tableName))

	// column names.
	formatColumn(&builder, opts.colNames)

	// index type.
	_, _ = fmt.Fprintf(&builder, ")\nAS '%s'\n", opts.indexType)

	// with deferred rebuild.
	if opts.isWithDeferredRebuild {
		_, _ = builder.WriteString("WITH DEFERRED REBUILD\n")
	}

	// index properties.
	if opts.idxProperties != nil {
		_, _ = builder.WriteString("IDXPROPERTIES ")
		formatKVPair(&builder, opts.tableProperties)
	}

	// index table.
	if opts.indexTableName != "" {
		_, _ = fmt.Fprintf(&builder, "IN TABLE %s\n", opts.indexTableName)
	}

	// row format.
	if opts.rowFmt != "" {
		_, _ = builder.WriteString(opts.rowFmt)
		_, _ = builder.WriteRune('\n')
	}

	// stored as or stored by.
	if opts.storedAs != "" && opts.storedBy != "" {
		return "", errors.New("keywords 'STORED AS' and 'STORED BY' cannot appear at the same time")
	}
	if opts.storedAs != "" {
		_, _ = fmt.Fprintf(&builder, "STORED AS %s\n", opts.storedAs)
	}
	if opts.storedBy != "" {
		_, _ = fmt.Fprintf(&builder, "STORED BY '%s'\n", opts.storedBy)
	}

	// location.
	if opts.location != "" {
		_, _ = builder.WriteString(opts.location)
		_, _ = builder.WriteRune('\n')
	}

	// table properties.
	if opts.tableProperties != nil {
		_, _ = builder.WriteString("TBLPROPERTIES (")
		formatKVPair(&builder, opts.tableProperties)
	}

	// comment.
	if opts.comment != "" {
		_, _ = fmt.Fprintf(&builder, "COMMENT '%s'\n", opts.comment)
	}

	return builder.String(), nil
}

func genMaterializedViewDDL(opts *MaterializedViewDDLOptions) (string, error) {
	var builder strings.Builder
	viewNameWithDB := fmt.Sprintf("`%s`.`%s`", opts.databaseName, opts.mtViewName)

	// mtView name.
	_, _ = fmt.Fprintf(&builder, "CREATE MATERIALIZED VIEW %s\n", viewNameWithDB)

	// disable rewrite.
	if opts.disableRewrite {
		_, _ = builder.WriteString("DISABLE REWRITE\n")
	}

	// comment.
	if opts.comment != "" {
		_, _ = fmt.Fprintf(&builder, "COMMENT '%s'\n", opts.comment)
	}

	// partitioned on.
	if opts.partitionedOn != nil {
		_, _ = builder.WriteString("PARTITIONED ON ")
		formatColumn(&builder, opts.partitionedOn)
	}

	if opts.clusteredOn != nil && opts.distributedOn != nil {
		return "", errors.New("keywords 'CLUSTERED ON' and 'DISTRIBUTED ON' cannot appear at the same time")
	}
	//   [CLUSTERED ON (col_name, ...) | DISTRIBUTED ON (col_name, ...) SORTED ON (col_name, ...)].
	if opts.distributedOn != nil && opts.sortedOn != nil {
		// distributed on.
		_, _ = builder.WriteString("DISTRIBUTED ON ")
		formatColumn(&builder, opts.distributedOn)

		// sorted on.
		_, _ = builder.WriteString("SORTED ON ")
		formatColumn(&builder, opts.sortedOn)
	}
	if opts.clusteredOn != nil {
		// clusteredOn
		_, _ = builder.WriteString("CLUSTERED ON ")
		formatColumn(&builder, opts.clusteredOn)
	}

	// stored as or stored by.
	if opts.storedAs != "" && opts.storedBy != "" {
		return "", errors.New("keywords 'STORED AS' and 'STORED BY' cannot appear at the same time")
	}
	if opts.storedAs != "" {
		_, _ = fmt.Fprintf(&builder, "STORED AS %s\n", opts.storedAs)
	}
	if opts.storedBy != "" {
		_, _ = fmt.Fprintf(&builder, "STORED BY '%s'\n", opts.storedBy)

		// if with serde properties.
		if opts.serdProperties != nil {
			_, _ = builder.WriteString("WITH SERDEPROPERTIES ")
			formatKVPair(&builder, opts.serdProperties)
		}
	}

	// location.
	if opts.location != "" {
		_, _ = builder.WriteString(opts.location)
		_, _ = builder.WriteRune('\n')
	}

	// table properties.
	if opts.tblProperties != nil {
		_, _ = builder.WriteString("TBLPROPERTIES ")
		formatKVPair(&builder, opts.tblProperties)
	}

	// as.
	_, _ = fmt.Fprintf(&builder, "AS %s \n", opts.as)

	return builder.String(), nil
}

func formatColumn(builder *strings.Builder, columnNames []string) {
	_, _ = builder.WriteString("(\n")
	for idx, colName := range columnNames {
		_, _ = fmt.Fprintf(builder, "  `%s`", colName)
		if idx != len(columnNames)-1 {
			_, _ = builder.WriteString(",\n")
		}
	}
	_, _ = builder.WriteString(")\n")
}

func formatKVPair(builder *strings.Builder, kvMap map[string]string) {
	_, _ = builder.WriteString("(\n")
	count := 0
	for key, value := range kvMap {
		count++
		_, _ = fmt.Fprintf(builder, "  '%s' = '%s'", key, value)
		if count != len(kvMap) {
			_, _ = builder.WriteString(",\n")
		}
	}
	_, _ = builder.WriteString(")\n")
}