Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
package hive
import (
"context"
"fmt"
"io"
"strings"
// Import gohive v2 driver for database/sql registration.
_ "github.com/beltran/gohive/v2"
metadatapb "github.com/bytebase/omni/metadata"
"github.com/pkg/errors"
)
const (
schemaStmtFmt = "" +
"--\n" +
"-- %s structure for %s\n" +
"--\n" +
"%s;\n"
)
type IndexDDLOptions struct {
databaseName string
indexName string
tableName string
indexType string
colNames []string
isWithDeferredRebuild bool
idxProperties map[string]string
indexTableName string
rowFmt string
storedAs string
storedBy string
location string
tableProperties map[string]string
comment string
}
type MaterializedViewDDLOptions struct {
databaseName string
mtViewName string
disableRewrite bool
comment string
partitionedOn []string
clusteredOn []string
distributedOn []string
sortedOn []string
rowFmt string
storedAs string
storedBy string
serdProperties map[string]string
location string
tblProperties map[string]string
as string
}
func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
if d.db == nil {
return errors.New("connection not initialized")
}
var builder strings.Builder
databaseName := d.config.ConnectionContext.DatabaseName
database, err := d.SyncDBSchema(ctx)
if err != nil {
return err
}
if len(database.Schemas) == 0 {
return errors.New("database schemas is empty")
}
schema := database.Schemas[0]
// Set current database
if _, err := d.db.ExecContext(ctx, fmt.Sprintf("USE %s", databaseName)); err != nil {
return errors.Wrap(err, "failed to set database")
}
// Dump managed tables
for _, table := range schema.GetTables() {
tableDDL, err := d.showCreateDDL(ctx, "TABLE", table.Name)
if err != nil {
return errors.Wrapf(err, "failed to dump table %s", table.Name)
}
// Dump indexes
for _, index := range table.GetIndexes() {
indexDDL, err := genIndexDDL(&IndexDDLOptions{
databaseName: databaseName,
indexName: index.Name,
tableName: table.Name,
indexType: index.Type,
colNames: index.Expressions,
isWithDeferredRebuild: true,
idxProperties: nil,
indexTableName: "",
rowFmt: "",
storedAs: "",
storedBy: "",
location: "",
tableProperties: nil,
comment: index.Comment,
})
if err != nil {
return errors.Wrapf(err, "failed to generate DDL for index %s", index.Name)
}
_, _ = fmt.Fprintf(&builder, schemaStmtFmt, "INDEX", fmt.Sprintf("`%s`", index.Name), indexDDL)
}
_, _ = builder.WriteString(tableDDL)
}
// Dump external tables
for _, extTable := range schema.GetExternalTables() {
tabDDL, err := d.showCreateDDL(ctx, "TABLE", extTable.Name)
if err != nil {
return errors.Wrapf(err, "failed to dump table %s", extTable.Name)
}
_, _ = builder.WriteString(tabDDL)
}
// Dump views
for _, view := range schema.GetViews() {
viewDDL, err := d.showCreateDDL(ctx, "VIEW", view.Name)
if err != nil {
return errors.Wrapf(err, "failed to dump view %s", view.Name)
}
_, _ = builder.WriteString(viewDDL)
}
// Dump materialized views
for _, mtView := range schema.GetMaterializedViews() {
mtViewDDL, err := genMaterializedViewDDL(&MaterializedViewDDLOptions{
databaseName: databaseName,
mtViewName: mtView.Name,
disableRewrite: false,
comment: mtView.Comment,
partitionedOn: nil,
clusteredOn: nil,
distributedOn: nil,
sortedOn: nil,
rowFmt: "",
storedAs: "",
storedBy: "",
serdProperties: nil,
location: "",
tblProperties: nil,
as: mtView.Definition,
})
if err != nil {
return errors.Wrapf(err, "failed to generate DDL for materialized view %s", mtView.Name)
}
_, _ = fmt.Fprintf(&builder, schemaStmtFmt, "MATERIALIZED VIEW", fmt.Sprintf("`%s`.`%s`", databaseName, mtView.Name), mtViewDDL)
}
if _, err = io.WriteString(out, builder.String()); err != nil {
return err
}
return nil
}
func (d *Driver) showCreateDDL(ctx context.Context, objectType string, objectName string) (string, error) {
objectName = fmt.Sprintf("`%s`", objectName)
// 'SHOW CREATE TABLE' can also be used for dumping views
stmt := fmt.Sprintf("SHOW CREATE %s %s", objectType, objectName)
if objectType == "VIEW" {
stmt = fmt.Sprintf("SHOW CREATE TABLE %s", objectName)
}
rows, err := d.db.QueryContext(ctx, stmt)
if err != nil {
return "", errors.Wrapf(err, "failed to execute %s", stmt)
}
defer rows.Close()
var schemaDDL strings.Builder
for rows.Next() {
var ddlLine string
if err := rows.Scan(&ddlLine); err != nil {
return "", errors.Wrap(err, "failed to scan DDL row")
}
schemaDDL.WriteString(ddlLine)
schemaDDL.WriteString("\n")
}
if err := rows.Err(); err != nil {
return "", errors.Wrap(err, "error reading DDL rows")
}
if schemaDDL.Len() == 0 {
return "", errors.New("empty DDL result")
}
return fmt.Sprintf(schemaStmtFmt, objectType, objectName, schemaDDL.String()), nil
}
func genIndexDDL(opts *IndexDDLOptions) (string, error) {
var builder strings.Builder
// index name, table name.
_, _ = fmt.Fprintf(&builder, "CREATE INDEX `%s`\nON TABLE %s ", opts.indexName, fmt.Sprintf("`%s`.`%s`", opts.databaseName, opts.tableName))
// column names.
formatColumn(&builder, opts.colNames)
// index type.
_, _ = fmt.Fprintf(&builder, ")\nAS '%s'\n", opts.indexType)
// with deferred rebuild.
if opts.isWithDeferredRebuild {
_, _ = builder.WriteString("WITH DEFERRED REBUILD\n")
}
// index properties.
if opts.idxProperties != nil {
_, _ = builder.WriteString("IDXPROPERTIES ")
formatKVPair(&builder, opts.tableProperties)
}
// index table.
if opts.indexTableName != "" {
_, _ = fmt.Fprintf(&builder, "IN TABLE %s\n", opts.indexTableName)
}
// row format.
if opts.rowFmt != "" {
_, _ = builder.WriteString(opts.rowFmt)
_, _ = builder.WriteRune('\n')
}
// stored as or stored by.
if opts.storedAs != "" && opts.storedBy != "" {
return "", errors.New("keywords 'STORED AS' and 'STORED BY' cannot appear at the same time")
}
if opts.storedAs != "" {
_, _ = fmt.Fprintf(&builder, "STORED AS %s\n", opts.storedAs)
}
if opts.storedBy != "" {
_, _ = fmt.Fprintf(&builder, "STORED BY '%s'\n", opts.storedBy)
}
// location.
if opts.location != "" {
_, _ = builder.WriteString(opts.location)
_, _ = builder.WriteRune('\n')
}
// table properties.
if opts.tableProperties != nil {
_, _ = builder.WriteString("TBLPROPERTIES (")
formatKVPair(&builder, opts.tableProperties)
}
// comment.
if opts.comment != "" {
_, _ = fmt.Fprintf(&builder, "COMMENT '%s'\n", opts.comment)
}
return builder.String(), nil
}
func genMaterializedViewDDL(opts *MaterializedViewDDLOptions) (string, error) {
var builder strings.Builder
viewNameWithDB := fmt.Sprintf("`%s`.`%s`", opts.databaseName, opts.mtViewName)
// mtView name.
_, _ = fmt.Fprintf(&builder, "CREATE MATERIALIZED VIEW %s\n", viewNameWithDB)
// disable rewrite.
if opts.disableRewrite {
_, _ = builder.WriteString("DISABLE REWRITE\n")
}
// comment.
if opts.comment != "" {
_, _ = fmt.Fprintf(&builder, "COMMENT '%s'\n", opts.comment)
}
// partitioned on.
if opts.partitionedOn != nil {
_, _ = builder.WriteString("PARTITIONED ON ")
formatColumn(&builder, opts.partitionedOn)
}
if opts.clusteredOn != nil && opts.distributedOn != nil {
return "", errors.New("keywords 'CLUSTERED ON' and 'DISTRIBUTED ON' cannot appear at the same time")
}
// [CLUSTERED ON (col_name, ...) | DISTRIBUTED ON (col_name, ...) SORTED ON (col_name, ...)].
if opts.distributedOn != nil && opts.sortedOn != nil {
// distributed on.
_, _ = builder.WriteString("DISTRIBUTED ON ")
formatColumn(&builder, opts.distributedOn)
// sorted on.
_, _ = builder.WriteString("SORTED ON ")
formatColumn(&builder, opts.sortedOn)
}
if opts.clusteredOn != nil {
// clusteredOn
_, _ = builder.WriteString("CLUSTERED ON ")
formatColumn(&builder, opts.clusteredOn)
}
// stored as or stored by.
if opts.storedAs != "" && opts.storedBy != "" {
return "", errors.New("keywords 'STORED AS' and 'STORED BY' cannot appear at the same time")
}
if opts.storedAs != "" {
_, _ = fmt.Fprintf(&builder, "STORED AS %s\n", opts.storedAs)
}
if opts.storedBy != "" {
_, _ = fmt.Fprintf(&builder, "STORED BY '%s'\n", opts.storedBy)
// if with serde properties.
if opts.serdProperties != nil {
_, _ = builder.WriteString("WITH SERDEPROPERTIES ")
formatKVPair(&builder, opts.serdProperties)
}
}
// location.
if opts.location != "" {
_, _ = builder.WriteString(opts.location)
_, _ = builder.WriteRune('\n')
}
// table properties.
if opts.tblProperties != nil {
_, _ = builder.WriteString("TBLPROPERTIES ")
formatKVPair(&builder, opts.tblProperties)
}
// as.
_, _ = fmt.Fprintf(&builder, "AS %s \n", opts.as)
return builder.String(), nil
}
func formatColumn(builder *strings.Builder, columnNames []string) {
_, _ = builder.WriteString("(\n")
for idx, colName := range columnNames {
_, _ = fmt.Fprintf(builder, " `%s`", colName)
if idx != len(columnNames)-1 {
_, _ = builder.WriteString(",\n")
}
}
_, _ = builder.WriteString(")\n")
}
func formatKVPair(builder *strings.Builder, kvMap map[string]string) {
_, _ = builder.WriteString("(\n")
count := 0
for key, value := range kvMap {
count++
_, _ = fmt.Fprintf(builder, " '%s' = '%s'", key, value)
if count != len(kvMap) {
_, _ = builder.WriteString(",\n")
}
}
_, _ = builder.WriteString(")\n")
}