Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
package databricks
import (
"context"
"errors"
"fmt"
"io"
"regexp"
"strings"
metadatapb "github.com/bytebase/omni/metadata"
"github.com/databricks/databricks-sdk-go/service/catalog"
)
const (
schemaStmtFmt = "" +
"--\n" +
"-- %s structure for %s\n" +
"--\n" +
"%s;\n"
)
var (
sysCatalog = "system"
infoSchema = "information_schema"
dftSchema = "default"
)
func (d *Driver) Dump(ctx context.Context, writer io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
catalogMap, err := d.listCatologTables(ctx, "")
if err != nil {
return err
}
for catalogName, schemaMap := range catalogMap {
if catalogName == sysCatalog {
continue
}
if _, err := fmt.Fprintf(writer, "CREATE CATALOG %s\n", catalogName); err != nil {
return err
}
for schemaName, tableList := range schemaMap {
if schemaName == infoSchema {
continue
}
if schemaName != dftSchema {
if _, err := fmt.Fprintf(writer, "CREATE SCHEMA `%s`.`%s`\n", catalogName, schemaName); err != nil {
return err
}
}
viewDDL := strings.Builder{}
mtViewDDL := strings.Builder{}
extTblDDL := strings.Builder{}
mngTblDDL := strings.Builder{}
for _, tblUnion := range tableList {
qualifiedName, err := getQualifiedTblName(catalogName, schemaName, tblUnion.name)
if err != nil {
return err
}
switch tblUnion.typeName {
case catalog.TableTypeView:
ddl, err := d.showCreateTable(ctx, qualifiedName)
if err != nil {
return err
}
formatDDL := fmt.Sprintf(schemaStmtFmt, "VIEW", qualifiedName, ddl)
if _, err := viewDDL.WriteString(formatDDL); err != nil {
return err
}
case catalog.TableTypeMaterializedView:
tblAddInfo, err := d.descTbl(ctx, qualifiedName)
if err != nil {
return err
}
tblProperties, err := formatTblProperties(tblAddInfo)
if err != nil {
return err
}
formatDDL, err := genMaterializedViewDDL(qualifiedName, tblUnion.materialView, tblProperties)
if err != nil {
return err
}
if _, err := mtViewDDL.WriteString(formatDDL); err != nil {
return err
}
case catalog.TableTypeExternal:
ddl, err := d.showCreateTable(ctx, qualifiedName)
if err != nil {
return err
}
formatDDL := fmt.Sprintf(schemaStmtFmt, "EXTERNAL TABLE", qualifiedName, ddl)
if _, err := extTblDDL.WriteString(formatDDL); err != nil {
return err
}
case catalog.TableTypeManaged:
ddl, err := d.showCreateTable(ctx, qualifiedName)
if err != nil {
return err
}
formatDDL := fmt.Sprintf(schemaStmtFmt, "MANAGED TABLE", qualifiedName, ddl)
if _, err := mngTblDDL.WriteString(formatDDL); err != nil {
return err
}
default:
// we do not sync streaming table.
continue
}
}
if _, err := writer.Write([]byte(mngTblDDL.String())); err != nil {
return err
}
if _, err := writer.Write([]byte(extTblDDL.String())); err != nil {
return err
}
if _, err := writer.Write([]byte(viewDDL.String())); err != nil {
return err
}
if _, err := writer.Write([]byte(mtViewDDL.String())); err != nil {
return err
}
}
}
return nil
}
func (d *Driver) showCreateTable(ctx context.Context, qualifiedTblName string) (string, error) {
rows, colInfo, err := d.execStatementSync(ctx, fmt.Sprintf("SHOW CREATE TABLE %s", qualifiedTblName), 0)
if err != nil {
return "", err
}
if len(rows) != 1 || len(colInfo) != 1 {
return "", errors.New("invalid ddl format")
}
return rows[0][0], nil
}
func genMaterializedViewDDL(qualifiedName string, mtViewMeta *metadatapb.MaterializedViewMetadata, tblProperties string) (string, error) {
builder := strings.Builder{}
if _, err := builder.WriteString(""); err != nil {
return "", err
}
if _, err := fmt.Fprintf(&builder, "CREATE MATERIALIZED VIEW %s\n", qualifiedName); err != nil {
return "", err
}
if _, err := fmt.Fprintf(&builder, "COMMENT %s\n", mtViewMeta.Comment); err != nil {
return "", err
}
if _, err := fmt.Fprintf(&builder, "TBLPROPERTIES %s\n", tblProperties); err != nil {
return "", err
}
return builder.String(), nil
}
type tableAdditionalInfo struct {
tblProps map[string]string
}
func (d *Driver) descTbl(ctx context.Context, tblName string) (*tableAdditionalInfo, error) {
tblAddInfo := &tableAdditionalInfo{
tblProps: make(map[string]string),
}
rows, _, err := d.execStatementSync(ctx, fmt.Sprintf("DESC FORMATTED %s", tblName), 0)
if err != nil {
return nil, err
}
for _, row := range rows {
infoKey := row[0]
infoVal := row[1]
if infoKey == "Table Properties" {
pattern := regexp.MustCompile("([a-zA-Z0-9-.]*)=([a-zA-Z0-9-.]*)")
matches := pattern.FindAllStringSubmatch(infoVal, -1)
if matches == nil {
continue
}
for _, match := range matches {
tblAddInfo.tblProps[match[1]] = match[2]
}
}
}
return tblAddInfo, nil
}
func formatTblProperties(tblAddInfo *tableAdditionalInfo) (string, error) {
if tblAddInfo == nil || tblAddInfo.tblProps == nil {
return "", errors.New("properties cannot be nil value")
}
builder := strings.Builder{}
idx := 0
for key, val := range tblAddInfo.tblProps {
comma := ","
if idx == len(tblAddInfo.tblProps)-1 {
comma = ""
}
if _, err := fmt.Fprintf(&builder, "%s = %s%s\n", key, val, comma); err != nil {
return "", err
}
idx++
}
return builder.String(), nil
}