Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
package databricks
import (
"context"
"fmt"
"strings"
metadatapb "github.com/bytebase/omni/metadata"
"github.com/databricks/databricks-sdk-go/service/catalog"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/plugin/db"
)
type tableUnion struct {
externalTable *metadatapb.ExternalTableMetadata
table *metadatapb.TableMetadata
view *metadatapb.ViewMetadata
materialView *metadatapb.MaterializedViewMetadata
typeName catalog.TableType
name string
}
type databricksSchemaMap = map[string][]*tableUnion
type databricksCatalogMap = map[string]databricksSchemaMap
// sync catalog.
func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMetadata, error) {
// return nothing if no catalogs are specified.
if d.curCatalog == "" {
return nil, nil
}
// fetch table data from databricks.
catalogMap, err := d.listCatologTables(ctx, d.curCatalog)
if err != nil {
return nil, err
}
dbMetadataMeta := metadatapb.DatabaseSchemaMetadata{}
schemaMap, ok := (catalogMap)[d.curCatalog]
if !ok {
return nil, errors.Errorf("cannot find metadata for catalog '%s'", d.curCatalog)
}
dbMetadataMeta.Name = d.curCatalog
schemas := convertToStorepbSchemas(schemaMap)
dbMetadataMeta.Schemas = schemas
return &dbMetadataMeta, nil
}
func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {
instanceMetadata := &db.InstanceMetadata{}
// fetch version.
versionData, _, err := d.execStatementSync(ctx, "SELECT VERSION()", 0)
if err != nil {
return nil, err
}
if len(versionData) != 1 || len(versionData[0]) != 1 {
return nil, errors.New("invalid version format")
}
splitVersion := strings.Split(versionData[0][0], " ")
if len(splitVersion) != 2 {
return nil, errors.New("invalid version format")
}
instanceMetadata.Version = splitVersion[0]
// fetch table data from databricks.
catalogMap, err := d.listCatologTables(ctx, "")
if err != nil {
return nil, err
}
for catalogName, schemaMap := range catalogMap {
dbMetadataMeta := metadatapb.DatabaseSchemaMetadata{}
schemas := convertToStorepbSchemas(schemaMap)
dbMetadataMeta.Name = catalogName
dbMetadataMeta.Schemas = schemas
instanceMetadata.Databases = append(instanceMetadata.Databases, &dbMetadataMeta)
}
// fetch workspace users.
// TODO(tommy): complete this part when Permissions API for Golang is implemented.
return instanceMetadata, nil
}
// list all tables in the workspace when catalogName is set to "".
func (d *Driver) listCatologTables(ctx context.Context, targetCatalogName string) (databricksCatalogMap, error) {
catalogMap := make(databricksCatalogMap)
catalogsInfo, err := d.Client.Catalogs.ListAll(ctx, catalog.ListCatalogsRequest{})
if err != nil {
return nil, err
}
for _, catalogInfo := range catalogsInfo {
if targetCatalogName != "" && targetCatalogName != catalogInfo.Name {
continue
}
schemasInfo, err := d.Client.Schemas.ListAll(ctx, catalog.ListSchemasRequest{
CatalogName: catalogInfo.Name,
})
if err != nil {
return nil, err
}
for _, schemaInfo := range schemasInfo {
// init map.
scMap, ok := catalogMap[catalogInfo.Name]
if !ok {
catalogMap[catalogInfo.Name] = databricksSchemaMap{
schemaInfo.Name: []*tableUnion{},
}
} else {
_, ok := scMap[schemaInfo.Name]
if !ok {
scMap[schemaInfo.Name] = []*tableUnion{}
}
catalogMap[catalogInfo.Name] = scMap
}
// list tables in the schema.
tablesInfo, err := d.Client.Tables.ListAll(ctx, catalog.ListTablesRequest{
CatalogName: catalogInfo.Name,
SchemaName: schemaInfo.Name,
})
if err != nil {
return nil, err
}
if err := appendSchemaTables(catalogMap, tablesInfo); err != nil {
return nil, err
}
}
}
return catalogMap, nil
}
// extract tables' metadata from 'tablesInfo' and store it in 'catalogMap'.
func appendSchemaTables(catalogMap databricksCatalogMap, tablesInfo []catalog.TableInfo) error {
for _, tableInfo := range tablesInfo {
table := tableUnion{
typeName: tableInfo.TableType,
name: tableInfo.Name,
}
switch tableInfo.TableType {
case catalog.TableTypeView:
table.view = &metadatapb.ViewMetadata{
Name: table.name,
Definition: tableInfo.ViewDefinition,
Comment: tableInfo.Comment,
DependencyColumns: convertToDependencyColumns(tableInfo.SchemaName, tableInfo.Name, tableInfo.Columns),
}
case catalog.TableTypeMaterializedView:
table.materialView = &metadatapb.MaterializedViewMetadata{
Name: table.name,
Definition: tableInfo.ViewDefinition,
Comment: tableInfo.Comment,
}
case catalog.TableTypeExternal:
table.externalTable = &metadatapb.ExternalTableMetadata{
Name: table.name,
}
case catalog.TableTypeManaged:
table.table = &metadatapb.TableMetadata{
Name: table.name,
Columns: convertToColumnMetadata(tableInfo.Columns),
Comment: tableInfo.Comment,
}
default:
// we do not sync streaming table.
continue
}
// catalogMap[tableInfo.CatalogName] must not be nil.
if tblList, ok := catalogMap[tableInfo.CatalogName][tableInfo.SchemaName]; ok {
tblList = append(tblList, &table)
catalogMap[tableInfo.CatalogName][tableInfo.SchemaName] = tblList
} else {
return errors.New("table list not initialized")
}
}
return nil
}
func convertToColumnMetadata(columnInfo []catalog.ColumnInfo) []*metadatapb.ColumnMetadata {
columns := []*metadatapb.ColumnMetadata{}
for _, col := range columnInfo {
columns = append(columns, &metadatapb.ColumnMetadata{
Name: col.Name,
Position: int32(col.Position),
Nullable: col.Nullable,
Type: string(col.TypeName),
Comment: col.Comment,
})
}
return columns
}
func convertToDependencyColumns(schema, table string, columnInfo []catalog.ColumnInfo) []*metadatapb.DependencyColumn {
columns := []*metadatapb.DependencyColumn{}
for _, col := range columnInfo {
columns = append(columns, &metadatapb.DependencyColumn{
Schema: schema,
Table: table,
Column: col.Name,
})
}
return columns
}
func convertToStorepbSchemas(schemaMap databricksSchemaMap) []*metadatapb.SchemaMetadata {
schemas := []*metadatapb.SchemaMetadata{}
for schemaName, tableList := range schemaMap {
schemaMetadata := &metadatapb.SchemaMetadata{
Name: schemaName,
}
for _, table := range tableList {
switch table.typeName {
case catalog.TableTypeView:
schemaMetadata.Views = append(schemaMetadata.Views, table.view)
case catalog.TableTypeMaterializedView:
schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, table.materialView)
case catalog.TableTypeExternal:
schemaMetadata.ExternalTables = append(schemaMetadata.ExternalTables, table.externalTable)
case catalog.TableTypeManaged:
schemaMetadata.Tables = append(schemaMetadata.Tables, table.table)
default:
// we do not sync streaming table.
}
}
schemas = append(schemas, schemaMetadata)
}
return schemas
}
func getQualifiedTblName(catalogName, schemaName, tblName string) (string, error) {
if tblName == "" || schemaName == "" {
return "", errors.New("table name and schema name must be specified")
}
qualifiedName := fmt.Sprintf("`%s`.`%s`", schemaName, tblName)
if catalogName != "" {
qualifiedName = fmt.Sprintf("`%s`.%s", catalogName, qualifiedName)
}
return qualifiedName, nil
}