Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

9.5 KB
Raw Blame
package hive

import (
	"context"
	"fmt"
	"log/slog"
	"strconv"
	"strings"

	metadatapb "github.com/bytebase/omni/metadata"
	"github.com/pkg/errors"

	"github.com/bytebase/bytebase/backend/common/log"
	"github.com/bytebase/bytebase/backend/plugin/db"
)

func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {
	var instanceMetadata db.InstanceMetadata

	// version.
	version, err := d.getVersion(ctx)
	if err != nil {
		return nil, err
	}

	// databases.
	databaseNames, err := d.getDatabaseNames(ctx)
	if err != nil {
		return nil, err
	}

	for _, v := range databaseNames {
		instanceMetadata.Databases = append(instanceMetadata.Databases, &metadatapb.DatabaseSchemaMetadata{
			Name: v,
		})
	}

	instanceMetadata.Version = version
	return &instanceMetadata, nil
}

func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMetadata, error) {
	dbName := d.config.ConnectionContext.DatabaseName
	if dbName == "" {
		dbName = "default"
	}

	schemaMetadata, err := d.getDatabaseInfoByName(ctx, dbName)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get info from database %s", dbName)
	}
	return &metadatapb.DatabaseSchemaMetadata{
		Name:    dbName,
		Schemas: []*metadatapb.SchemaMetadata{schemaMetadata},
	}, nil
}

func (d *Driver) getVersion(ctx context.Context) (string, error) {
	result, err := d.queryStatementWithLimit(ctx, "SELECT VERSION()", 0)
	if err != nil {
		return "", errors.Wrap(err, "failed to get version from instance")
	}

	if len(result.Rows) == 0 || len(result.Rows[0].Values) == 0 {
		return "", errors.New("invalid version result")
	}
	// rawVersion has the format of "1.2.3 commitID".
	rawVersion := result.Rows[0].Values[0].GetStringValue()
	tokens := strings.Split(rawVersion, " ")
	if len(tokens) == 0 {
		return "", errors.Errorf("invalid version %q", rawVersion)
	}
	return tokens[0], nil
}

func (d *Driver) getDatabaseNames(ctx context.Context) ([]string, error) {
	var databaseNames []string
	result, err := d.queryStatementWithLimit(ctx, "SHOW DATABASES", 0)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get version from instance")
	}
	for _, row := range result.Rows {
		if row == nil || len(row.Values) == 0 {
			return nil, errors.New("row values have zero length")
		}
		databaseNames = append(databaseNames, row.Values[0].GetStringValue())
	}
	return databaseNames, nil
}

func (d *Driver) listTablesNames(ctx context.Context, databaseName string) ([]string, error) {
	result, err := d.queryStatementWithLimit(ctx, fmt.Sprintf("SHOW TABLES FROM %s", databaseName), 0)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get version from instance")
	}

	var tableNames []string
	for _, row := range result.Rows {
		if row == nil || len(row.Values) == 0 {
			return nil, errors.New("row values have zero length")
		}
		tableNames = append(tableNames, row.Values[0].GetStringValue())
	}
	return tableNames, nil
}

// getTables fetches table info and returns structed table data.
func (d *Driver) getTables(ctx context.Context, databaseName string) (
	[]*metadatapb.TableMetadata,
	[]*metadatapb.ExternalTableMetadata,
	[]*metadatapb.ViewMetadata,
	[]*metadatapb.MaterializedViewMetadata,
	error,
) {
	var (
		tableMetadatas    []*metadatapb.TableMetadata
		extTableMetadatas []*metadatapb.ExternalTableMetadata
		viewMetadatas     []*metadatapb.ViewMetadata
		mtViewMetadatas   []*metadatapb.MaterializedViewMetadata
	)

	// list tables' names.
	tableNames, err := d.listTablesNames(ctx, databaseName)
	if err != nil {
		return nil, nil, nil, nil, errors.Wrapf(err, "failed to list tables")
	}

	// iterations in tables of certain database.
	for _, tableName := range tableNames {
		// filter out index table names.
		if strings.HasSuffix(tableName, "__") {
			continue
		}

		tableInfo, err := d.getTableInfo(ctx, tableName, databaseName)
		if err != nil {
			return nil, nil, nil, nil, errors.Wrapf(err, "failed to describe table %s's type", tableName)
		}

		// different processing way according to the type of the table.
		switch tableInfo.tableType {
		case "MATERIALIZED_VIEW":
			mtViewMetadatas = append(mtViewMetadatas, &metadatapb.MaterializedViewMetadata{
				Name:       tableName,
				Definition: tableInfo.viewDef,
				Comment:    tableInfo.comment,
			})
		case "VIRTUAL_VIEW":
			viewMetadatas = append(viewMetadatas, &metadatapb.ViewMetadata{
				Name:       tableName,
				Definition: tableInfo.viewDef,
				Comment:    tableInfo.comment,
			})
		case "EXTERNAL_TABLE":
			extTableMetadatas = append(extTableMetadatas, &metadatapb.ExternalTableMetadata{
				Name:    tableName,
				Columns: tableInfo.colMetadatas,
			})
		case "MANAGED_TABLE":
			partitions, err := d.getPartitions(ctx, databaseName, tableName)
			if err != nil {
				// Ignore partitions error as some tables aren't partitioned.
				slog.Debug("failed to get partitions", log.BBError(err))
				continue
			}
			tableMetadatas = append(tableMetadatas, &metadatapb.TableMetadata{
				Engine:     "HDFS",
				Comment:    tableInfo.comment,
				Columns:    tableInfo.colMetadatas,
				DataSize:   int64(tableInfo.totalSize),
				RowCount:   int64(tableInfo.numRows),
				Partitions: partitions,
				Name:       tableName,
			})
		default:
			// ignore other types of tables.
		}
	}

	return tableMetadatas, extTableMetadatas, viewMetadatas, mtViewMetadatas, nil
}

func (d *Driver) getPartitions(ctx context.Context, databaseName, tableName string) ([]*metadatapb.TablePartitionMetadata, error) {
	// partitions.
	partitionResult, err := d.queryStatementWithLimit(ctx, fmt.Sprintf("SHOW PARTITIONS `%s`.`%s`", databaseName, tableName), 0)
	if err != nil {
		slog.Debug("failed to get partitions", log.BBError(err))
		return nil, nil
	}
	if partitionResult == nil {
		return nil, nil
	}
	var partitions []*metadatapb.TablePartitionMetadata
	for _, row := range partitionResult.Rows {
		if row == nil || len(row.Values) == 0 {
			return nil, errors.New("partitions result row has zero length")
		}
		partitions = append(partitions, &metadatapb.TablePartitionMetadata{
			Name: row.Values[0].GetStringValue(),
		})
	}
	return partitions, nil
}

// This function gets certain database info by name.
func (d *Driver) getDatabaseInfoByName(ctx context.Context, databaseName string) (*metadatapb.SchemaMetadata, error) {
	// fetch table metadata.
	tableMetadata, extTabMetadata, viewMetadata, mtViewMetadata, err := d.getTables(ctx, databaseName)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to get table metadata from database %s", databaseName)
	}

	return &metadatapb.SchemaMetadata{
		Name:              databaseName,
		Tables:            tableMetadata,
		ExternalTables:    extTabMetadata,
		Views:             viewMetadata,
		MaterializedViews: mtViewMetadata,
	}, nil
}

type TableInfo struct {
	tableType    string
	colMetadatas []*metadatapb.ColumnMetadata
	numRows      int
	viewDef      string
	totalSize    int
	comment      string
}

const (
	columnSection = iota
	tableInfoSection
	storageSection
	viewSection
)

func (d *Driver) getTableInfo(ctx context.Context, tableName string, databaseName string) (
	*TableInfo,
	error,
) {
	tableInfo := &TableInfo{}

	query := fmt.Sprintf("DESCRIBE FORMATTED `%s`.`%s`", databaseName, tableName)
	rows, err := d.db.QueryContext(ctx, query)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to describe table %s", tableName)
	}
	defer rows.Close()

	section := columnSection
	for rows.Next() {
		var colName, dataType, comment string
		if err := rows.Scan(&colName, &dataType, &comment); err != nil {
			return nil, errors.Wrap(err, "failed to scan row")
		}

		// The first rows are column metadata, followed by "# Detailed Table Information" and "# Storage Information"
		switch {
		case strings.HasPrefix(colName, "# col_name"):
			continue
		case strings.HasPrefix(colName, "# Detailed Table Information"):
			section = tableInfoSection
		case strings.HasPrefix(colName, "# Storage Information"):
			section = storageSection
		case strings.HasPrefix(colName, "# View Information"):
			section = viewSection
		default:
			// No action needed for other cases
		}
		switch section {
		case columnSection:
			if colName != "" && dataType != "" {
				// Column metadata.
				position := len(tableInfo.colMetadatas) + 1
				tableInfo.colMetadatas = append(tableInfo.colMetadatas, &metadatapb.ColumnMetadata{
					Name:     colName,
					Type:     dataType,
					Comment:  comment,
					Position: int32(position),
				})
			}
		case tableInfoSection:
			switch {
			case trimField(colName) == "Table Type:":
				tableInfo.tableType = trimField(dataType)
			case trimField(dataType) == "numRows":
				n, err := strconv.Atoi(trimField(comment))
				if err != nil {
					return nil, errors.Wrapf(err, "failed to parse row count")
				}
				tableInfo.numRows = n
			case trimField(dataType) == "totalSize":
				n, err := strconv.Atoi(trimField(comment))
				if err != nil {
					return nil, errors.Wrapf(err, "failed to parse row count")
				}
				tableInfo.totalSize = n
			case trimField(dataType) == "comment":
				tableInfo.comment = trimField(comment)
			default:
				// No action needed for other table info fields
			}
		case viewSection:
			switch {
			case strings.HasPrefix(colName, "View Original Text:"):
				tableInfo.viewDef = trimField(dataType)
			case strings.HasPrefix(colName, "Original Query:"):
				tableInfo.viewDef = trimField(dataType)
			default:
				// No action needed for other view fields
			}
		default:
			// No action needed for other sections
		}
	}

	if err := rows.Err(); err != nil {
		return nil, errors.Wrap(err, "error iterating rows")
	}

	return tableInfo, nil
}

func trimField(field string) string {
	return strings.Trim(field, " ")
}