Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

19.8 KB
Raw Blame
import { markRaw, reactive, shallowReactive } from "vue";
import { uuid } from "@/lib/common/utils";
import * as api from "@/lib/backend/api";
import { openDataCompareSession } from "@/composables/useDialogSources";
import { useExportTracker } from "@/composables/useExportTracker";
import { inferCompareKeyColumns, intersectCompareColumns, matchColumnNameIgnoreCase, normalizeKeyColumns, type DataCompareFromTablesPreparation, type DataCompareResult, type DataCompareSyncPlan, type DataCompareSyncPlanTableOptions } from "@/lib/dataGrid/dataCompare";
import { createDataCompareSelection, selectedCompareRows, type DataCompareSelection } from "@/lib/dataGrid/dataCompareSelection";
import type { ColumnInfo, DatabaseType } from "@/types/database";

export type CompareColumn = ColumnInfo;

/**
 * One source-table -> target-table compare task.
 *
 * Match columns are not part of the task itself: they are stored per source
 * table in `DataCompareSessionConfig.keyColumnsByTable` so that every task of a
 * batch compare can carry its own selection instead of sharing one global list.
 */
export interface DataCompareTableTask {
  sourceTable: string;
  targetTable: string;
}

export type DataCompareTableStatus = "different" | "same" | "error";
export type DiffKind = "added" | "removed" | "modified";

export interface DataCompareTableResult {
  sourceTable: string;
  targetTable: string;
  keyColumns: string[];
  columns: string[];
  columnInfo: CompareColumn[];
  status: DataCompareTableStatus;
  added: number;
  removed: number;
  modified: number;
  sourceRowCount: number;
  targetRowCount: number;
  sourceTruncated: boolean;
  targetTruncated: boolean;
  /** Rows the backend was allowed to inspect per side; incomplete results are never synchronization-eligible. */
  rowBudget: number;
  syncEligible: boolean;
  databaseType?: DatabaseType;
  preSyncStatements?: string[];
  diff: DataCompareResult;
  selection: Record<DiffKind, DataCompareSelection>;
  expanded: boolean;
  showAll: Record<DiffKind, boolean>;
  error?: string;
}

export interface DataCompareSessionConfig {
  sourceConnectionId: string;
  sourceDatabase: string;
  sourceSchema: string;
  sourceDatabases: string[];
  sourceSchemas: string[];
  sourceTables: string[];
  selectedSourceTables: string[];
  targetConnectionId: string;
  targetDatabase: string;
  targetSchema: string;
  targetDatabases: string[];
  targetSchemas: string[];
  targetTables: string[];
  targetTable: string;
  /**
   * Explicit match columns per source table (keyed by the source table name).
   *
   * An absent entry means "no override": the primary key of that source table is
   * inferred automatically. A present entry is an explicit user choice, so an
   * empty array means the user cleared the selection and must pick columns
   * again — it must not silently fall back to the primary key.
   */
  keyColumnsByTable: Record<string, string[]>;
  label: string;
}

export interface DataCompareSessionProgress {
  current: number;
  total: number;
  table: string;
}

export type DataCompareSessionStatus = "running" | "completed" | "failed";
export type DataCompareSessionErrorKind = "noKeyColumns" | "missingKeyColumns" | "noCommonColumns";

export interface DataCompareSession {
  id: string;
  version: number;
  status: DataCompareSessionStatus;
  config: DataCompareSessionConfig;
  progress: DataCompareSessionProgress | null;
  batchResults: DataCompareTableResult[];
  syncPlan: DataCompareSyncPlan;
  error: string | null;
  startedAt: number;
  finishedAt?: number;
}

export interface DataCompareSessionDependencies {
  ensureConnected(connectionId: string): Promise<unknown>;
  getConfig(connectionId: string): { db_type?: DatabaseType } | undefined;
  formatError?: (kind: DataCompareSessionErrorKind, columns?: string) => string;
}

const sessions = reactive(new Map<string, DataCompareSession>());

export function emptyDataCompareSyncPlan(): DataCompareSyncPlan {
  return {
    insertCount: 0,
    updateCount: 0,
    deleteCount: 0,
    statementCount: 0,
    syncStatements: [],
    syncSql: "",
  };
}

function cloneConfig(config: DataCompareSessionConfig): DataCompareSessionConfig {
  return {
    ...config,
    sourceDatabases: [...config.sourceDatabases],
    sourceSchemas: [...config.sourceSchemas],
    sourceTables: [...config.sourceTables],
    selectedSourceTables: [...config.selectedSourceTables],
    targetDatabases: [...config.targetDatabases],
    targetSchemas: [...config.targetSchemas],
    targetTables: [...config.targetTables],
    keyColumnsByTable: normalizeKeyColumnOverrides(config.keyColumnsByTable),
  };
}

/**
 * Normalizes a per-table match-column map so a cloned session never shares
 * arrays with the caller: every entry is trimmed and de-duplicated while an
 * explicit empty selection stays an empty array (it must not become "auto").
 */
export function normalizeKeyColumnOverrides(overrides: Record<string, string[] | undefined> | null | undefined): Record<string, string[]> {
  const normalized: Record<string, string[]> = {};
  for (const [table, columns] of Object.entries(overrides ?? {})) {
    if (!table) continue;
    normalized[table] = normalizeKeyColumns(columns ?? []);
  }
  return normalized;
}

function sessionError(error: unknown): string {
  return error instanceof Error ? error.message : (error as { message?: string } | null)?.message || String(error);
}

function fallbackCompareErrorMessage(kind: DataCompareSessionErrorKind, columns?: string): string {
  if (kind === "missingKeyColumns") return `Missing key columns: ${columns ?? ""}`;
  if (kind === "noCommonColumns") return "No common columns";
  return "No key columns available";
}

function compareErrorMessage(dependencies: DataCompareSessionDependencies, kind: DataCompareSessionErrorKind, columns?: string): string {
  return dependencies.formatError?.(kind, columns) ?? fallbackCompareErrorMessage(kind, columns);
}

function tableResult(result: Omit<DataCompareTableResult, "selection">): DataCompareTableResult {
  return shallowReactive({
    ...result,
    diff: markRaw(result.diff),
    showAll: reactive(result.showAll),
    selection: {
      added: createDataCompareSelection(result.syncEligible),
      removed: createDataCompareSelection(result.syncEligible),
      modified: createDataCompareSelection(result.syncEligible),
    },
  });
}

export function buildDataCompareSyncPlanTables(results: DataCompareTableResult[], targetSchema: string): DataCompareSyncPlanTableOptions[] {
  return results
    .filter((table) => table.status === "different" && table.syncEligible)
    .map((table) => ({
      tableName: table.targetTable,
      schema: targetSchema,
      columns: table.columns,
      keyColumns: table.keyColumns,
      columnInfo: table.columnInfo,
      diff: {
        added: selectedCompareRows(table.diff.added, table.selection.added),
        removed: selectedCompareRows(table.diff.removed, table.selection.removed),
        modified: selectedCompareRows(table.diff.modified, table.selection.modified),
      },
      databaseType: table.databaseType,
      preSyncStatements: table.preSyncStatements ?? [],
      syncEligible: true,
    }))
    .filter((table) => table.preSyncStatements.length > 0 || table.diff.added.length > 0 || table.diff.removed.length > 0 || table.diff.modified.length > 0);
}

function publishProgress(session: DataCompareSession, progress: DataCompareSessionProgress): void {
  session.version += 1;
  session.progress = progress;
  const results = session.batchResults;
  const failedTableCount = results.filter((item) => item.status === "error").length;
  const tracker = useExportTracker();
  tracker.updateCompareTask(session.id, {
    status: "Running",
    compareCurrent: progress.current,
    compareTotal: progress.total,
    compareCurrentObject: progress.table,
    compareResultCount: results.length,
    compareSameCount: results.filter((item) => item.status === "same").length,
    compareDifferentCount: results.filter((item) => item.status === "different").length,
    compareFailedCount: failedTableCount,
    compareAddedCount: results.reduce((sum, item) => sum + item.added, 0),
    compareRemovedCount: results.reduce((sum, item) => sum + item.removed, 0),
    compareModifiedCount: results.reduce((sum, item) => sum + item.modified, 0),
  });
}

function updateResults(session: DataCompareSession, results: DataCompareTableResult[]): void {
  session.batchResults = [...results];
}

async function loadColumnsWithCache(cache: Map<string, CompareColumn[]>, connectionId: string, database: string, schema: string, table: string): Promise<CompareColumn[]> {
  const key = `${connectionId}:${database}:${schema}:${table}`;
  const cached = cache.get(key);
  if (cached) return cached;
  const columns = (await api.getColumns(connectionId, database, schema, table)) as CompareColumn[];
  cache.set(key, columns);
  return columns;
}

async function inferKeyColumnsForTable(input: DataCompareSessionConfig, table: string, sourceColumnCache: Map<string, CompareColumn[]>): Promise<string[]> {
  if (!input.sourceConnectionId || !input.sourceDatabase || !input.sourceSchema || !table) return [];
  const columns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, table);
  return inferCompareKeyColumns(columns);
}

function buildTableResult(task: DataCompareTableTask, preparation: DataCompareFromTablesPreparation, keyColumns: string[], columns: string[], columnInfo: CompareColumn[], databaseType: DatabaseType | undefined): DataCompareTableResult {
  const added = preparation.result.added.length;
  const removed = preparation.result.removed.length;
  const modified = preparation.result.modified.length;
  const status: DataCompareTableStatus = added || removed || modified ? "different" : "same";
  return tableResult({
    sourceTable: task.sourceTable,
    targetTable: task.targetTable,
    keyColumns,
    columns,
    columnInfo,
    status,
    added,
    removed,
    modified,
    sourceRowCount: preparation.sourceRowCount,
    targetRowCount: preparation.targetRowCount,
    sourceTruncated: preparation.sourceTruncated,
    targetTruncated: preparation.targetTruncated,
    rowBudget: preparation.rowBudget,
    syncEligible: preparation.syncEligible,
    databaseType,
    preSyncStatements: preparation.preSyncStatements,
    diff: preparation.result,
    expanded: status === "different",
    showAll: { added: false, removed: false, modified: false },
  });
}

function buildMissingTargetResult(task: DataCompareTableTask, preparation: DataCompareFromTablesPreparation, sourceColumns: CompareColumn[], keyColumns: string[], databaseType: DatabaseType | undefined): DataCompareTableResult {
  return tableResult({
    sourceTable: task.sourceTable,
    targetTable: task.targetTable,
    keyColumns,
    columns: sourceColumns.map((column) => column.name),
    columnInfo: sourceColumns,
    status: "different",
    added: preparation.result.added.length,
    removed: 0,
    modified: 0,
    sourceRowCount: preparation.sourceRowCount,
    targetRowCount: 0,
    sourceTruncated: preparation.sourceTruncated,
    targetTruncated: false,
    rowBudget: preparation.rowBudget,
    syncEligible: preparation.syncEligible,
    databaseType,
    preSyncStatements: preparation.preSyncStatements,
    diff: preparation.result,
    expanded: preparation.result.added.length > 0,
    showAll: { added: false, removed: false, modified: false },
  });
}

function buildErrorResult(task: DataCompareTableTask, keyColumns: string[], databaseType: DatabaseType | undefined, error: unknown): DataCompareTableResult {
  return tableResult({
    sourceTable: task.sourceTable,
    targetTable: task.targetTable,
    keyColumns,
    columns: [],
    columnInfo: [],
    status: "error",
    added: 0,
    removed: 0,
    modified: 0,
    sourceRowCount: 0,
    targetRowCount: 0,
    sourceTruncated: false,
    targetTruncated: false,
    rowBudget: 0,
    syncEligible: false,
    databaseType,
    preSyncStatements: [],
    diff: { added: [], removed: [], modified: [] },
    expanded: false,
    showAll: { added: false, removed: false, modified: false },
    error: sessionError(error),
  });
}

/**
 * Resolves the configured match columns of one compare task.
 *
 * Returns the explicit per-source-table override when the user configured one
 * (an empty array counts as "explicitly nothing selected") and `undefined`
 * otherwise, which means the caller must infer the primary key for that table.
 */
function configuredTaskKeyColumns(input: DataCompareSessionConfig, task: DataCompareTableTask): string[] | undefined {
  const override = input.keyColumnsByTable[task.sourceTable];
  return override === undefined ? undefined : normalizeKeyColumns(override);
}

async function runDataCompareSession(session: DataCompareSession, tasks: DataCompareTableTask[], dependencies: DataCompareSessionDependencies): Promise<void> {
  const input = session.config;
  const sourceColumnCache = new Map<string, CompareColumn[]>();
  const targetColumnCache = new Map<string, CompareColumn[]>();
  const results: DataCompareTableResult[] = [];
  const databaseType = dependencies.getConfig(input.targetConnectionId)?.db_type;

  try {
    publishProgress(session, { current: 0, total: tasks.length, table: "" });
    await Promise.all([dependencies.ensureConnected(input.sourceConnectionId), dependencies.ensureConnected(input.targetConnectionId)]);

    for (const [index, task] of tasks.entries()) {
      publishProgress(session, { current: index + 1, total: tasks.length, table: task.sourceTable });
      const configuredKeys = configuredTaskKeyColumns(input, task);
      let keyColumns = configuredKeys ?? [];
      try {
        if (input.targetTables.includes(task.targetTable)) {
          const resolvedKeys = configuredKeys ?? (await inferKeyColumnsForTable(input, task.sourceTable, sourceColumnCache));
          if (resolvedKeys.length === 0) throw new Error(compareErrorMessage(dependencies, "noKeyColumns"));

          const sourceColumns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, task.sourceTable);
          const targetColumns = await loadColumnsWithCache(targetColumnCache, input.targetConnectionId, input.targetDatabase, input.targetSchema, task.targetTable);
          const matched = intersectCompareColumns(sourceColumns, targetColumns);
          const columns = matched.columns;
          const columnInfo = columns.map((column) => targetColumns.find((target) => target.name === column)).filter((column): column is CompareColumn => !!column);
          const canonicalKeyColumns: string[] = [];
          const missingKeys: string[] = [];
          for (const key of resolvedKeys) {
            const canonical = matchColumnNameIgnoreCase(key, columns);
            if (!canonical) missingKeys.push(key);
            else if (!canonicalKeyColumns.includes(canonical)) canonicalKeyColumns.push(canonical);
          }
          if (missingKeys.length > 0) throw new Error(compareErrorMessage(dependencies, "missingKeyColumns", missingKeys.join(", ")));
          if (columns.length === 0) throw new Error(compareErrorMessage(dependencies, "noCommonColumns"));

          const preparation = await api.prepareDataCompareFromTables({
            sourceConnectionId: input.sourceConnectionId,
            sourceDatabase: input.sourceDatabase,
            sourceSchema: input.sourceSchema,
            sourceTable: task.sourceTable,
            targetConnectionId: input.targetConnectionId,
            targetDatabase: input.targetDatabase,
            targetSchema: input.targetSchema,
            targetTable: task.targetTable,
            columns,
            keyColumns: canonicalKeyColumns,
            sourceColumns: matched.sourceColumns,
          });
          keyColumns = canonicalKeyColumns;
          results.push(buildTableResult(task, preparation, canonicalKeyColumns, columns, columnInfo, databaseType));
        } else {
          const sourceColumns = await loadColumnsWithCache(sourceColumnCache, input.sourceConnectionId, input.sourceDatabase, input.sourceSchema, task.sourceTable);
          const resolvedKeys = configuredTaskKeyColumns(input, task) ?? [];
          const sourceColumnNames = sourceColumns.map((column) => column.name);
          keyColumns = resolvedKeys.map((key) => matchColumnNameIgnoreCase(key, sourceColumnNames) ?? key);
          const preparation = await api.prepareDataCompareMissingTarget({
            sourceConnectionId: input.sourceConnectionId,
            sourceDatabase: input.sourceDatabase,
            sourceSchema: input.sourceSchema,
            sourceTable: task.sourceTable,
            targetConnectionId: input.targetConnectionId,
            targetDatabase: input.targetDatabase,
            targetSchema: input.targetSchema,
            targetTable: task.targetTable,
            keyColumns,
          });
          results.push(buildMissingTargetResult(task, preparation, sourceColumns, keyColumns, databaseType));
        }
      } catch (error: unknown) {
        results.push(buildErrorResult(task, keyColumns, databaseType, error));
      }
      updateResults(session, results);
      publishProgress(session, { current: index + 1, total: tasks.length, table: task.sourceTable });
    }

    const plan = await api.buildDataCompareSyncPlan({ tables: buildDataCompareSyncPlanTables(results, input.targetSchema) });
    session.syncPlan = plan;
    session.progress = null;
    session.status = "completed";
    session.version += 1;
    session.finishedAt = Date.now();
    const sameCount = results.filter((item) => item.status === "same").length;
    const differentCount = results.filter((item) => item.status === "different").length;
    const failedCount = results.filter((item) => item.status === "error").length;
    useExportTracker().updateCompareTask(session.id, {
      status: failedCount > 0 ? "Error" : "Done",
      compareCurrent: tasks.length,
      compareTotal: tasks.length,
      compareCurrentObject: "",
      compareResultCount: results.length,
      compareSameCount: sameCount,
      compareDifferentCount: differentCount,
      compareFailedCount: failedCount,
      compareAddedCount: results.reduce((sum, item) => sum + item.added, 0),
      compareRemovedCount: results.reduce((sum, item) => sum + item.removed, 0),
      compareModifiedCount: results.reduce((sum, item) => sum + item.modified, 0),
      errorMessage: failedCount > 0 ? `${failedCount} table(s) failed` : null,
    });
  } catch (error: unknown) {
    const message = sessionError(error);
    const lastProgress = session.progress;
    session.error = message;
    session.progress = null;
    session.status = "failed";
    session.version += 1;
    session.finishedAt = Date.now();
    useExportTracker().updateCompareTask(session.id, {
      status: "Error",
      errorMessage: message,
      compareCurrent: lastProgress?.current,
      compareTotal: lastProgress?.total,
      compareCurrentObject: lastProgress?.table,
    });
  }
}

export function startDataCompareSession(input: DataCompareSessionConfig, tasks: DataCompareTableTask[], dependencies: DataCompareSessionDependencies): DataCompareSession {
  const id = uuid();
  const session = shallowReactive<DataCompareSession>({
    id,
    version: 0,
    status: "running",
    config: cloneConfig(input),
    progress: null,
    batchResults: [],
    syncPlan: emptyDataCompareSyncPlan(),
    error: null,
    startedAt: Date.now(),
  });
  sessions.set(id, session);

  const tracker = useExportTracker();
  tracker.addDataCompareTask(
    id,
    input.label,
    () => openDataCompareSession(id),
    () => removeDataCompareSession(id),
  );
  void runDataCompareSession(session, tasks, dependencies);
  return session;
}

export function getDataCompareSession(id: string | null | undefined): DataCompareSession | undefined {
  return id ? sessions.get(id) : undefined;
}

export function removeDataCompareSession(id: string): boolean {
  const session = sessions.get(id);
  if (session?.status === "running") return false;
  return sessions.delete(id);
}