Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
perf(sql-editor): export up to 4 databases at a time in batch export (#21882)
p0ny507eba945243
diff --git a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.test.ts b/frontend/src/lib/concurrency.test.tssimilarity index 100%rename from frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.test.tsrename to frontend/src/lib/concurrency.test.tsdiff --git a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.ts b/frontend/src/lib/concurrency.tssimilarity index 93%rename from frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.tsrename to frontend/src/lib/concurrency.tsindex f259f3605e..e2b4b6c1bf 100644--- a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.ts+++ b/frontend/src/lib/concurrency.ts@@ -1,6 +1,6 @@// Runs `task` over `items` with at most `limit` in flight, preserving order in-// the result. Rejections are collected as `undefined` so one failed sheet-// fetch does not abort the others.+// the result. Rejections are collected as `undefined` so one failed task does+// not abort the others.export async function runWithConcurrency<T, R>(items: readonly T[],limit: number,diff --git a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsxindex 2e7e76b8f3..1326113b51 100644--- a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx+++ b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx@@ -26,6 +26,8 @@ const mocks = vi.hoisted(() => ({useExportGrantBypass: vi.fn(),destinations: [] as number[],exportData: vi.fn(),+ exportDestination: 2,+ notify: vi.fn(),resolveExport: vi.fn(),reportProgress: vi.fn(),}));@@ -57,7 +59,7 @@ vi.mock("@/components/DataExportButton", () => ({format: 1,limit: 100,password: "",- destination: 2,+ destination: mocks.exportDestination,},resolve: mocks.resolveExport,reject: vi.fn(),@@ -179,7 +181,7 @@ vi.mock("@/stores/app", () => {exportData: mocks.exportData,getDatabaseByName: (name: string) => mocks.databasesByName[name],getEnvironmentByName: () => mocks.environment,- notify: vi.fn(),+ notify: mocks.notify,});return {useAppStore: Object.assign(@@ -217,6 +219,7 @@ describe("BatchQuerySelect", () => {root = createRoot(container);mocks.databasesWithExport = [];mocks.destinations = [ExportDestination.DOWNLOAD];+ mocks.exportDestination = ExportDestination.SHAREPOINT;grantLookup([], []);cellA = {@@ -419,6 +422,177 @@ describe("BatchQuerySelect", () => {expect(mocks.reportProgress.mock.calls).toEqual([[0, 2],[1, 2],+ [2, 2],+ ]);+ });++ const useDatabases = (count: number) => {+ const databases = Array.from(+ { length: count },+ (_, i) =>+ ({+ name: `instances/cell-${i}/databases/db${i}`,+ effectiveEnvironment: mocks.environment.name,+ instanceResource: {+ engine: Engine.POSTGRES,+ name: `instances/cell-${i}`,+ title: `cell-${i}`,+ },+ }) as Database+ );+ mocks.databasesByName = Object.fromEntries(+ databases.map((database) => [database.name, database])+ );+ mocks.databaseQueryContexts = new Map(+ databases.map((database, i) => [+ database.name,+ [+ {+ id: `query-${i}`,+ params: { connection: {}, statement: "SELECT 1" },+ status: "PENDING",+ },+ ],+ ])+ );+ mocks.databasesWithExport = databases.map((database) => database.name);+ return databases.map((database) => database.name);+ };++ // Each export stays pending until the test settles it, so the test decides+ // the order in which exports finish.+ const deferExports = () => {+ const pending = new Map<+ string,+ { resolve: (uri: string) => void; reject: (error: Error) => void }+ >();+ let inFlight = 0;+ let peak = 0;+ mocks.exportData.mockImplementation(+ (request: { name: string }) =>+ new Promise((resolve, reject) => {+ inFlight++;+ peak = Math.max(peak, inFlight);+ pending.set(request.name, {+ resolve: (uri) => {+ inFlight--;+ resolve({ content: new TextEncoder().encode(uri), uri });+ },+ reject: (error) => {+ inFlight--;+ reject(error);+ },+ });+ })+ );+ const started = () =>+ mocks.exportData.mock.calls.map(([request]) => request.name);+ return { pending, started, peak: () => peak };+ };++ const flush = () =>+ act(() => new Promise<void>((resolve) => setTimeout(resolve, 0)));++ const exportAll = async () => {+ render();+ act(() => {+ testId("select-all")?.dispatchEvent(+ new MouseEvent("click", { bubbles: true })+ );+ });+ act(() => {+ testId("batch-export-submit")?.dispatchEvent(+ new MouseEvent("click", { bubbles: true })+ );+ });+ await flush();+ };++ it("exports at most four databases at a time and reports SharePoint results in selection order", async () => {+ mocks.destinations = [ExportDestination.SHAREPOINT];+ const names = useDatabases(6);+ const exports = deferExports();+ await exportAll();++ expect(exports.started()).toEqual(names.slice(0, 4));++ const settle = async (index: number, failure?: string) => {+ const entry = exports.pending.get(names[index]);+ if (failure) {+ entry?.reject(new ConnectError(failure, Code.FailedPrecondition));+ }+ else entry?.resolve(`https://example.com/exports/db${index}`);+ await flush();+ };+ await settle(3);+ expect(exports.started()).toEqual(names.slice(0, 5));+ await settle(0, "quota exceeded");+ expect(exports.started()).toEqual(names);+ await settle(5);+ await settle(2);+ await settle(4);+ expect(mocks.resolveExport).not.toHaveBeenCalled();+ await settle(1);++ expect(exports.peak()).toBe(4);+ expect(mocks.reportProgress.mock.calls).toEqual(+ [0, 1, 2, 3, 4, 5, 6].map((done) => [done, 6])+ );+ expect(mocks.resolveExport).toHaveBeenCalledTimes(1);+ const { sharepoint } = mocks.resolveExport.mock.calls[0][0];+ expect(sharepoint).toEqual([+ expect.objectContaining({+ database: "cell-0/db0",+ status: "FAILED",+ reason: "quota exceeded",+ }),+ ...[1, 2, 3, 4, 5].map((i) =>+ expect.objectContaining({+ database: `cell-${i}/db${i}`,+ status: "EXPORTED",+ uri: `https://example.com/exports/db${i}`,+ })+ ),+ ]);+ });++ it("downloads the successful exports in selection order when they finish out of order", async () => {+ mocks.exportDestination = ExportDestination.DOWNLOAD;+ const names = useDatabases(3);+ const exports = deferExports();+ await exportAll();++ expect(exports.started()).toEqual(names);+ exports.pending.get(names[2])?.resolve("db2-content");+ await flush();+ exports.pending.get(names[1])?.reject(new Error("connection refused"));+ await flush();+ exports.pending.get(names[0])?.resolve("db0-content");+ await flush();++ expect(mocks.reportProgress.mock.calls).toEqual(+ [0, 1, 2, 3].map((done) => [done, 3])+ );+ expect(mocks.notify).toHaveBeenCalledTimes(1);+ expect(mocks.notify).toHaveBeenCalledWith(+ expect.objectContaining({+ style: "CRITICAL",+ description: "Error: connection refused",+ })+ );+ expect(mocks.resolveExport).toHaveBeenCalledTimes(1);+ const files = mocks.resolveExport.mock.calls[0][0] as {+ content: Uint8Array;+ filename: string;+ }[];+ expect(+ files.map((file) => [+ new TextDecoder().decode(file.content),+ file.filename.split(".")[0],+ ])+ ).toEqual([+ ["db0-content", "db0"],+ ["db2-content", "db2"],]);});});diff --git a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsxindex 791118b82e..f28e2725eb 100644--- a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx+++ b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx@@ -18,6 +18,7 @@ import {} from "@/components/SharePointExportResultDialog";import { Button } from "@/components/ui/button";import { Tooltip } from "@/components/ui/tooltip";+import { runWithConcurrency } from "@/lib/concurrency";import { cn } from "@/lib/utils";import { RequestExportButton } from "@/modules/sql-editor/components/RequestExportButton";import { sharePointExportDatabaseLabel } from "@/modules/sql-editor/components/sharePointExportLabel";@@ -49,6 +50,10 @@ import { TabContextMenu } from "./ContextMenu";import { type CloseTabAction, resultTabEvents } from "./resultTabContext";const MAX_EXPORT = 20;+// Browsers allow about six connections per origin over HTTP/1.1, so four+// leaves room for the SQL Editor's other requests. Each running export also+// holds its whole result in server memory.+const EXPORT_CONCURRENCY = 4;type Props = {readonly selectedDatabase: Database | undefined;@@ -60,6 +65,10 @@ type BatchQueryItem = {context: SQLEditorDatabaseQueryContext | undefined;};+type DatabaseExportResult =+ | { download: DownloadContent }+ | { sharepoint: SharePointExportResult };+const isEmptyQueryItem = (item: BatchQueryItem) => {if (!item.context) return true;if (item.context.resultSet?.error) return false;@@ -300,8 +309,6 @@ export function BatchQuerySelect({void (async () => {// === Prod path: per-database backend Export RPC ===const toSharePoint = options.destination === ExportDestination.SHAREPOINT;- const contents: DownloadContent[] = [];- const sharePointResults: SharePointExportResult[] = [];const tabsState = getSQLEditorTabsState();const tab = tabsState.tabsById.get(tabsState.currentTabId);// Defensive intersect — the drawer already filters to authorized@@ -311,15 +318,16 @@ export function BatchQuerySelect({const targets = Array.from(selectedDatabaseNames).filter((name) =>exportableSet.has(name));- for (const [index, databaseName] of targets.entries()) {- onProgress?.(index, targets.length);+ const exportDatabase = async (+ databaseName: string+ ): Promise<DatabaseExportResult | undefined> => {const database = getDatabaseByName(databaseName);const { databaseName: shortName } = extractDatabaseResourceName(database.name);const label = sharePointExportDatabaseLabel(database);const context = head(tab?.databaseQueryContexts?.get(databaseName));- if (!context) continue;+ if (!context) return undefined;try {const { content, uri } = await exportData(create(ExportRequestSchema, {@@ -338,23 +346,23 @@ export function BatchQuerySelect({}));if (toSharePoint) {- sharePointResults.push(- toExportedSharePointResult(+ return {+ sharepoint: toExportedSharePointResult(label,uri,exportDestination.sharepointUrl ?? ""- )- );- continue;+ ),+ };}- contents.push({- content,- filename: `${shortName}.${dayjs(new Date()).format("YYYY-MM-DDTHH-mm-ss")}.zip`,- });+ return {+ download: {+ content,+ filename: `${shortName}.${dayjs(new Date()).format("YYYY-MM-DDTHH-mm-ss")}.zip`,+ },+ };} catch (e) {if (toSharePoint) {- sharePointResults.push(toFailedSharePointExport(label, e));- continue;+ return { sharepoint: toFailedSharePointExport(label, e) };}useAppStore.getState().notify({module: "bytebase",@@ -364,9 +372,35 @@ export function BatchQuerySelect({}),description: String(e),});+ return undefined;}- }- resolve(toSharePoint ? { sharepoint: sharePointResults } : contents);+ };++ let done = 0;+ onProgress?.(done, targets.length);+ // Results keep the selection order whatever order the exports finish in.+ const results = await runWithConcurrency(+ targets,+ EXPORT_CONCURRENCY,+ async (databaseName) => {+ try {+ return await exportDatabase(databaseName);+ } finally {+ onProgress?.(++done, targets.length);+ }+ }+ );+ resolve(+ toSharePoint+ ? {+ sharepoint: results.flatMap((result) =>+ result && "sharepoint" in result ? [result.sharepoint] : []+ ),+ }+ : results.flatMap((result) =>+ result && "download" in result ? [result.download] : []+ )+ );setSelectedDatabaseNames(new Set());})();};diff --git a/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts b/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.tsindex 7a6a5c983a..d35bc29925 100644--- a/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts+++ b/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts@@ -1,3 +1,4 @@+import { runWithConcurrency } from "@/lib/concurrency";import { useAppStore } from "@/stores/app";import type { Sheet } from "@/types/proto-es/v1/sheet_service_pb";import {@@ -9,7 +10,6 @@ import {PLACEMENT_BUDGETS,type PlacementBudgets,} from "../../components/threads/placement/budgets";-import { runWithConcurrency } from "../../components/threads/placement/concurrency";import {type Placement,UNAVAILABLE,