Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

perf(sql-editor): export up to 4 databases at a time in batch export (#21882)

p0ny507eba945243
diff --git a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.test.ts b/frontend/src/lib/concurrency.test.ts
similarity index 100%
rename from frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.test.ts
rename to frontend/src/lib/concurrency.test.ts
diff --git a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.ts b/frontend/src/lib/concurrency.ts
similarity index 93%
rename from frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.ts
rename to frontend/src/lib/concurrency.ts
index f259f3605e..e2b4b6c1bf 100644
--- a/frontend/src/routes/project/plan-detail/components/threads/placement/concurrency.ts
+++ b/frontend/src/lib/concurrency.ts
@@ -1,6 +1,6 @@
// Runs `task` over `items` with at most `limit` in flight, preserving order in
-// the result. Rejections are collected as `undefined` so one failed sheet
-// fetch does not abort the others.
+// the result. Rejections are collected as `undefined` so one failed task does
+// not abort the others.
export async function runWithConcurrency<T, R>(
items: readonly T[],
limit: number,
diff --git a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx
index 2e7e76b8f3..1326113b51 100644
--- a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx
+++ b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.test.tsx
@@ -26,6 +26,8 @@ const mocks = vi.hoisted(() => ({
useExportGrantBypass: vi.fn(),
destinations: [] as number[],
exportData: vi.fn(),
+ exportDestination: 2,
+ notify: vi.fn(),
resolveExport: vi.fn(),
reportProgress: vi.fn(),
}));
@@ -57,7 +59,7 @@ vi.mock("@/components/DataExportButton", () => ({
format: 1,
limit: 100,
password: "",
- destination: 2,
+ destination: mocks.exportDestination,
},
resolve: mocks.resolveExport,
reject: vi.fn(),
@@ -179,7 +181,7 @@ vi.mock("@/stores/app", () => {
exportData: mocks.exportData,
getDatabaseByName: (name: string) => mocks.databasesByName[name],
getEnvironmentByName: () => mocks.environment,
- notify: vi.fn(),
+ notify: mocks.notify,
});
return {
useAppStore: Object.assign(
@@ -217,6 +219,7 @@ describe("BatchQuerySelect", () => {
root = createRoot(container);
mocks.databasesWithExport = [];
mocks.destinations = [ExportDestination.DOWNLOAD];
+ mocks.exportDestination = ExportDestination.SHAREPOINT;
grantLookup([], []);
cellA = {
@@ -419,6 +422,177 @@ describe("BatchQuerySelect", () => {
expect(mocks.reportProgress.mock.calls).toEqual([
[0, 2],
[1, 2],
+ [2, 2],
+ ]);
+ });
+
+ const useDatabases = (count: number) => {
+ const databases = Array.from(
+ { length: count },
+ (_, i) =>
+ ({
+ name: `instances/cell-${i}/databases/db${i}`,
+ effectiveEnvironment: mocks.environment.name,
+ instanceResource: {
+ engine: Engine.POSTGRES,
+ name: `instances/cell-${i}`,
+ title: `cell-${i}`,
+ },
+ }) as Database
+ );
+ mocks.databasesByName = Object.fromEntries(
+ databases.map((database) => [database.name, database])
+ );
+ mocks.databaseQueryContexts = new Map(
+ databases.map((database, i) => [
+ database.name,
+ [
+ {
+ id: `query-${i}`,
+ params: { connection: {}, statement: "SELECT 1" },
+ status: "PENDING",
+ },
+ ],
+ ])
+ );
+ mocks.databasesWithExport = databases.map((database) => database.name);
+ return databases.map((database) => database.name);
+ };
+
+ // Each export stays pending until the test settles it, so the test decides
+ // the order in which exports finish.
+ const deferExports = () => {
+ const pending = new Map<
+ string,
+ { resolve: (uri: string) => void; reject: (error: Error) => void }
+ >();
+ let inFlight = 0;
+ let peak = 0;
+ mocks.exportData.mockImplementation(
+ (request: { name: string }) =>
+ new Promise((resolve, reject) => {
+ inFlight++;
+ peak = Math.max(peak, inFlight);
+ pending.set(request.name, {
+ resolve: (uri) => {
+ inFlight--;
+ resolve({ content: new TextEncoder().encode(uri), uri });
+ },
+ reject: (error) => {
+ inFlight--;
+ reject(error);
+ },
+ });
+ })
+ );
+ const started = () =>
+ mocks.exportData.mock.calls.map(([request]) => request.name);
+ return { pending, started, peak: () => peak };
+ };
+
+ const flush = () =>
+ act(() => new Promise<void>((resolve) => setTimeout(resolve, 0)));
+
+ const exportAll = async () => {
+ render();
+ act(() => {
+ testId("select-all")?.dispatchEvent(
+ new MouseEvent("click", { bubbles: true })
+ );
+ });
+ act(() => {
+ testId("batch-export-submit")?.dispatchEvent(
+ new MouseEvent("click", { bubbles: true })
+ );
+ });
+ await flush();
+ };
+
+ it("exports at most four databases at a time and reports SharePoint results in selection order", async () => {
+ mocks.destinations = [ExportDestination.SHAREPOINT];
+ const names = useDatabases(6);
+ const exports = deferExports();
+ await exportAll();
+
+ expect(exports.started()).toEqual(names.slice(0, 4));
+
+ const settle = async (index: number, failure?: string) => {
+ const entry = exports.pending.get(names[index]);
+ if (failure) {
+ entry?.reject(new ConnectError(failure, Code.FailedPrecondition));
+ }
+ else entry?.resolve(`https://example.com/exports/db${index}`);
+ await flush();
+ };
+ await settle(3);
+ expect(exports.started()).toEqual(names.slice(0, 5));
+ await settle(0, "quota exceeded");
+ expect(exports.started()).toEqual(names);
+ await settle(5);
+ await settle(2);
+ await settle(4);
+ expect(mocks.resolveExport).not.toHaveBeenCalled();
+ await settle(1);
+
+ expect(exports.peak()).toBe(4);
+ expect(mocks.reportProgress.mock.calls).toEqual(
+ [0, 1, 2, 3, 4, 5, 6].map((done) => [done, 6])
+ );
+ expect(mocks.resolveExport).toHaveBeenCalledTimes(1);
+ const { sharepoint } = mocks.resolveExport.mock.calls[0][0];
+ expect(sharepoint).toEqual([
+ expect.objectContaining({
+ database: "cell-0/db0",
+ status: "FAILED",
+ reason: "quota exceeded",
+ }),
+ ...[1, 2, 3, 4, 5].map((i) =>
+ expect.objectContaining({
+ database: `cell-${i}/db${i}`,
+ status: "EXPORTED",
+ uri: `https://example.com/exports/db${i}`,
+ })
+ ),
+ ]);
+ });
+
+ it("downloads the successful exports in selection order when they finish out of order", async () => {
+ mocks.exportDestination = ExportDestination.DOWNLOAD;
+ const names = useDatabases(3);
+ const exports = deferExports();
+ await exportAll();
+
+ expect(exports.started()).toEqual(names);
+ exports.pending.get(names[2])?.resolve("db2-content");
+ await flush();
+ exports.pending.get(names[1])?.reject(new Error("connection refused"));
+ await flush();
+ exports.pending.get(names[0])?.resolve("db0-content");
+ await flush();
+
+ expect(mocks.reportProgress.mock.calls).toEqual(
+ [0, 1, 2, 3].map((done) => [done, 3])
+ );
+ expect(mocks.notify).toHaveBeenCalledTimes(1);
+ expect(mocks.notify).toHaveBeenCalledWith(
+ expect.objectContaining({
+ style: "CRITICAL",
+ description: "Error: connection refused",
+ })
+ );
+ expect(mocks.resolveExport).toHaveBeenCalledTimes(1);
+ const files = mocks.resolveExport.mock.calls[0][0] as {
+ content: Uint8Array;
+ filename: string;
+ }[];
+ expect(
+ files.map((file) => [
+ new TextDecoder().decode(file.content),
+ file.filename.split(".")[0],
+ ])
+ ).toEqual([
+ ["db0-content", "db0"],
+ ["db2-content", "db2"],
]);
});
});
diff --git a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx
index 791118b82e..f28e2725eb 100644
--- a/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx
+++ b/frontend/src/modules/sql-editor/components/ResultPanel/BatchQuerySelect.tsx
@@ -18,6 +18,7 @@ import {
} from "@/components/SharePointExportResultDialog";
import { Button } from "@/components/ui/button";
import { Tooltip } from "@/components/ui/tooltip";
+import { runWithConcurrency } from "@/lib/concurrency";
import { cn } from "@/lib/utils";
import { RequestExportButton } from "@/modules/sql-editor/components/RequestExportButton";
import { sharePointExportDatabaseLabel } from "@/modules/sql-editor/components/sharePointExportLabel";
@@ -49,6 +50,10 @@ import { TabContextMenu } from "./ContextMenu";
import { type CloseTabAction, resultTabEvents } from "./resultTabContext";
const MAX_EXPORT = 20;
+// Browsers allow about six connections per origin over HTTP/1.1, so four
+// leaves room for the SQL Editor's other requests. Each running export also
+// holds its whole result in server memory.
+const EXPORT_CONCURRENCY = 4;
type Props = {
readonly selectedDatabase: Database | undefined;
@@ -60,6 +65,10 @@ type BatchQueryItem = {
context: SQLEditorDatabaseQueryContext | undefined;
};
+type DatabaseExportResult =
+ | { download: DownloadContent }
+ | { sharepoint: SharePointExportResult };
+
const isEmptyQueryItem = (item: BatchQueryItem) => {
if (!item.context) return true;
if (item.context.resultSet?.error) return false;
@@ -300,8 +309,6 @@ export function BatchQuerySelect({
void (async () => {
// === Prod path: per-database backend Export RPC ===
const toSharePoint = options.destination === ExportDestination.SHAREPOINT;
- const contents: DownloadContent[] = [];
- const sharePointResults: SharePointExportResult[] = [];
const tabsState = getSQLEditorTabsState();
const tab = tabsState.tabsById.get(tabsState.currentTabId);
// Defensive intersect — the drawer already filters to authorized
@@ -311,15 +318,16 @@ export function BatchQuerySelect({
const targets = Array.from(selectedDatabaseNames).filter((name) =>
exportableSet.has(name)
);
- for (const [index, databaseName] of targets.entries()) {
- onProgress?.(index, targets.length);
+ const exportDatabase = async (
+ databaseName: string
+ ): Promise<DatabaseExportResult | undefined> => {
const database = getDatabaseByName(databaseName);
const { databaseName: shortName } = extractDatabaseResourceName(
database.name
);
const label = sharePointExportDatabaseLabel(database);
const context = head(tab?.databaseQueryContexts?.get(databaseName));
- if (!context) continue;
+ if (!context) return undefined;
try {
const { content, uri } = await exportData(
create(ExportRequestSchema, {
@@ -338,23 +346,23 @@ export function BatchQuerySelect({
})
);
if (toSharePoint) {
- sharePointResults.push(
- toExportedSharePointResult(
+ return {
+ sharepoint: toExportedSharePointResult(
label,
uri,
exportDestination.sharepointUrl ?? ""
- )
- );
- continue;
+ ),
+ };
}
- contents.push({
- content,
- filename: `${shortName}.${dayjs(new Date()).format("YYYY-MM-DDTHH-mm-ss")}.zip`,
- });
+ return {
+ download: {
+ content,
+ filename: `${shortName}.${dayjs(new Date()).format("YYYY-MM-DDTHH-mm-ss")}.zip`,
+ },
+ };
} catch (e) {
if (toSharePoint) {
- sharePointResults.push(toFailedSharePointExport(label, e));
- continue;
+ return { sharepoint: toFailedSharePointExport(label, e) };
}
useAppStore.getState().notify({
module: "bytebase",
@@ -364,9 +372,35 @@ export function BatchQuerySelect({
}),
description: String(e),
});
+ return undefined;
}
- }
- resolve(toSharePoint ? { sharepoint: sharePointResults } : contents);
+ };
+
+ let done = 0;
+ onProgress?.(done, targets.length);
+ // Results keep the selection order whatever order the exports finish in.
+ const results = await runWithConcurrency(
+ targets,
+ EXPORT_CONCURRENCY,
+ async (databaseName) => {
+ try {
+ return await exportDatabase(databaseName);
+ } finally {
+ onProgress?.(++done, targets.length);
+ }
+ }
+ );
+ resolve(
+ toSharePoint
+ ? {
+ sharepoint: results.flatMap((result) =>
+ result && "sharepoint" in result ? [result.sharepoint] : []
+ ),
+ }
+ : results.flatMap((result) =>
+ result && "download" in result ? [result.download] : []
+ )
+ );
setSelectedDatabaseNames(new Set());
})();
};
diff --git a/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts b/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts
index 7a6a5c983a..d35bc29925 100644
--- a/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts
+++ b/frontend/src/routes/project/plan-detail/shared/stores/placementSlice.ts
@@ -1,3 +1,4 @@
+import { runWithConcurrency } from "@/lib/concurrency";
import { useAppStore } from "@/stores/app";
import type { Sheet } from "@/types/proto-es/v1/sheet_service_pb";
import {
@@ -9,7 +10,6 @@ import {
PLACEMENT_BUDGETS,
type PlacementBudgets,
} from "../../components/threads/placement/budgets";
-import { runWithConcurrency } from "../../components/threads/placement/concurrency";
import {
type Placement,
UNAVAILABLE,