Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(generate): stream bulk data generation with progress and configurable timeout

gggaiitxa3772fb44206
diff --git a/apps/desktop/src/components/generate/DataGenerateDialog.vue b/apps/desktop/src/components/generate/DataGenerateDialog.vue
index ca9ee9a92b..3979aa77b7 100644
--- a/apps/desktop/src/components/generate/DataGenerateDialog.vue
+++ b/apps/desktop/src/components/generate/DataGenerateDialog.vue
@@ -3,14 +3,28 @@ import { reactive, ref, computed, onMounted, watch, type ComponentPublicInstance
import { useI18n } from "vue-i18n";
import { useConnectionStore } from "@/stores/connectionStore";
import * as api from "@/lib/backend/api";
-import type { GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
-import { displayGeneratedValue, findGeneratorKey, formatGeneratedValue, generateTableData, defaultGeneratorParams, supportsGeneratedMultiRowValues, UniqueValueGenerationError } from "@/lib/dataGrid/dataGenerate";
+import type { ColumnGenerateConfig, GenerateResult, TableGenerateConfig } from "@/lib/dataGrid/dataGenerate";
+import {
+ createTableGenerateState,
+ defaultGeneratorParams,
+ displayGeneratedValue,
+ findGeneratorKey,
+ formatGeneratedRowValues,
+ formatGeneratedValue,
+ generateInsertBatches,
+ generateTableData,
+ generateTableRowsChunk,
+ splitValueRowsByByteBudget,
+ supportsGeneratedMultiRowValues,
+ UniqueValueGenerationError,
+} from "@/lib/dataGrid/dataGenerate";
+import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "@/lib/dataGrid/generateInsertAccounting";
import { qualifiedTableName, quoteTableIdentifier } from "@/lib/table/tableSelectSql";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { effectiveDatabaseTypeForConnection } from "@/lib/database/jdbcDialect";
import { executeWithProductionSqlGuard } from "@/lib/database/productionExecutionGuard";
import GeneratorParamsPanel from "./params/GeneratorParamsPanel.vue";
-import type { ColumnInfo, TableInfo } from "@/types/database";
+import type { ColumnInfo, QueryResult, TableInfo } from "@/types/database";
import { Dialog, DialogHeader, DialogTitle, DialogScrollContent, DialogContent, DialogFooter } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
@@ -59,10 +73,36 @@ const currentStep = ref<"config" | "preview" | "result">("config");
interface GeneratedTableResult extends GenerateResult {
tableName: string;
schema: string;
+ /** Rows that will actually be inserted. `rows` only holds a preview sample. */
+ targetRowCount: number;
+ isSample: boolean;
+ /** Column config after auto-increment start values have been resolved. */
+ resolvedColumns: ColumnGenerateConfig[];
}
const generatedResults = ref<GeneratedTableResult[]>([]);
const generationError = ref("");
+/**
+ * Preview only materializes a small sample. Generating the full row set up
+ * front froze the UI for hundreds of thousands of rows and exhausted memory
+ * at millions of rows.
+ */
+const PREVIEW_SAMPLE_ROWS = 50;
+const MAX_ROW_COUNT = 100_000_000;
+const DEFAULT_BATCH_ROWS = 1000;
+/**
+ * Conservative per-statement budget. The backend clamps MySQL batches at 4MB
+ * and derives a `max_allowed_packet` margin, so staying near 1MB keeps the
+ * common 4MB / 16MB / 64MB server settings safe without extra round trips.
+ */
+const MAX_BATCH_BYTES = 1024 * 1024;
+const LARGE_ROW_COUNT_HINT = 100_000;
+
+function normalizeRowCount(value: number): number {
+ if (!Number.isFinite(value)) return 0;
+ return Math.max(0, Math.min(MAX_ROW_COUNT, Math.floor(value)));
+}
+
const connectionName = computed(() => (props.prefillConnectionId ? store.getConfig(props.prefillConnectionId)?.name : ""));
function tableKey(schema: string, table: string) {
@@ -366,7 +406,20 @@ function onPreviewColResizeStart(ci: number, event: MouseEvent) {
document.addEventListener("pointerup", onUp);
document.body.classList.add("select-none", "cursor-col-resize");
}
-const currentPreview = computed<GeneratedTableResult>(() => generatedResults.value[previewTableIndex.value] ?? { tableName: "", schema: "", columns: [], rows: [], sql: "", statements: [] });
+const currentPreview = computed<GeneratedTableResult>(
+ () =>
+ generatedResults.value[previewTableIndex.value] ?? {
+ tableName: "",
+ schema: "",
+ columns: [],
+ rows: [],
+ sql: "",
+ statements: [],
+ targetRowCount: 0,
+ isSample: false,
+ resolvedColumns: [],
+ },
+);
function displayPreviewCell(cell: unknown): string {
return displayGeneratedValue(cell);
@@ -395,6 +448,19 @@ async function fetchMaxValues(cfg: TableGenerateConfig): Promise<Record<string,
return starts;
}
+function buildSampleResult(cfg: TableGenerateConfig, columns: ColumnGenerateConfig[], targetRowCount: number): GeneratedTableResult {
+ const sampleCount = Math.min(targetRowCount, PREVIEW_SAMPLE_ROWS);
+ const result = generateTableData({ ...cfg, columns, rowCount: sampleCount }, dbType.value);
+ return {
+ tableName: cfg.tableName,
+ schema: cfg.schema,
+ targetRowCount,
+ isSample: targetRowCount > sampleCount,
+ resolvedColumns: columns,
+ ...result,
+ };
+}
+
async function doGenerate() {
if (!Object.values(checkedTables).some(Boolean)) return;
generationError.value = "";
@@ -416,8 +482,7 @@ async function doGenerate() {
return col;
});
}
- const result = generateTableData({ ...cfg, columns }, dbType.value);
- results.push({ tableName: cfg.tableName, schema: cfg.schema, ...result });
+ results.push(buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount)));
}
} catch (error) {
generationError.value = generationErrorMessage(error);
@@ -447,8 +512,7 @@ async function regenerate() {
}
generationError.value = "";
try {
- const result = generateTableData({ ...cfg, columns }, dbType.value);
- generatedResults.value[previewTableIndex.value] = { tableName: cfg.tableName, schema: cfg.schema, ...result };
+ generatedResults.value[previewTableIndex.value] = buildSampleResult(cfg, columns, normalizeRowCount(cfg.rowCount));
} catch (error) {
generationError.value = generationErrorMessage(error);
}
@@ -461,7 +525,13 @@ function generationErrorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
-function copyAllSql() {
+/**
+ * Copies the script behind the preview. The preview only materializes a
+ * `PREVIEW_SAMPLE_ROWS` sample, so this is the sample script rather than the
+ * full batch — the label says so, and the insert step generates the remaining
+ * rows incrementally instead of holding them in memory.
+ */
+function copySampleSql() {
const allSql = allSqlStatements().join("\n\n");
void navigator.clipboard.writeText(allSql);
}
@@ -474,6 +544,7 @@ interface TableResult {
ok: number;
err: number;
error?: string;
+ cancelled?: boolean;
}
const executeResults = ref<TableResult[]>([]);
@@ -482,6 +553,9 @@ const generateOptions = reactive({
truncate: false,
useTransaction: true,
extendedInsert: true,
+ /** 0 means "no timeout" — the backend maps 0 to an unbounded wait. */
+ timeoutSecs: 0,
+ batchRows: DEFAULT_BATCH_ROWS,
});
const supportsExtendedInsert = computed(() => supportsGeneratedMultiRowValues(dbType.value));
watch(
@@ -494,6 +568,35 @@ watch(
const optionsDialogOpen = ref(false);
+interface InsertProgress {
+ tableName: string;
+ tableIndex: number;
+ tableCount: number;
+ insertedRows: number;
+ totalRows: number;
+ elapsedMs: number;
+}
+const insertProgress = ref<InsertProgress | null>(null);
+const insertCancelled = ref(false);
+let activeExecutionId: string | null = null;
+
+const insertPercent = computed(() => {
+ const p = insertProgress.value;
+ if (!p || p.totalRows <= 0) return 0;
+ return Math.min(100, Math.round((p.insertedRows / p.totalRows) * 100));
+});
+
+function cancelInsert() {
+ if (!executing.value) return;
+ insertCancelled.value = true;
+ if (activeExecutionId) void api.cancelQuery(activeExecutionId);
+}
+
+function normalizeTimeoutSecs(value: number): number {
+ if (!Number.isFinite(value) || value < 0) return 0;
+ return Math.min(86_400, Math.floor(value));
+}
+
function sqlStatementsForTable(r: GeneratedTableResult): string[] {
const stmts: string[] = [];
const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
@@ -516,71 +619,183 @@ function allSqlStatements(): string[] {
return generatedResults.value.flatMap((r) => sqlStatementsForTable(r));
}
+/**
+ * Streams one table: generate a chunk, insert it, drop the references, repeat.
+ *
+ * Peak memory stays proportional to `batchRows` instead of the requested row
+ * count, and each awaited round trip gives the main thread a chance to paint
+ * the progress bar.
+ */
+async function streamInsertTable(cid: string, db: string, r: GeneratedTableResult, executionId: string, onRows: (insertedRows: number) => void): Promise<{ ok: number; attempted: number; error: string; cancelled: boolean }> {
+ const cfg = configs[tableKey(r.schema, r.tableName)];
+ const genCfg: TableGenerateConfig = {
+ tableName: r.tableName,
+ schema: r.schema,
+ database: props.prefillDatabase ?? cfg?.database ?? "",
+ tableType: cfg?.tableType,
+ rowCount: r.targetRowCount,
+ columns: r.resolvedColumns,
+ };
+ const state = createTableGenerateState(genCfg, dbType.value);
+ const schema = r.schema || props.prefillSchema;
+ const forceSingleRow = !generateOptions.extendedInsert;
+ const timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs);
+ const batchRows = Math.max(1, Math.floor(generateOptions.batchRows) || DEFAULT_BATCH_ROWS);
+ let ok = 0;
+ let attempted = 0;
+ let lastError = "";
+ let cancelled = false;
+
+ const run = (sql: string): Promise<QueryResult[]> =>
+ api.executeMultiWithProgress(cid, db, sql, () => {}, schema, {
+ timeoutSecs,
+ useTransaction: generateOptions.useTransaction,
+ continueOnError: generateOptions.continueOnError,
+ executionId,
+ });
+
+ /**
+ * Executes one statement batch and folds the outcome into the running totals.
+ * Returns false when the caller has to stop the table.
+ *
+ * The returned per-statement results are inspected instead of trusting the
+ * `await`: several backend paths report a failed statement inside the result
+ * array while still resolving, so counting rows on the await alone would
+ * report failed batches as inserted.
+ */
+ const executeBatch = async (statements: string[], rowsPerStatement: number[]): Promise<boolean> => {
+ const expectedRows = rowsPerStatement.reduce((sum, rows) => sum + rows, 0);
+ attempted += expectedRows;
+ try {
+ const results = await run(statements.join("\n"));
+ // A row-carrying batch that comes back without per-statement results is
+ // unconfirmed; counting it would over-report inserts.
+ if (expectedRows > 0 && (!results || results.length === 0)) {
+ if (!lastError) lastError = "Backend returned no result for the batch";
+ return generateOptions.continueOnError;
+ }
+ const outcome = summarizeBatchResults(results, rowsPerStatement);
+ ok += outcome.insertedRows;
+ if (!outcome.failed) return true;
+ if (!lastError) lastError = outcome.error ?? "Statement failed";
+ console.error("[startInsert] SQL error:", lastError);
+ return generateOptions.continueOnError;
+ } catch (e: unknown) {
+ // Cancelling a transaction or a single-statement round trip surfaces as
+ // an error, so it has to be told apart from a real failure.
+ if (insertCancelled.value || isQueryCanceledError(e)) {
+ cancelled = true;
+ return false;
+ }
+ const msg = errorMessage(e);
+ console.error("[startInsert] SQL error:", msg);
+ if (!lastError) lastError = msg;
+ return generateOptions.continueOnError;
+ }
+ };
+
+ const finish = () => ({ ok, attempted, error: cancelled ? "" : lastError, cancelled });
+
+ if (generateOptions.truncate) {
+ const targetTable = qualifiedTableName({ databaseType: dbType.value, schema: r.schema, tableName: r.tableName, database: props.prefillDatabase });
+ // TRUNCATE carries no generated rows, so it only gates the rest of the run.
+ const truncateOk = await executeBatch([`TRUNCATE TABLE ${targetTable};`], [0]);
+ if (!truncateOk) return finish();
+ }
+
+ while (state.nextIndex < r.targetRowCount && !insertCancelled.value) {
+ let valueRows: string[];
+ try {
+ const chunkRows = generateTableRowsChunk(genCfg, state, batchRows);
+ if (chunkRows.length === 0) break;
+ valueRows = chunkRows.map((row) => formatGeneratedRowValues(genCfg, dbType.value, state, row));
+ } catch (e: unknown) {
+ // Generation failures (unique space exhausted at row 500k, say) must not
+ // discard the rows that earlier chunks already inserted.
+ if (!lastError) lastError = generationErrorMessage(e);
+ return finish();
+ }
+
+ for (const group of splitValueRowsByByteBudget(state, valueRows, MAX_BATCH_BYTES)) {
+ const { statements, rowsPerStatement } = generateInsertBatches(dbType.value, state, group, forceSingleRow);
+ if (statements.length === 0) continue;
+ const keepGoing = await executeBatch(statements, rowsPerStatement);
+ onRows(ok);
+ if (!keepGoing) return finish();
+ }
+ onRows(ok);
+ // Yield so the progress bar repaints between chunks.
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ }
+
+ if (insertCancelled.value) cancelled = true;
+ return finish();
+}
+
async function startInsert() {
if (executing.value || generationError.value) return;
const cid = props.prefillConnectionId;
const db = props.prefillDatabase;
if (!cid || !db) return;
- const sql = allSqlStatements().join("\n");
- if (!sql.trim()) return;
+ const targets = generatedResults.value.filter((r) => r.targetRowCount > 0 && r.resolvedColumns.length > 0);
+ if (targets.length === 0) return;
+ const guardSql = allSqlStatements().join("\n") || `INSERT INTO ${targets[0].tableName}`;
try {
await executeWithProductionSqlGuard({
connection: store.getConfig(cid),
database: db,
- sql,
+ sql: guardSql,
source: t("production.sourceDataGenerate"),
execute: async () => {
executing.value = true;
+ insertCancelled.value = false;
+ const executionId = crypto.randomUUID();
+ activeExecutionId = executionId;
+ const startedAt = performance.now();
const perTable: TableResult[] = [];
- let stopAfterTable = false;
- for (const r of generatedResults.value) {
- const stmts = sqlStatementsForTable(r);
- const rowCount = r.rows.length;
- let ok = 0;
- let lastError = "";
-
- const executeAsStatementGroup = generateOptions.useTransaction && !supportsGeneratedMultiRowValues(dbType.value);
- if (executeAsStatementGroup) {
+ let stopAll = false;
+ try {
+ for (let ti = 0; ti < targets.length && !stopAll; ti++) {
+ const r = targets[ti];
+ insertProgress.value = {
+ tableName: r.tableName,
+ tableIndex: ti + 1,
+ tableCount: targets.length,
+ insertedRows: 0,
+ totalRows: r.targetRowCount,
+ elapsedMs: 0,
+ };
+ let outcome: { ok: number; attempted: number; error: string; cancelled: boolean };
+ let insertedSoFar = 0;
try {
- await api.executeInTransaction(cid, db, stmts, r.schema || props.prefillSchema);
- ok = rowCount;
+ outcome = await streamInsertTable(cid, db, r, executionId, (insertedRows) => {
+ insertedSoFar = insertedRows;
+ if (insertProgress.value) {
+ insertProgress.value = { ...insertProgress.value, insertedRows, elapsedMs: Math.round(performance.now() - startedAt) };
+ }
+ });
} catch (e: unknown) {
- const msg = e instanceof Error ? e.message : String(e);
- console.error("[startInsert] SQL error:", msg);
- lastError = msg;
- stopAfterTable = !generateOptions.continueOnError;
+ // Safety net: even an unexpected throw must not hide the rows that
+ // earlier batches already inserted.
+ const wasCancelled = insertCancelled.value || isQueryCanceledError(e);
+ outcome = { ok: insertedSoFar, attempted: insertedSoFar, error: wasCancelled ? "" : errorMessage(e), cancelled: wasCancelled };
}
- } else {
- const usesMultiRowStatement = generateOptions.extendedInsert && supportsGeneratedMultiRowValues(dbType.value);
- for (let si = 0; si < stmts.length; si++) {
- try {
- if (generateOptions.useTransaction) {
- await api.executeInTransaction(cid, db, [stmts[si]], r.schema || props.prefillSchema);
- } else {
- await api.executeQuery(cid, db, stmts[si], r.schema || props.prefillSchema);
- }
- if (usesMultiRowStatement) {
- ok = rowCount;
- } else if (!(generateOptions.truncate && si === 0)) {
- ok++;
- }
- } catch (e: unknown) {
- const msg = e instanceof Error ? e.message : String(e);
- console.error("[startInsert] SQL error:", msg);
- if (!lastError) lastError = msg;
- if (usesMultiRowStatement) ok = 0;
- if (!generateOptions.continueOnError) {
- stopAfterTable = true;
- break;
- }
- }
+ const failedRows = Math.max(0, Math.max(outcome.attempted, outcome.ok) - outcome.ok);
+ perTable.push({
+ table: r.tableName,
+ total: r.targetRowCount,
+ ok: outcome.ok,
+ err: failedRows,
+ error: outcome.error || undefined,
+ cancelled: outcome.cancelled || undefined,
+ });
+ if (outcome.ok > 0) {
+ store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
}
+ if ((outcome.error && !generateOptions.continueOnError) || outcome.cancelled) stopAll = true;
}
- perTable.push({ table: r.tableName, total: rowCount, ok, err: rowCount - ok, error: lastError || undefined });
- if (ok > 0) {
- store.invalidateMetadataCache(cid, db, r.schema || props.prefillSchema || undefined, r.tableName);
- }
- if (stopAfterTable) break;
+ } finally {
+ activeExecutionId = null;
}
executeResults.value = perTable;
currentStep.value = "result";
@@ -589,6 +804,7 @@ async function startInsert() {
});
} finally {
executing.value = false;
+ insertProgress.value = null;
}
}
@@ -919,8 +1135,11 @@ async function onFileSelected(event: Event) {
</div>
<div class="flex items-center gap-3 rounded-md bg-muted/20 px-3 py-2">
<Label class="text-xs shrink-0">{{ t("dataGenerate.rowCount") }}:</Label>
- <Input v-model.number="activeCfg.rowCount" class="h-7 w-24 text-xs" />
+ <Input v-model.number="activeCfg.rowCount" type="number" min="0" :max="MAX_ROW_COUNT" class="h-7 w-28 text-xs" @blur="activeCfg.rowCount = normalizeRowCount(activeCfg.rowCount)" />
</div>
+ <p v-if="activeCfg.rowCount > LARGE_ROW_COUNT_HINT" class="px-1 text-[11px] leading-relaxed text-muted-foreground">
+ {{ t("dataGenerate.largeRowCountHint", { count: activeCfg.rowCount.toLocaleString() }) }}
+ </p>
</div>
</template>
</div>
@@ -951,9 +1170,24 @@ async function onFileSelected(event: Event) {
<option v-for="(r, i) in generatedResults" :key="i" :value="i">{{ r.tableName }}</option>
</select>
<span v-else class="text-sm font-medium">{{ generatedResults[0].tableName }}</span>
- <span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.rows.length }) }}</span>
+ <span class="text-xs text-muted-foreground">{{ t("dataGenerate.previewRowCount", { count: currentPreview.targetRowCount }) }}</span>
+ </div>
+ <Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
+ </div>
+ <div v-if="executing && insertProgress" class="space-y-1.5 border-b bg-muted/10 px-3 py-2">
+ <div class="flex items-center justify-between gap-2 text-xs">
+ <span class="truncate text-muted-foreground">
+ {{ t("dataGenerate.insertingTable", { table: insertProgress.tableName, index: insertProgress.tableIndex, count: insertProgress.tableCount }) }}
+ </span>
+ <span class="shrink-0 font-medium tabular-nums">{{ insertProgress.insertedRows.toLocaleString() }} / {{ insertProgress.totalRows.toLocaleString() }}</span>
+ </div>
+ <div class="h-1.5 w-full overflow-hidden rounded-full bg-muted">
+ <div class="h-full bg-primary transition-[width] duration-150" :style="{ width: insertPercent + '%' }" />
+ </div>
+ <div class="flex items-center justify-between text-[11px] text-muted-foreground">
+ <span>{{ insertPercent }}%</span>
+ <span>{{ t("dataGenerate.elapsed", { seconds: Math.round(insertProgress.elapsedMs / 1000) }) }}</span>
</div>
- <Button variant="outline" size="sm" class="h-7 text-xs" @click="regenerate">{{ t("dataGenerate.regenerate") }}</Button>
</div>
<div class="flex flex-col h-[380px]">
<div class="flex-1 overflow-auto overscroll-none bg-background">
@@ -982,7 +1216,9 @@ async function onFileSelected(event: Event) {
</tr>
</tbody>
</table>
- <div v-if="currentPreview.rows.length > 50" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">{{ t("dataGenerate.first50Rows", { count: currentPreview.rows.length }) }}</div>
+ <div v-if="currentPreview.isSample" class="sticky left-0 text-xs text-muted-foreground px-3 py-1.5 border-t border-border bg-background">
+ {{ t("dataGenerate.samplePreviewNotice", { shown: currentPreview.rows.length, count: currentPreview.targetRowCount }) }}
+ </div>
</div>
</div>
</template>
@@ -1003,6 +1239,7 @@ async function onFileSelected(event: Event) {
</span>
</div>
<div v-if="r.error" class="mt-1 text-destructive/80 break-all leading-relaxed">{{ r.error }}</div>
+ <div v-else-if="r.cancelled" class="mt-1 text-muted-foreground">{{ t("dataGenerate.cancelledLabel") }}</div>
</div>
</div>
<div v-if="executeResults.length === 0" class="flex h-24 items-center justify-center text-xs text-muted-foreground">{{ t("dataGenerate.noResult") }}</div>
@@ -1030,7 +1267,7 @@ async function onFileSelected(event: Event) {
</Button>
</template>
<template v-else-if="currentStep === 'preview' && generatedResults.length > 0">
- <Button variant="outline" size="sm" class="h-7 text-xs" @click="copyAllSql">{{ t("dataGenerate.copyAllSql") }}</Button>
+ <Button variant="outline" size="sm" class="h-7 text-xs" @click="copySampleSql">{{ t("dataGenerate.copySampleSql") }}</Button>
</template>
</div>
<div class="flex items-center gap-2">
@@ -1044,7 +1281,8 @@ async function onFileSelected(event: Event) {
</Button>
</template>
<template v-else-if="currentStep === 'preview'">
- <Button variant="outline" size="sm" class="h-7 text-xs" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
+ <Button variant="outline" size="sm" class="h-7 text-xs" :disabled="!executing" @click="cancelInsert">{{ t("dataGenerate.cancelInsert") }}</Button>
+ <Button variant="outline" size="sm" class="h-7 text-xs" :disabled="executing" @click="currentStep = 'config'">{{ t("dataGenerate.prevStep") }}</Button>
<Button variant="default" size="sm" class="h-7 text-xs" :disabled="executing || !!generationError" @click="startInsert">
<Loader2 v-if="executing" class="mr-1 h-3 w-3 animate-spin" />
{{ t("dataGenerate.startInsert") }}
@@ -1081,6 +1319,16 @@ async function onFileSelected(event: Event) {
<input type="checkbox" v-model="generateOptions.extendedInsert" class="h-4 w-4 accent-primary" :disabled="!supportsExtendedInsert" />
<span class="text-xs">{{ t("dataGenerate.extendedInsert") }}</span>
</label>
+ <div class="flex items-center gap-3">
+ <Label class="text-xs shrink-0">{{ t("dataGenerate.timeoutSecs") }}:</Label>
+ <Input v-model.number="generateOptions.timeoutSecs" type="number" min="0" max="86400" class="h-7 w-24 text-xs" @blur="generateOptions.timeoutSecs = normalizeTimeoutSecs(generateOptions.timeoutSecs)" />
+ </div>
+ <p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.timeoutHint") }}</p>
+ <div class="flex items-center gap-3">
+ <Label class="text-xs shrink-0">{{ t("dataGenerate.batchRows") }}:</Label>
+ <Input v-model.number="generateOptions.batchRows" type="number" min="1" max="100000" class="h-7 w-24 text-xs" />
+ </div>
+ <p class="text-[11px] leading-relaxed text-muted-foreground">{{ t("dataGenerate.batchRowsHint") }}</p>
</div>
<DialogFooter>
<Button size="sm" class="h-7 text-xs" @click="optionsDialogOpen = false">{{ t("dataGenerate.ok") }}</Button>
diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts
index 2ad1ace2b9..2e6ca753a9 100644
--- a/apps/desktop/src/i18n/locales/en.ts
+++ b/apps/desktop/src/i18n/locales/en.ts
@@ -5643,7 +5643,7 @@ export default {
successLabel: "{count} succeeded",
failLabel: "{count} failed",
totalLabel: "/ {count} entries",
- copyAllSql: "Copy All SQL",
+ copySampleSql: "Copy sample SQL",
prevStep: "Previous",
startInsert: "Start Insert",
confirm: "Confirm",
@@ -5869,6 +5869,16 @@ export default {
drivers_license: "Driver's License",
custom: "Custom",
},
+ insertingTable: "Writing to {table} (table {index}/{count})",
+ elapsed: "{seconds}s elapsed",
+ samplePreviewNotice: "Showing the first {shown} sample rows; all {count} rows are generated while inserting",
+ largeRowCountHint: "Large row count ({count}); rows are generated and inserted in batches and you can cancel at any time",
+ timeoutSecs: "Timeout (seconds)",
+ timeoutHint: "0 means no timeout; keep 0 or raise it for large data volumes",
+ batchRows: "Rows per batch",
+ batchRowsHint: "Rows generated and inserted per round trip; oversized batches are split automatically",
+ cancelInsert: "Cancel insert",
+ cancelledLabel: "Cancelled; rows already written remain in the table",
},
tableToolbox: {
title: "Data Tools",
diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts
index 722d0237af..71775c35f2 100644
--- a/apps/desktop/src/i18n/locales/es.ts
+++ b/apps/desktop/src/i18n/locales/es.ts
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
successLabel: "{count} exitosas",
failLabel: "{count} fallidas",
totalLabel: "/ {count} filas",
- copyAllSql: "Copy All SQL",
+ copySampleSql: "Copiar SQL de ejemplo",
prevStep: "Anterior",
startInsert: "Start Insert",
confirm: "Confirmar",
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
drivers_license: "Carnet de conducir",
custom: "Personalizado",
},
+ insertingTable: "Escribiendo en {table} (tabla {index}/{count})",
+ elapsed: "{seconds} s transcurridos",
+ samplePreviewNotice: "Se muestran las primeras {shown} filas de ejemplo; las {count} filas se generan al insertar",
+ largeRowCountHint: "Número de filas elevado ({count}); se generan e insertan por lotes y puede cancelar en cualquier momento",
+ timeoutSecs: "Tiempo de espera (segundos)",
+ timeoutHint: "0 significa sin límite; mantenga 0 o auméntelo para grandes volúmenes",
+ batchRows: "Filas por lote",
+ batchRowsHint: "Filas generadas e insertadas en cada envío; los lotes grandes se dividen automáticamente",
+ cancelInsert: "Cancelar inserción",
+ cancelledLabel: "Cancelado; las filas ya insertadas permanecen en la tabla",
},
tableToolbox: {
title: "Herramientas de datos",
diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts
index 2eaf53c59c..c902d05dbe 100644
--- a/apps/desktop/src/i18n/locales/it.ts
+++ b/apps/desktop/src/i18n/locales/it.ts
@@ -5361,7 +5361,7 @@ export default withEnglishFallback({
successLabel: "{count} riuscite",
failLabel: "{count} non riuscite",
totalLabel: "/ {count} righe",
- copyAllSql: "Copia tutto SQL",
+ copySampleSql: "Copia SQL di esempio",
prevStep: "Precedente",
startInsert: "Avvia inserimento",
confirm: "Conferma",
@@ -5587,6 +5587,16 @@ export default withEnglishFallback({
drivers_license: "Patente di guida",
custom: "Personalizzato",
},
+ insertingTable: "Scrittura in {table} (tabella {index}/{count})",
+ elapsed: "{seconds}s trascorsi",
+ samplePreviewNotice: "Prime {shown} righe di esempio; le {count} righe vengono generate durante l'inserimento",
+ largeRowCountHint: "Numero elevato di righe ({count}); le righe vengono generate e inserite in batch ed è possibile annullare in qualsiasi momento",
+ timeoutSecs: "Timeout (secondi)",
+ timeoutHint: "0 significa nessun timeout; mantieni 0 o aumentalo per grandi volumi",
+ batchRows: "Righe per batch",
+ batchRowsHint: "Righe generate e inserite per invio; i batch troppo grandi vengono divisi automaticamente",
+ cancelInsert: "Annulla inserimento",
+ cancelledLabel: "Annullato; le righe già scritte restano nella tabella",
},
tableToolbox: {
title: "Strumenti dati",
diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts
index d2f62b03a6..e98d3bc060 100644
--- a/apps/desktop/src/i18n/locales/ja.ts
+++ b/apps/desktop/src/i18n/locales/ja.ts
@@ -5399,7 +5399,7 @@ export default withEnglishFallback({
successLabel: "{count}件成功",
failLabel: "{count}件失敗",
totalLabel: "/ {count}エントリ",
- copyAllSql: "すべてのSQLをコピー",
+ copySampleSql: "サンプル SQL をコピー",
prevStep: "戻る",
startInsert: "挿入開始",
confirm: "確認",
@@ -5625,6 +5625,16 @@ export default withEnglishFallback({
drivers_license: "運転免許証",
custom: "カスタム",
},
+ insertingTable: "{table} に書き込み中({index}/{count} テーブル)",
+ elapsed: "経過時間 {seconds} 秒",
+ samplePreviewNotice: "先頭 {shown} 行はサンプルです。{count} 行は挿入時に順次生成されます",
+ largeRowCountHint: "行数が多いため({count} 行)、生成と挿入をバッチで行います。いつでもキャンセルできます",
+ timeoutSecs: "タイムアウト(秒)",
+ timeoutHint: "0 はタイムアウトなしです。大量データでは 0 のままにするか、より長い時間を設定してください",
+ batchRows: "1 バッチの行数",
+ batchRowsHint: "1 回で生成・挿入する行数。上限を超える場合は自動的に分割されます",
+ cancelInsert: "挿入をキャンセル",
+ cancelledLabel: "キャンセルしました。書き込み済みの行はテーブルに残ります",
},
tableToolbox: {
title: "データツール",
diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts
index 97d18958bc..33e2ad67cc 100644
--- a/apps/desktop/src/i18n/locales/ko.ts
+++ b/apps/desktop/src/i18n/locales/ko.ts
@@ -5060,7 +5060,7 @@ export default withEnglishFallback({
successLabel: "{count}개 성공",
failLabel: "{count}개 실패",
totalLabel: "/ {count}개 항목",
- copyAllSql: "모든 SQL 복사",
+ copySampleSql: "샘플 SQL 복사",
prevStep: "이전",
startInsert: "삽입 시작",
confirm: "확인",
@@ -5286,6 +5286,16 @@ export default withEnglishFallback({
drivers_license: "운전면허증",
custom: "사용자 지정",
},
+ insertingTable: "{table}에 쓰는 중 (표 {index}/{count})",
+ elapsed: "{seconds}초 경과",
+ samplePreviewNotice: "앞의 {shown}행은 예시입니다. 전체 {count}행은 삽입할 때 생성됩니다",
+ largeRowCountHint: "행 수가 많습니다({count}행). 배치로 생성과 삽입을 진행하며 언제든 취소할 수 있습니다",
+ timeoutSecs: "제한 시간(초)",
+ timeoutHint: "0이면 제한 시간 없음입니다. 대량 데이터에서는 0을 유지하거나 더 길게 설정하세요",
+ batchRows: "배치당 행 수",
+ batchRowsHint: "한 번에 생성하고 삽입할 행 수입니다. 서버 한도를 넘으면 자동으로 분할됩니다",
+ cancelInsert: "삽입 취소",
+ cancelledLabel: "취소되었습니다. 이미 삽입된 행은 테이블에 남습니다",
},
tableToolbox: {
title: "데이터 도구",
diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts
index f621bf2e6a..2d45f073b1 100644
--- a/apps/desktop/src/i18n/locales/pt-BR.ts
+++ b/apps/desktop/src/i18n/locales/pt-BR.ts
@@ -5363,7 +5363,7 @@ export default withEnglishFallback({
successLabel: "{count} bem-sucedidas",
failLabel: "{count} falhas",
totalLabel: "/ {count} linhas",
- copyAllSql: "Copy All SQL",
+ copySampleSql: "Copiar SQL de exemplo",
prevStep: "Anterior",
startInsert: "Start Insert",
confirm: "Confirmar",
@@ -5589,6 +5589,16 @@ export default withEnglishFallback({
drivers_license: "Carteira de motorista",
custom: "Personalizado",
},
+ insertingTable: "Gravando em {table} (tabela {index}/{count})",
+ elapsed: "{seconds}s decorridos",
+ samplePreviewNotice: "Exibindo as primeiras {shown} linhas de exemplo; as {count} linhas são geradas durante a inserção",
+ largeRowCountHint: "Grande número de linhas ({count}); as linhas são geradas e inseridas em lotes e você pode cancelar a qualquer momento",
+ timeoutSecs: "Tempo limite (segundos)",
+ timeoutHint: "0 significa sem limite; mantenha 0 ou aumente para grandes volumes",
+ batchRows: "Linhas por lote",
+ batchRowsHint: "Linhas geradas e inseridas por envio; lotes grandes são divididos automaticamente",
+ cancelInsert: "Cancelar inserção",
+ cancelledLabel: "Cancelado; as linhas já gravadas permanecem na tabela",
},
tableToolbox: {
title: "Ferramentas de dados",
diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts
index 3e8cbff671..d3c0ad5be6 100644
--- a/apps/desktop/src/i18n/locales/tr.ts
+++ b/apps/desktop/src/i18n/locales/tr.ts
@@ -5633,7 +5633,7 @@ export default withEnglishFallback({
successLabel: "{count} başarılı",
failLabel: "{count} başarısız",
totalLabel: "/ {count} girdi",
- copyAllSql: "Tüm SQL'i Kopyala",
+ copySampleSql: "Örnek SQL'i kopyala",
prevStep: "Geri",
startInsert: "Eklemeyi Başlat",
confirm: "Onayla",
@@ -5859,6 +5859,16 @@ export default withEnglishFallback({
drivers_license: "Sürücü Belgesi",
custom: "Özel",
},
+ insertingTable: "{table} yazılıyor (tablo {index}/{count})",
+ elapsed: "{seconds}s geçti",
+ samplePreviewNotice: "İlk {shown} örnek satır gösteriliyor; {count} satır ekleme sırasında oluşturulur",
+ largeRowCountHint: "Satır sayısı yüksek ({count}); satırlar toplu olarak oluşturulup eklenir ve istediğiniz zaman iptal edebilirsiniz",
+ timeoutSecs: "Zaman aşımı (saniye)",
+ timeoutHint: "0 sınırsız anlamına gelir; büyük verilerde 0 olarak bırakın veya daha uzun bir süre ayarlayın",
+ batchRows: "Toplu iş başına satır",
+ batchRowsHint: "Her gönderimde oluşturulup eklenen satır sayısı; büyük toplu işler otomatik olarak bölünür",
+ cancelInsert: "Eklemeyi iptal et",
+ cancelledLabel: "İptal edildi; yazılan satırlar tabloda kalır",
},
tableToolbox: {
title: "Veri Araçları",
diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts
index 81715a4c6a..0067b00625 100644
--- a/apps/desktop/src/i18n/locales/zh-CN.ts
+++ b/apps/desktop/src/i18n/locales/zh-CN.ts
@@ -5627,7 +5627,7 @@ export default withEnglishFallback({
successLabel: "{count} 成功",
failLabel: "{count} 失败",
totalLabel: "/ {count} 条",
- copyAllSql: "复制全部 SQL",
+ copySampleSql: "复制样例 SQL",
prevStep: "上一步",
startInsert: "开始插入",
confirm: "确定",
@@ -5853,6 +5853,16 @@ export default withEnglishFallback({
drivers_license: "驾驶证号",
custom: "自定义",
},
+ insertingTable: "正在写入 {table}(第 {index}/{count} 张表)",
+ elapsed: "已用时 {seconds} 秒",
+ samplePreviewNotice: "以上为前 {shown} 行样例数据,共 {count} 行将在插入时流式生成",
+ largeRowCountHint: "行数较大({count} 行),插入时将采用边生成边写入的批量模式,可随时取消",
+ timeoutSecs: "超时时间(秒)",
+ timeoutHint: "0 表示不限制超时;数据量较大时建议保持 0 或设置更长时间",
+ batchRows: "每批行数",
+ batchRowsHint: "每次生成并写入的行数;超过服务端报文上限时会自动再拆分",
+ cancelInsert: "取消插入",
+ cancelledLabel: "已取消,已写入的行保留在表中",
},
tableToolbox: {
title: "数据工具",
diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts
index 22974fb1ae..b5a51b9d42 100644
--- a/apps/desktop/src/i18n/locales/zh-TW.ts
+++ b/apps/desktop/src/i18n/locales/zh-TW.ts
@@ -4683,7 +4683,7 @@ export default withEnglishFallback({
successLabel: "{count} 成功",
failLabel: "{count} 失敗",
totalLabel: "/ {count} 條",
- copyAllSql: "複製全部 SQL",
+ copySampleSql: "複製範例 SQL",
prevStep: "上一步",
startInsert: "開始插入",
confirm: "確定",
@@ -4909,6 +4909,16 @@ export default withEnglishFallback({
drivers_license: "駕駛證號",
custom: "自訂",
},
+ insertingTable: "正在寫入 {table}(第 {index}/{count} 張表)",
+ elapsed: "已用時 {seconds} 秒",
+ samplePreviewNotice: "以上為前 {shown} 列範例資料,共 {count} 列將在插入時串流產生",
+ largeRowCountHint: "列數較大({count} 列),插入時將採用邊產生邊寫入的批次模式,可隨時取消",
+ timeoutSecs: "逾時時間(秒)",
+ timeoutHint: "0 表示不限制逾時;資料量較大時建議保持 0 或設定更長時間",
+ batchRows: "每批列數",
+ batchRowsHint: "每次產生並寫入的列數;超過伺服端封包上限時會自動再拆分",
+ cancelInsert: "取消插入",
+ cancelledLabel: "已取消,已寫入的列保留在表中",
},
tableToolbox: {
title: "數據工具",
diff --git a/apps/desktop/src/lib/dataGrid/dataGenerate.ts b/apps/desktop/src/lib/dataGrid/dataGenerate.ts
index 433b8452ab..8e8571d365 100644
--- a/apps/desktop/src/lib/dataGrid/dataGenerate.ts
+++ b/apps/desktop/src/lib/dataGrid/dataGenerate.ts
@@ -1827,24 +1827,62 @@ function generateTdengineChildTableName(): string {
return `dbx_gen_${Date.now().toString(36)}_${randomSuffix}`;
}
-export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
+/**
+ * Cross-chunk generation state.
+ *
+ * Bulk generation is streamed one chunk at a time so the UI never holds the
+ * full row set in memory. Anything that must stay coherent across chunks
+ * (uniqueness sets, TDengine tag values, the global row index, the TDengine
+ * child-table name) therefore has to live outside the chunk function.
+ */
+export interface TableGenerateChunkState {
+ readonly isTdengineStable: boolean;
+ readonly shouldAddTbname: boolean;
+ readonly tbname: string | null;
+ readonly colNames: string[];
+ readonly targetTable: string;
+ readonly columnList: string;
+ readonly insertPrefix: string;
+ readonly uniqueValues: (Set<string> | null)[];
+ readonly tagValues: Map<string, unknown>;
+ nextIndex: number;
+}
+
+export function createTableGenerateState(config: TableGenerateConfig, databaseType?: DatabaseType): TableGenerateChunkState {
const isTdengineStable = isTdengineStableGenerate(config, databaseType);
const hasTbnameColumn = config.columns.some((column) => column.columnName.toLowerCase() === "tbname");
const shouldAddTbname = isTdengineStable && !hasTbnameColumn;
- const tdengineChildTableName = shouldAddTbname ? generateTdengineChildTableName() : null;
- const tagValues = new Map<string, unknown>();
- const uniqueValues = config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null));
const colNames = shouldAddTbname ? ["tbname", ...config.columns.map((c) => c.columnName)] : config.columns.map((c) => c.columnName);
+ const columnList = colNames.map((column) => quoteTableIdentifier(databaseType, column)).join(", ");
+ const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
+ return {
+ isTdengineStable,
+ shouldAddTbname,
+ tbname: shouldAddTbname ? generateTdengineChildTableName() : null,
+ colNames,
+ targetTable,
+ columnList,
+ insertPrefix: `INSERT INTO ${targetTable} (${columnList}) VALUES`,
+ uniqueValues: config.columns.map((column) => (column.generatorParams?.unique ? new Set<string>() : null)),
+ tagValues: new Map<string, unknown>(),
+ nextIndex: 0,
+ };
+}
+
+/** Generates the next `chunkSize` rows (or whatever remains) and advances `state.nextIndex`. */
+export function generateTableRowsChunk(config: TableGenerateConfig, state: TableGenerateChunkState, chunkSize: number): unknown[][] {
const rows: unknown[][] = [];
-
- for (let i = 0; i < config.rowCount; i++) {
+ if (chunkSize <= 0) return rows;
+ const total = Math.max(0, config.rowCount);
+ const end = Math.min(total, state.nextIndex + chunkSize);
+ for (let i = state.nextIndex; i < end; i++) {
const row = config.columns.map((col, columnIndex) => {
- if (isTdengineStable && col.isTag && tagValues.has(col.columnName)) {
- return tagValues.get(col.columnName);
+ if (state.isTdengineStable && col.isTag && state.tagValues.has(col.columnName)) {
+ return state.tagValues.get(col.columnName);
}
let value: unknown;
if (col.generatorParams?.unique) {
- const seen = uniqueValues[columnIndex]!;
+ const seen = state.uniqueValues[columnIndex]!;
let found = false;
for (let attempt = 0; attempt < MAX_UNIQUE_GENERATION_ATTEMPTS; attempt++) {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
@@ -1861,29 +1899,98 @@ export function generateTableData(config: TableGenerateConfig, databaseType?: Da
} else {
value = generateValue(col.columnName, col.dataType, col.generatorKey, i, col.generatorParams, col.isAutoIncrement ? null : col.columnDefault);
}
- if (isTdengineStable && col.isTag) {
- tagValues.set(col.columnName, value);
+ if (state.isTdengineStable && col.isTag) {
+ state.tagValues.set(col.columnName, value);
}
return value;
});
- rows.push(shouldAddTbname ? [tdengineChildTableName, ...row] : row);
+ rows.push(state.tbname !== null ? [state.tbname, ...row] : row);
+ }
+ state.nextIndex = end;
+ return rows;
+}
+
+export function formatGeneratedRowValues(config: TableGenerateConfig, databaseType: DatabaseType | undefined, state: TableGenerateChunkState, row: unknown[]): string {
+ const values = row
+ .map((value, index) => {
+ const configIndex = state.shouldAddTbname ? index - 1 : index;
+ return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
+ })
+ .join(", ");
+ return `(${values})`;
+}
+
+/**
+ * Builds the INSERT statements for a batch together with the number of value
+ * rows each statement carries.
+ *
+ * `rowsPerStatement` is aligned index-by-index with `statements`, so a caller
+ * that executes the statements can map each returned per-statement result back
+ * to the rows it inserted. This is what makes failed-batch accounting possible
+ * on backends that report failures inside the result array instead of throwing.
+ */
+export function generateInsertBatches(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): { statements: string[]; rowsPerStatement: number[] } {
+ if (databaseType === "oracle") {
+ const statements = buildOracleInsertStatements(state.targetTable, state.columnList, valueRows);
+ return { statements, rowsPerStatement: oracleRowsPerStatement(valueRows, statements) };
+ }
+ if (!supportsGeneratedMultiRowValues(databaseType) || forceSingleRow) {
+ return { statements: valueRows.map((values) => `${state.insertPrefix} ${values};`), rowsPerStatement: valueRows.map(() => 1) };
+ }
+ if (valueRows.length === 0) return { statements: [], rowsPerStatement: [] };
+ return { statements: [`${state.insertPrefix}\n${valueRows.join(",\n")};`], rowsPerStatement: [valueRows.length] };
+}
+
+/**
+ * Oracle batches INSERT ALL statements `ORACLE_INSERT_ALL_BATCH_SIZE` rows at a
+ * time, so the per-statement row counts have to mirror that chunking.
+ */
+function oracleRowsPerStatement(valueRows: string[], statements: string[]): number[] {
+ if (statements.length === 0) return [];
+ // A single value row is emitted as a plain VALUES statement, not INSERT ALL.
+ if (valueRows.length === 1) return [1];
+ const counts: number[] = [];
+ for (let remaining = valueRows.length; remaining > 0; remaining -= ORACLE_INSERT_ALL_BATCH_SIZE) {
+ counts.push(Math.min(ORACLE_INSERT_ALL_BATCH_SIZE, remaining));
+ }
+ return counts;
+}
+
+export function buildGenerateInsertStatements(databaseType: DatabaseType | undefined, state: TableGenerateChunkState, valueRows: string[], forceSingleRow = false): string[] {
+ return generateInsertBatches(databaseType, state, valueRows, forceSingleRow).statements;
+}
+
+/**
+ * Splits formatted value rows into statement groups that stay within a byte budget.
+ *
+ * Servers reject statements larger than `max_allowed_packet`, so an oversized
+ * multi-row INSERT has to be cut into several statements before it is sent.
+ * Splitting happens after generation (never by rewinding the generator state)
+ * so uniqueness bookkeeping stays intact.
+ */
+export function splitValueRowsByByteBudget(state: TableGenerateChunkState, valueRows: string[], maxBytes: number): string[][] {
+ if (maxBytes <= 0) return valueRows.length > 0 ? [valueRows] : [];
+ const groups: string[][] = [];
+ let current: string[] = [];
+ let currentBytes = 0;
+ for (const values of valueRows) {
+ const bytes = values.length + state.insertPrefix.length + 4;
+ if (current.length > 0 && currentBytes + bytes > maxBytes) {
+ groups.push(current);
+ current = [];
+ currentBytes = 0;
+ }
+ current.push(values);
+ currentBytes += bytes;
}
+ if (current.length > 0) groups.push(current);
+ return groups;
+}
- const quotedCols = colNames.map((column) => quoteTableIdentifier(databaseType, column));
- const targetTable = qualifiedTableName({ databaseType, schema: config.schema, tableName: config.tableName, database: config.database });
- const columnList = quotedCols.join(", ");
- const insertPrefix = `INSERT INTO ${targetTable} (${columnList}) VALUES`;
- const valueRows = rows.map(
- (row) =>
- `(${row
- .map((value, index) => {
- const configIndex = shouldAddTbname ? index - 1 : index;
- return formatGeneratedValue(value, databaseType, configIndex >= 0 ? config.columns[configIndex]?.dataType : undefined);
- })
- .join(", ")})`,
- );
- const statements = databaseType === "oracle" ? buildOracleInsertStatements(targetTable, columnList, valueRows) : supportsGeneratedMultiRowValues(databaseType) ? [`${insertPrefix}\n${valueRows.join(",\n")};`] : valueRows.map((values) => `${insertPrefix} ${values};`);
- const sql = statements.join("\n");
-
- return { columns: colNames, rows, sql, statements };
+export function generateTableData(config: TableGenerateConfig, databaseType?: DatabaseType): GenerateResult {
+ const state = createTableGenerateState(config, databaseType);
+ const rows = generateTableRowsChunk(config, state, config.rowCount);
+ const valueRows = rows.map((row) => formatGeneratedRowValues(config, databaseType, state, row));
+ const statements = buildGenerateInsertStatements(databaseType, state, valueRows);
+ return { columns: state.colNames, rows, sql: statements.join("\n"), statements };
}
diff --git a/apps/desktop/src/lib/dataGrid/generateInsertAccounting.ts b/apps/desktop/src/lib/dataGrid/generateInsertAccounting.ts
new file mode 100644
index 0000000000..467c21c22f
--- /dev/null
+++ b/apps/desktop/src/lib/dataGrid/generateInsertAccounting.ts
@@ -0,0 +1,95 @@
+import { BackendErrorException } from "@/lib/backend/errorUtils";
+import type { QueryResult } from "@/types/database";
+
+export interface BatchExecutionOutcome {
+ /** Rows the backend confirmed, counted only from statements that succeeded. */
+ readonly insertedRows: number;
+ /** True when at least one statement in the batch failed. */
+ readonly failed: boolean;
+ /** Message of the first failed statement, when one is available. */
+ readonly error?: string;
+ /** Index of the first failed statement, when one is available. */
+ readonly failedStatement?: number;
+}
+
+const EMPTY_OUTCOME: BatchExecutionOutcome = { insertedRows: 0, failed: false };
+
+/**
+ * Maps the per-statement results of one executed batch back to the rows that
+ * were really inserted.
+ *
+ * Not every backend throws when a statement fails: the sequential executor
+ * (`query.rs`) records the failure in the result array and still returns `Ok`,
+ * and the MySQL batch route returns `Ok` with an embedded error result even on
+ * connection checkout failure. Counting rows on the `await` alone therefore
+ * over-reports inserts and can leave `continueOnError` silently dropping rows.
+ *
+ * `rowsPerStatement` comes from `generateInsertBatches()` and is aligned with
+ * the statements that were sent; a statement counts as successful only when it
+ * carries neither `execution_error` nor an `error`.
+ */
+export function summarizeBatchResults(results: QueryResult[] | null | undefined, rowsPerStatement: number[]): BatchExecutionOutcome {
+ if (!results || results.length === 0 || rowsPerStatement.length === 0) return EMPTY_OUTCOME;
+
+ let insertedRows = 0;
+ let error: string | undefined;
+ let failedStatement: number | undefined;
+
+ for (let index = 0; index < results.length && index < rowsPerStatement.length; index++) {
+ const result = results[index];
+ const statementError = statementErrorMessage(result);
+ if (statementError === undefined) {
+ insertedRows += Math.max(0, rowsPerStatement[index]);
+ continue;
+ }
+ if (failedStatement === undefined) {
+ failedStatement = index;
+ error = statementError;
+ }
+ }
+
+ const failed = failedStatement !== undefined;
+ return failed ? { insertedRows, failed, error, failedStatement } : { insertedRows, failed };
+}
+
+function statementErrorMessage(result: QueryResult): string | undefined {
+ if (!result) return "Unknown statement failure";
+ if (result.execution_error === true) {
+ return result.error?.detail ?? result.error?.messageKey ?? errorMessageFromRows(result);
+ }
+ if (result.error) {
+ return result.error.detail ?? result.error.messageKey;
+ }
+ return undefined;
+}
+
+/** Error-shaped results carry the driver message in a single "Error" row. */
+function errorMessageFromRows(result: QueryResult): string | undefined {
+ const firstRow = result.rows?.[0];
+ const firstCell = Array.isArray(firstRow) ? firstRow[0] : undefined;
+ return typeof firstCell === "string" && firstCell.trim().length > 0 ? firstCell : undefined;
+}
+
+const CANCELLATION_PATTERN = /\bcancel(?:ed|led|ing|ling|lation)?\b/i;
+
+/**
+ * Recognizes a failure produced by cancelling a query.
+ *
+ * Cancelling a transaction or a single-statement round trip surfaces as an
+ * error instead of a quiet stop, so a user-initiated cancel has to be told
+ * apart from a real SQL failure. The backend reports cancellation as
+ * "Query canceled" (`QUERY_CANCELED`); structured backend errors are matched on
+ * their code/message key as well.
+ */
+export function isQueryCanceledError(error: unknown): boolean {
+ if (error instanceof BackendErrorException) {
+ const { code, messageKey, detail } = error.backendError;
+ if (CANCELLATION_PATTERN.test(code) || CANCELLATION_PATTERN.test(messageKey)) return true;
+ if (detail !== undefined && CANCELLATION_PATTERN.test(detail)) return true;
+ }
+ return CANCELLATION_PATTERN.test(errorMessage(error));
+}
+
+export function errorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
diff --git a/packages/app-tests/dataGenerateChunk.test.ts b/packages/app-tests/dataGenerateChunk.test.ts
new file mode 100644
index 0000000000..79342d7d26
--- /dev/null
+++ b/packages/app-tests/dataGenerateChunk.test.ts
@@ -0,0 +1,209 @@
+import { strict as assert } from "node:assert";
+import { test } from "vitest";
+import {
+ buildGenerateInsertStatements,
+ createTableGenerateState,
+ generateTableData,
+ generateTableRowsChunk,
+ splitValueRowsByByteBudget,
+ type TableGenerateConfig,
+} from "../../apps/desktop/src/lib/dataGrid/dataGenerate.ts";
+
+function makeConfig(overrides: Partial<TableGenerateConfig> = {}): TableGenerateConfig {
+ return {
+ tableName: "users",
+ schema: "public",
+ database: "app",
+ rowCount: 10,
+ columns: [
+ { columnName: "id", dataType: "bigint", rowCount: 10, generatorKey: "sequence", generatorParams: { startValue: 1, increment: 1 } },
+ { columnName: "name", dataType: "varchar(32)", rowCount: 10, generatorKey: "text" },
+ ],
+ ...overrides,
+ };
+}
+
+function deterministicConfig(rowCount: number): TableGenerateConfig {
+ return {
+ tableName: "users",
+ schema: "public",
+ database: "app",
+ rowCount,
+ columns: [
+ { columnName: "id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 100, increment: 5 } },
+ { columnName: "copy_of_id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 0, increment: -1 } },
+ ],
+ };
+}
+
+test("chunked generation produces exactly the same rows as one-shot generation", () => {
+ const config = deterministicConfig(11);
+ const expected = generateTableData(config, "mysql");
+
+ const state = createTableGenerateState(config, "mysql");
+ const chunked: unknown[][] = [];
+ for (;;) {
+ const rows = generateTableRowsChunk(config, state, 3);
+ if (rows.length === 0) break;
+ chunked.push(...rows);
+ }
+
+ assert.deepEqual(chunked, expected.rows);
+});
+
+test("unique values stay distinct across chunks", () => {
+ const config = makeConfig({
+ rowCount: 24,
+ columns: [{ columnName: "code", dataType: "varchar(8)", rowCount: 24, generatorKey: "text", generatorParams: { unique: true } }],
+ });
+
+ const state = createTableGenerateState(config, "mysql");
+ const all: unknown[] = [];
+ for (;;) {
+ const rows = generateTableRowsChunk(config, state, 5);
+ if (rows.length === 0) break;
+ all.push(...rows.map((row) => row[0]));
+ }
+
+ assert.equal(all.length, 24);
+ assert.equal(new Set(all.map((value) => String(value))).size, 24);
+});
+
+test("sequence values continue monotonically across chunk boundaries", () => {
+ const config = makeConfig({ rowCount: 7 });
+
+ const state = createTableGenerateState(config, "mysql");
+ const ids: number[] = [];
+ for (;;) {
+ const rows = generateTableRowsChunk(config, state, 2);
+ if (rows.length === 0) break;
+ ids.push(...rows.map((row) => row[0] as number));
+ }
+
+ assert.deepEqual(ids, [1, 2, 3, 4, 5, 6, 7]);
+});
+
+test("chunking respects exact boundaries and terminates cleanly", () => {
+ const config = makeConfig({ rowCount: 6 });
+ const state = createTableGenerateState(config, "mysql");
+
+ assert.equal(generateTableRowsChunk(config, state, 0).length, 0);
+ assert.equal(generateTableRowsChunk(config, state, -1).length, 0);
+ assert.equal(generateTableRowsChunk(config, state, 6).length, 6);
+ assert.equal(generateTableRowsChunk(config, state, 6).length, 0);
+ assert.equal(generateTableRowsChunk(config, state, 100).length, 0);
+});
+
+test("a chunk larger than the remaining row count returns only the remainder", () => {
+ const config = makeConfig({ rowCount: 4 });
+ const state = createTableGenerateState(config, "mysql");
+
+ assert.equal(generateTableRowsChunk(config, state, 3).length, 3);
+ assert.equal(generateTableRowsChunk(config, state, 10).length, 1);
+});
+
+test("tdengine stable table keeps tag values and tbname coherent across chunks", () => {
+ const config = makeConfig({
+ tableName: "metrics",
+ schema: "",
+ database: "db",
+ rowCount: 9,
+ tableType: "STABLE",
+ columns: [
+ { columnName: "ts", dataType: "timestamp", rowCount: 9, generatorKey: "time" },
+ { columnName: "region", dataType: "varchar(16)", rowCount: 9, generatorKey: "text", isTag: true },
+ ],
+ });
+
+ const state = createTableGenerateState(config, "tdengine");
+ assert.equal(state.shouldAddTbname, true);
+ assert.ok(state.tbname);
+
+ const tagValues = new Set<unknown>();
+ const tbnames = new Set<unknown>();
+ let total = 0;
+ for (;;) {
+ const rows = generateTableRowsChunk(config, state, 4);
+ if (rows.length === 0) break;
+ for (const row of rows) {
+ assert.equal(row.length, 3); // tbname + ts + region
+ tbnames.add(row[0]);
+ tagValues.add(row[2]);
+ total++;
+ }
+ }
+
+ assert.equal(total, 9);
+ assert.equal(tbnames.size, 1);
+ assert.equal(tagValues.size, 1); // tag is generated once and reused
+ assert.equal(state.colNames[0], "tbname");
+});
+
+test("buildGenerateInsertStatements merges mysql rows into one multi-row INSERT", () => {
+ const config = makeConfig({ rowCount: 3 });
+ const state = createTableGenerateState(config, "mysql");
+ const statements = buildGenerateInsertStatements("mysql", state, ["(1, 'a')", "(2, 'b')", "(3, 'c')"]);
+
+ assert.equal(statements.length, 1);
+ assert.match(statements[0], /^INSERT INTO /);
+ assert.match(statements[0], /VALUES\n\(1, 'a'\),\n\(2, 'b'\),\n\(3, 'c'\);$/);
+});
+
+test("buildGenerateInsertStatements falls back to one statement per row without multi-row support", () => {
+ const config = makeConfig({ rowCount: 2 });
+ const state = createTableGenerateState(config, "iris");
+ const statements = buildGenerateInsertStatements("iris", state, ["(1, 'a')", "(2, 'b')"]);
+
+ assert.equal(statements.length, 2);
+ assert.match(statements[0], / VALUES \(1, 'a'\);$/);
+ assert.match(statements[1], / VALUES \(2, 'b'\);$/);
+});
+
+test("buildGenerateInsertStatements honours forceSingleRow and empty input", () => {
+ const config = makeConfig();
+ const state = createTableGenerateState(config, "mysql");
+
+ assert.deepEqual(buildGenerateInsertStatements("mysql", state, [], false), []);
+ assert.deepEqual(buildGenerateInsertStatements("mysql", state, [], true), []);
+
+ const forced = buildGenerateInsertStatements("mysql", state, ["(1, 'a')", "(2, 'b')"], true);
+ assert.equal(forced.length, 2);
+});
+
+test("splitValueRowsByByteBudget keeps every group within the budget", () => {
+ const config = makeConfig();
+ const state = createTableGenerateState(config, "mysql");
+ const valueRows = ["(1, 'a')", "(2, 'b')", "(3, 'c')", "(4, 'd')"];
+
+ // Budget that only fits the prefix plus one row forces per-row grouping.
+ const prefixBytes = state.insertPrefix.length + 4;
+ const groups = splitValueRowsByByteBudget(state, valueRows, prefixBytes + 12);
+
+ assert.ok(groups.length > 1);
+ for (const group of groups) {
+ const bytes = state.insertPrefix.length + 4 + group.reduce((sum, values) => sum + values.length + 4, 0);
+ assert.ok(bytes <= prefixBytes + 12, `group exceeds budget: ${bytes}`);
+ }
+ assert.deepEqual(groups.flat(), valueRows);
+});
+
+test("splitValueRowsByByteBudget keeps an oversized single row in its own group", () => {
+ const config = makeConfig();
+ const state = createTableGenerateState(config, "mysql");
+ const huge = `(${"x".repeat(500)})`;
+
+ const groups = splitValueRowsByByteBudget(state, [huge, "(2, 'b')"], 64);
+ assert.equal(groups.length, 2);
+ assert.equal(groups[0][0], huge);
+ assert.equal(groups[1][0], "(2, 'b')");
+});
+
+test("splitValueRowsByByteBudget handles degenerate inputs", () => {
+ const config = makeConfig();
+ const state = createTableGenerateState(config, "mysql");
+
+ assert.deepEqual(splitValueRowsByByteBudget(state, [], 1024), []);
+ // maxBytes <= 0 disables splitting and returns a single group.
+ assert.deepEqual(splitValueRowsByByteBudget(state, ["(1, 'a')", "(2, 'b')"], 0), [["(1, 'a')", "(2, 'b')"]]);
+ assert.deepEqual(splitValueRowsByByteBudget(state, ["(1, 'a')"], -5), [["(1, 'a')"]]);
+});
diff --git a/packages/app-tests/dataGenerateInsertAccounting.test.ts b/packages/app-tests/dataGenerateInsertAccounting.test.ts
new file mode 100644
index 0000000000..81f6b8f812
--- /dev/null
+++ b/packages/app-tests/dataGenerateInsertAccounting.test.ts
@@ -0,0 +1,138 @@
+import { strict as assert } from "node:assert";
+import { test } from "vitest";
+import { BackendErrorException } from "../../apps/desktop/src/lib/backend/errorUtils.ts";
+import { createTableGenerateState, generateInsertBatches, type TableGenerateConfig } from "../../apps/desktop/src/lib/dataGrid/dataGenerate.ts";
+import { errorMessage, isQueryCanceledError, summarizeBatchResults } from "../../apps/desktop/src/lib/dataGrid/generateInsertAccounting.ts";
+import type { QueryResult } from "../../apps/desktop/src/types/database.ts";
+
+function makeConfig(rowCount: number): TableGenerateConfig {
+ return {
+ tableName: "users",
+ schema: "public",
+ database: "app",
+ rowCount,
+ columns: [{ columnName: "id", dataType: "bigint", rowCount, generatorKey: "sequence", generatorParams: { startValue: 1, increment: 1 } }],
+ };
+}
+
+function state(databaseType: string, rowCount: number) {
+ return createTableGenerateState(makeConfig(rowCount), databaseType as never);
+}
+
+function result(options: { executionError?: boolean; error?: { detail?: string; messageKey?: string }; rows?: unknown[][]; affected?: number } = {}): QueryResult {
+ return {
+ columns: ["affected"],
+ rows: options.rows ?? [],
+ affected_rows: options.affected ?? 0,
+ execution_time_ms: 1,
+ ...(options.executionError ? { execution_error: true as const } : {}),
+ ...(options.error ? { error: { version: 1, code: "DBX-TEST-0001", messageKey: options.error.messageKey ?? "backendErrors.test", messageParams: {}, source: "test", operationOutcome: "unknown", ...(options.error.detail ? { detail: options.error.detail } : {}) } } : {}),
+ } as QueryResult;
+}
+
+function valueRows(count: number): string[] {
+ return Array.from({ length: count }, (_unused, index) => `(${index + 1})`);
+}
+
+test("generateInsertBatches maps one multi-row statement to all of its rows", () => {
+ const batches = generateInsertBatches("mysql", state("mysql", 3), valueRows(3));
+ assert.equal(batches.statements.length, 1);
+ assert.deepEqual(batches.rowsPerStatement, [3]);
+});
+
+test("generateInsertBatches maps single-row statements one row each", () => {
+ const batches = generateInsertBatches("mysql", state("mysql", 3), valueRows(3), true);
+ assert.equal(batches.statements.length, 3);
+ assert.deepEqual(batches.rowsPerStatement, [1, 1, 1]);
+});
+
+test("generateInsertBatches keeps counts aligned for databases without multi-row support", () => {
+ const batches = generateInsertBatches("iris", state("iris", 2), valueRows(2));
+ assert.equal(batches.statements.length, 2);
+ assert.deepEqual(batches.rowsPerStatement, [1, 1]);
+});
+
+test("generateInsertBatches mirrors Oracle INSERT ALL chunking", () => {
+ const batches = generateInsertBatches("oracle", state("oracle", 101), valueRows(101));
+ assert.equal(batches.statements.length, 2);
+ assert.deepEqual(batches.rowsPerStatement, [100, 1]);
+
+ const single = generateInsertBatches("oracle", state("oracle", 1), valueRows(1));
+ assert.equal(single.statements.length, 1);
+ assert.deepEqual(single.rowsPerStatement, [1]);
+});
+
+test("generateInsertBatches returns nothing for an empty batch", () => {
+ const batches = generateInsertBatches("mysql", state("mysql", 0), []);
+ assert.deepEqual(batches.statements, []);
+ assert.deepEqual(batches.rowsPerStatement, []);
+});
+
+test("summarizeBatchResults counts every row when all statements succeed", () => {
+ const outcome = summarizeBatchResults([result({ affected: 3 })], [3]);
+ assert.equal(outcome.insertedRows, 3);
+ assert.equal(outcome.failed, false);
+ assert.equal(outcome.error, undefined);
+});
+
+test("summarizeBatchResults ignores failed statements that do not throw", () => {
+ // Sequential executor and the MySQL batch route resolve with the failure
+ // recorded in the result array instead of rejecting.
+ const outcome = summarizeBatchResults(
+ [result({ affected: 2 }), result({ executionError: true, error: { detail: "Duplicate entry" } }), result({ affected: 1 })],
+ [2, 5, 1],
+ );
+ assert.equal(outcome.insertedRows, 3);
+ assert.equal(outcome.failed, true);
+ assert.equal(outcome.error, "Duplicate entry");
+ assert.equal(outcome.failedStatement, 1);
+});
+
+test("summarizeBatchResults reads driver messages from error-shaped rows", () => {
+ const outcome = summarizeBatchResults([result({ executionError: true, rows: [["relation missing_table does not exist"]] })], [4]);
+ assert.equal(outcome.insertedRows, 0);
+ assert.equal(outcome.failed, true);
+ assert.equal(outcome.error, "relation missing_table does not exist");
+});
+
+test("summarizeBatchResults flags results that carry a structured error", () => {
+ const outcome = summarizeBatchResults([result({ error: { messageKey: "backendErrors.poolCheckoutFailed" } })], [7]);
+ assert.equal(outcome.failed, true);
+ assert.equal(outcome.insertedRows, 0);
+ assert.equal(outcome.error, "backendErrors.poolCheckoutFailed");
+});
+
+test("summarizeBatchResults ignores results beyond the statements that were sent", () => {
+ const outcome = summarizeBatchResults([result({ affected: 1 }), result({ affected: 1 }), result({ executionError: true })], [1, 1]);
+ assert.equal(outcome.insertedRows, 2);
+ assert.equal(outcome.failed, false);
+});
+
+test("summarizeBatchResults treats a missing result array as zero confirmed rows", () => {
+ const outcome = summarizeBatchResults(null, [3]);
+ assert.equal(outcome.insertedRows, 0);
+ assert.equal(outcome.failed, false);
+});
+
+test("summarizeBatchResults handles an empty batch", () => {
+ const outcome = summarizeBatchResults([], []);
+ assert.equal(outcome.insertedRows, 0);
+ assert.equal(outcome.failed, false);
+});
+
+test("isQueryCanceledError recognizes the backend cancellation message", () => {
+ assert.equal(isQueryCanceledError(new Error("Query canceled")), true);
+ assert.equal(isQueryCanceledError("Query canceled"), true);
+ assert.equal(isQueryCanceledError(new BackendErrorException({ message: "Query canceled" })), true);
+});
+
+test("isQueryCanceledError does not treat SQL failures as cancellations", () => {
+ assert.equal(isQueryCanceledError(new Error("relation \"users\" does not exist")), false);
+ assert.equal(isQueryCanceledError(new Error("Deadlock found when trying to get lock")), false);
+ assert.equal(isQueryCanceledError(new BackendErrorException({ message: "Syntax error" })), false);
+});
+
+test("errorMessage unwraps Error instances and plain values", () => {
+ assert.equal(errorMessage(new Error("boom")), "boom");
+ assert.equal(errorMessage("plain failure"), "plain failure");
+});