Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

feat(import): add continue-on-error option to table import

Abeautifulsnowcde3ec4f286f
diff --git a/Cargo.lock b/Cargo.lock
index c1f87a58f7..66ee43b731 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -304,7 +304,7 @@ dependencies = [
"wayland-backend",
"wayland-client",
"wayland-protocols",
- "zbus",
+ "zbus 5.18.0",
]
[[package]]
diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue
index 0c8ba48902..3af16b30b8 100644
--- a/apps/desktop/src/components/import/TableImportDialog.vue
+++ b/apps/desktop/src/components/import/TableImportDialog.vue
@@ -1,6 +1,7 @@
<script setup lang="ts">
import { computed, onBeforeUnmount, ref, shallowRef, watch } from "vue";
import { uuid } from "@/lib/common/utils";
+import { copyToClipboard } from "@/lib/common/clipboard";
import { useI18n } from "vue-i18n";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { Dialog, DialogHeader, DialogTitle, DialogFooter, DialogScrollContent } from "@/components/ui/dialog";
@@ -101,6 +102,7 @@ interface BatchImportTask {
columnDataTypes: Record<string, string>;
status: "pending" | "running" | "done" | "error" | "cancelled";
rowsImported: number;
+ rowsFailed: number;
error?: string;
}
@@ -165,6 +167,9 @@ const dataStartRow = ref(2);
const lastDataRow = ref(0);
const trimValues = ref(false);
const conflictPolicy = ref<api.TableImportConflictPolicy>("error");
+// Off by default: a failed statement must still stop the import unless the
+// user explicitly asks to skip failed rows (issue #13683).
+const continueOnError = ref(false);
const emptyStringAsNull = ref(defaultTableImportEmptyStringAsNull(sourceFormat.value));
const selectedSheet = ref("");
const jsonShape = ref<api.TableImportJsonShape>("auto");
@@ -385,6 +390,26 @@ const progressLabelKey = computed(() => {
if (!running.value && !progress.value) return "tableImport.status_idle";
return `tableImport.phase_${progress.value?.phase || "writing"}`;
});
+const importFailures = computed(() => progress.value?.failures ?? []);
+const failedRowListText = computed(() =>
+ importFailures.value
+ .map((failure) => {
+ const row = t("tableImport.failureRow", { row: failure.row });
+ const table = failure.table ? `${t("tableImport.failureTable", { table: failure.table })} · ` : "";
+ return `${table}${row}: ${failure.error}`;
+ })
+ .join("\n"),
+);
+
+function importSummaryMessage(imported: number, failed: number): string {
+ return failed > 0 ? t("tableImport.successWithFailures", { count: imported, failed }) : t("tableImport.success", { count: imported });
+}
+
+async function copyFailedRows() {
+ if (!failedRowListText.value) return;
+ await copyToClipboard(failedRowListText.value);
+ toast(t("tableImport.failedRowsCopied"), 2000);
+}
function previewRowsLabel(currentPreview: api.TableImportPreview | null | undefined): string {
if (!currentPreview) return "-";
@@ -513,6 +538,7 @@ function resetState() {
lastDataRow.value = 0;
trimValues.value = false;
conflictPolicy.value = "error";
+ continueOnError.value = false;
emptyStringAsNull.value = defaultTableImportEmptyStringAsNull(sourceFormat.value);
selectedSheet.value = "";
jsonShape.value = "auto";
@@ -996,6 +1022,7 @@ async function prepareBatchSources(sources: ImportSource[]) {
columnDataTypes: suggestImportTargetDataTypes(taskPreview.columns, taskPreview.rows, structureDatabaseType.value),
status: "pending",
rowsImported: 0,
+ rowsFailed: 0,
});
}
}
@@ -1367,6 +1394,7 @@ async function startImport() {
batchSize: Math.max(1, Number(batchSize.value) || 500),
conflictPolicy: conflictPolicy.value,
skipDuplicateRows: conflictPolicy.value === "skip",
+ continueOnError: continueOnError.value,
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
preparedSource: preparedImportSource(currentPreview),
},
@@ -1374,8 +1402,17 @@ async function startImport() {
progress.value = { ...nextProgress, elapsedMs: nextProgress.elapsedMs ?? liveElapsedMs.value };
},
);
- progress.value = { importId: summary.importId, status: "done", rowsImported: summary.rowsImported, totalRows: summary.totalRows, elapsedMs: summary.elapsedMs };
- toast(t("tableImport.success", { count: summary.rowsImported }), 2500);
+ progress.value = {
+ importId: summary.importId,
+ status: "done",
+ rowsImported: summary.rowsImported,
+ totalRows: summary.totalRows,
+ elapsedMs: summary.elapsedMs,
+ rowsFailed: summary.rowsFailed ?? 0,
+ failures: summary.failures ?? [],
+ failuresTruncated: summary.failuresTruncated ?? false,
+ };
+ toast(importSummaryMessage(summary.rowsImported, summary.rowsFailed ?? 0), 2500);
store.invalidateMetadataCache(props.prefillConnectionId, props.prefillDatabase || "", targetSchema.value || undefined, tableName);
if (targetMode.value === "create") {
store.refreshObjectListTreeNode(props.prefillConnectionId, props.prefillDatabase || "", targetSchema.value || undefined).catch((error) => {
@@ -1392,6 +1429,10 @@ async function startImport() {
totalRows: progress.value?.totalRows ?? currentPreview.totalRows,
elapsedMs: progress.value?.elapsedMs ?? liveElapsedMs.value,
error: message,
+ // Keep failed rows already reported before the fatal error/cancel (#13683).
+ rowsFailed: progress.value?.rowsFailed ?? 0,
+ failures: progress.value?.failures ?? [],
+ failuresTruncated: progress.value?.failuresTruncated ?? false,
};
} finally {
refreshImportElapsedClock();
@@ -1418,6 +1459,9 @@ async function startBatchImport() {
const totalBytes = tasks.reduce((sum, { task }) => sum + task.preview.sizeBytes, 0);
let completedRows = 0;
let completedBytes = 0;
+ let completedFailures = 0;
+ let completedFailureList: api.TableImportFailure[] = [];
+ let completedFailuresTruncated = false;
importId.value = uuid();
startImportElapsedClock();
progress.value = { importId: importId.value, status: "running", phase: "preparing", rowsImported: 0, totalRows, totalRowsExact, bytesRead: 0, totalBytes, elapsedMs: 0 };
@@ -1468,12 +1512,14 @@ async function startBatchImport() {
batchSize: Math.max(1, Number(batchSize.value) || 500),
conflictPolicy: conflictPolicy.value,
skipDuplicateRows: conflictPolicy.value === "skip",
+ continueOnError: continueOnError.value,
dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined,
preparedSource: preparedImportSource(task.preview),
retainSource: true,
},
(nextProgress) => {
task.rowsImported = nextProgress.rowsImported;
+ task.rowsFailed = nextProgress.rowsFailed ?? 0;
const hasRemainingTasks = queueIndex < tasks.length - 1;
const aggregateStatus = nextProgress.status === "done" && hasRemainingTasks ? "running" : nextProgress.status;
const aggregatePhase = nextProgress.status === "done" && hasRemainingTasks ? "preparing" : nextProgress.phase;
@@ -1482,6 +1528,9 @@ async function startBatchImport() {
status: aggregateStatus,
phase: aggregatePhase,
rowsImported: completedRows + nextProgress.rowsImported,
+ rowsFailed: completedFailures + (nextProgress.rowsFailed ?? 0),
+ failures: [...completedFailureList, ...(nextProgress.failures ?? [])],
+ failuresTruncated: completedFailuresTruncated || !!nextProgress.failuresTruncated,
totalRows,
totalRowsExact,
bytesRead: completedBytes + (nextProgress.bytesRead ?? 0),
@@ -1492,13 +1541,30 @@ async function startBatchImport() {
);
task.status = "done";
task.rowsImported = summary.rowsImported;
+ task.rowsFailed = summary.rowsFailed ?? 0;
completedRows += summary.rowsImported;
+ completedFailures += summary.rowsFailed ?? 0;
+ completedFailureList = [...completedFailureList, ...(summary.failures ?? [])];
+ completedFailuresTruncated ||= summary.failuresTruncated ?? false;
completedBytes += task.preview.sizeBytes;
store.invalidateMetadataCache(props.prefillConnectionId, props.prefillDatabase || "", targetSchema.value || undefined, task.tableName);
}
refreshImportElapsedClock();
- progress.value = { importId: importId.value, status: "done", phase: "done", rowsImported: completedRows, totalRows: completedRows, totalRowsExact: true, bytesRead: totalBytes, totalBytes, elapsedMs: liveElapsedMs.value };
- toast(t("tableImport.success", { count: completedRows }), 2500);
+ progress.value = {
+ importId: importId.value,
+ status: "done",
+ phase: "done",
+ rowsImported: completedRows,
+ rowsFailed: completedFailures,
+ failures: completedFailureList,
+ failuresTruncated: completedFailuresTruncated,
+ totalRows: completedRows,
+ totalRowsExact: true,
+ bytesRead: totalBytes,
+ totalBytes,
+ elapsedMs: liveElapsedMs.value,
+ };
+ toast(importSummaryMessage(completedRows, completedFailures), 2500);
store.refreshObjectListTreeNode(props.prefillConnectionId, props.prefillDatabase || "", targetSchema.value || undefined).catch((error) => {
console.warn("[DBX][table-import:refresh-created-table-failed]", error);
});
@@ -1517,6 +1583,9 @@ async function startBatchImport() {
totalRows,
elapsedMs: progress.value?.elapsedMs ?? liveElapsedMs.value,
error: message,
+ rowsFailed: progress.value?.rowsFailed ?? completedFailures,
+ failures: progress.value?.failures ?? completedFailureList,
+ failuresTruncated: progress.value?.failuresTruncated ?? completedFailuresTruncated,
};
} finally {
refreshImportElapsedClock();
@@ -1736,6 +1805,7 @@ watch(rawProgressPercent, (percent) => {
<FileText v-else class="h-3.5 w-3.5 shrink-0" />
<span class="truncate">{{ task.tableName }}</span>
<span v-if="!task.preview.rows.length" class="shrink-0 text-muted-foreground">{{ t("tableImport.rows") }}: 0</span>
+ <span v-else-if="task.rowsFailed > 0" class="shrink-0 text-amber-600 dark:text-amber-400">{{ t("tableImport.failedRows", { count: task.rowsFailed }) }}</span>
</button>
</div>
</div>
@@ -2029,6 +2099,14 @@ watch(rawProgressPercent, (percent) => {
</p>
</div>
+ <div class="space-y-1 rounded-md border p-3">
+ <label class="flex items-center gap-2 text-xs">
+ <input v-model="continueOnError" type="checkbox" data-testid="table-import-continue-on-error" class="h-3.5 w-3.5 accent-primary" />
+ {{ t("tableImport.continueOnError") }}
+ </label>
+ <p class="text-[11px] text-muted-foreground">{{ t("tableImport.continueOnErrorHint") }}</p>
+ </div>
+
<div class="flex items-center gap-2">
<Button size="sm" :disabled="!selectedSource || loadingPreview" @click="loadPreview()">
<Loader2 v-if="loadingPreview" class="mr-1.5 h-3.5 w-3.5 animate-spin" />
@@ -2354,6 +2432,9 @@ watch(rawProgressPercent, (percent) => {
<template v-else>{{ progress?.rowsImported ?? 0 }} {{ t("tableImport.rowsImported") }}</template>
· {{ progressPercent }}% ·
{{ t("tableImport.elapsed", { duration: formatTableImportElapsed(displayedElapsedMs) }) }}
+ <template v-if="(progress?.rowsFailed ?? 0) > 0">
+ · <span class="text-amber-600 dark:text-amber-400">{{ t("tableImport.failedRows", { count: progress?.rowsFailed ?? 0 }) }}</span>
+ </template>
</div>
</div>
</div>
@@ -2373,6 +2454,26 @@ watch(rawProgressPercent, (percent) => {
<div v-if="errorMessage || progress?.error" class="rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-xs text-destructive">
{{ errorMessage || progress?.error }}
</div>
+ <div v-if="terminalStatus && importFailures.length" class="rounded-md border border-amber-300 bg-amber-50 dark:border-amber-800 dark:bg-amber-950/20">
+ <div class="flex items-center gap-2 border-b border-amber-200 px-3 py-2 text-xs font-medium text-amber-900 dark:border-amber-800 dark:text-amber-200">
+ <AlertTriangle class="h-3.5 w-3.5 shrink-0" />
+ <span class="flex-1">{{ t("tableImport.failedRowsTitle") }}</span>
+ <Button variant="outline" size="sm" class="h-6 px-2 text-xs" data-testid="table-import-copy-failures" @click="copyFailedRows">
+ {{ t("tableImport.copyFailedRows") }}
+ </Button>
+ </div>
+ <div class="max-h-40 overflow-auto p-2" data-testid="table-import-failures">
+ <div v-for="(failure, index) in importFailures" :key="`${failure.table}-${failure.row}-${index}`" class="flex gap-2 rounded px-1 py-0.5 text-[11px] font-mono">
+ <span class="shrink-0 text-amber-700 dark:text-amber-300">
+ <template v-if="failure.table">{{ t("tableImport.failureTable", { table: failure.table }) }} · </template>{{ t("tableImport.failureRow", { row: failure.row }) }}:
+ </span>
+ <span class="min-w-0 break-all text-foreground">{{ failure.error }}</span>
+ </div>
+ </div>
+ <p v-if="progress?.failuresTruncated" class="px-3 pb-2 text-[11px] text-amber-800 dark:text-amber-300">
+ {{ t("tableImport.failureListTruncated", { count: importFailures.length }) }}
+ </p>
+ </div>
</div>
<div v-if="loadingTarget" class="flex items-center gap-2 text-xs text-muted-foreground">
diff --git a/apps/desktop/src/components/import/__tests__/TableImportDialog.continueOnError.spec.ts b/apps/desktop/src/components/import/__tests__/TableImportDialog.continueOnError.spec.ts
new file mode 100644
index 0000000000..a1b0ba22a9
--- /dev/null
+++ b/apps/desktop/src/components/import/__tests__/TableImportDialog.continueOnError.spec.ts
@@ -0,0 +1,281 @@
+// @vitest-environment happy-dom
+
+import { createApp, defineComponent, h, nextTick, type App } from "vue";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import i18n from "@/i18n";
+import type { TableImportProgress, TableImportRequest, TableImportSummary } from "@/lib/backend/api";
+
+const mocks = vi.hoisted(() => ({
+ ensureConnected: vi.fn().mockResolvedValue(undefined),
+ listTables: vi.fn().mockResolvedValue([]),
+ getColumns: vi.fn().mockResolvedValue([
+ { name: "id", data_type: "INTEGER", nullable: false },
+ { name: "name", data_type: "TEXT", nullable: true },
+ ]),
+ listDataTypes: vi.fn().mockResolvedValue(["INTEGER", "TEXT"]),
+ previewTableImportFile: vi.fn(),
+ importTableFile: vi.fn<(request: TableImportRequest, onProgress: (progress: TableImportProgress) => void) => Promise<TableImportSummary>>(),
+ releaseTableImportSource: vi.fn().mockResolvedValue(undefined),
+ toast: vi.fn(),
+}));
+
+vi.mock("@/stores/connectionStore", () => ({
+ useConnectionStore: () => ({
+ getConfig: (id: string) => ({ id, name: "SQLite", db_type: "sqlite" }),
+ ensureConnected: mocks.ensureConnected,
+ invalidateMetadataCache: vi.fn(),
+ refreshObjectListTreeNode: vi.fn().mockResolvedValue(undefined),
+ }),
+}));
+
+vi.mock("@/stores/settingsStore", () => ({
+ useSettingsStore: () => ({ editorSettings: {} }),
+}));
+
+vi.mock("@/lib/backend/api", () => ({
+ listTables: mocks.listTables,
+ getColumns: mocks.getColumns,
+ listDataTypes: mocks.listDataTypes,
+ previewTableImportFile: mocks.previewTableImportFile,
+ releaseTableImportSource: mocks.releaseTableImportSource,
+ importTableFile: mocks.importTableFile,
+ cancelTableImport: vi.fn(),
+}));
+
+vi.mock("@/lib/backend/tauriRuntime", () => ({ isTauriRuntime: () => false }));
+vi.mock("@/composables/useToast", () => ({ useToast: () => ({ toast: mocks.toast }) }));
+
+vi.mock("@/components/ui/dialog", async () => {
+ const { defineComponent, h } = await import("vue");
+ const passthrough = defineComponent({
+ setup(_props, { attrs, slots }) {
+ return () => h("div", attrs, slots.default?.());
+ },
+ });
+ return {
+ Dialog: passthrough,
+ DialogHeader: passthrough,
+ DialogTitle: passthrough,
+ DialogFooter: passthrough,
+ DialogScrollContent: passthrough,
+ };
+});
+
+vi.mock("@/components/ui/button", async () => {
+ const { defineComponent, h } = await import("vue");
+ return {
+ Button: defineComponent({
+ props: { disabled: Boolean },
+ emits: ["click"],
+ setup(props, { attrs, slots, emit }) {
+ return () => h("button", { ...attrs, disabled: props.disabled, onClick: () => emit("click") }, slots.default?.());
+ },
+ }),
+ };
+});
+
+vi.mock("@/components/ui/input", async () => {
+ const { defineComponent, h } = await import("vue");
+ return {
+ Input: defineComponent({
+ props: { modelValue: [String, Number], disabled: Boolean },
+ emits: ["update:modelValue"],
+ setup(props, { attrs, emit }) {
+ return () =>
+ h("input", {
+ ...attrs,
+ value: props.modelValue,
+ disabled: props.disabled,
+ onInput: (event: Event) => emit("update:modelValue", (event.target as HTMLInputElement).value),
+ });
+ },
+ }),
+ };
+});
+
+vi.mock("@/components/ui/label", async () => {
+ const { defineComponent, h } = await import("vue");
+ return {
+ Label: defineComponent({
+ setup(_props, { slots }) {
+ return () => h("label", slots.default?.());
+ },
+ }),
+ };
+});
+
+vi.mock("@/components/ui/select", async () => {
+ const { defineComponent, h } = await import("vue");
+ const passthrough = defineComponent({
+ setup(_props, { slots }) {
+ return () => h("div", slots.default?.());
+ },
+ });
+ return {
+ Select: passthrough,
+ SelectContent: passthrough,
+ SelectItem: passthrough,
+ SelectTrigger: passthrough,
+ SelectValue: passthrough,
+ };
+});
+
+vi.mock("@/components/ui/searchable-select", async () => {
+ const { defineComponent, h } = await import("vue");
+ return {
+ SearchableSelect: defineComponent({
+ props: { modelValue: { type: String, default: "" }, options: { type: Array<string>, default: () => [] }, disabled: Boolean },
+ emits: ["update:modelValue"],
+ setup(props, { emit }) {
+ return () =>
+ h(
+ "select",
+ {
+ value: props.modelValue,
+ disabled: props.disabled,
+ onChange: (event: Event) => emit("update:modelValue", (event.target as HTMLInputElement).value),
+ },
+ [(props.options as string[]).map((option) => h("option", { value: option }, option))],
+ );
+ },
+ }),
+ };
+});
+
+import TableImportDialog from "@/components/import/TableImportDialog.vue";
+
+const mountedApps: App[] = [];
+
+function delimitedPreview() {
+ return {
+ fileName: "rows.csv",
+ filePath: "/tmp/rows.csv",
+ sourceRef: "delimited-source",
+ fileType: "csv" as const,
+ sizeBytes: 10,
+ columns: ["id", "name"],
+ rows: [[1, "a"]],
+ totalRows: 2,
+ totalRowsExact: true,
+ sourceFingerprint: "rows-csv",
+ };
+}
+
+async function flushAsyncUpdates() {
+ for (let index = 0; index < 4; index += 1) {
+ await nextTick();
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ }
+}
+
+function clickButton(text: string): boolean {
+ const button = [...document.body.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.trim() === text);
+ if (!button) return false;
+ button.click();
+ return true;
+}
+
+async function mountDialog() {
+ const container = document.createElement("div");
+ document.body.append(container);
+ const app = createApp(defineComponent({ setup: () => () => h(TableImportDialog, { open: true, prefillConnectionId: "connection-1", prefillDatabase: "main" }) }));
+ mountedApps.push(app);
+ app.use(i18n);
+ i18n.global.locale.value = "en";
+ app.mount(container);
+ await flushAsyncUpdates();
+ const input = document.body.querySelector<HTMLInputElement>('input[type="file"]')!;
+ Object.defineProperty(input, "files", { configurable: true, value: [new File(["id,name\n1,a"], "rows.csv")] });
+ input.dispatchEvent(new Event("change", { bubbles: true }));
+ await flushAsyncUpdates();
+}
+
+async function enableContinueOnErrorAndStart() {
+ const findToggle = () => document.body.querySelector<HTMLInputElement>('[data-testid="table-import-continue-on-error"]');
+ let checkbox = findToggle();
+ if (!checkbox) {
+ expect(clickButton("Next"), "options step").toBe(true);
+ await flushAsyncUpdates();
+ checkbox = findToggle();
+ }
+ expect(checkbox, "continue-on-error toggle").toBeTruthy();
+ expect(checkbox!.checked).toBe(false);
+ checkbox!.click();
+ await flushAsyncUpdates();
+ for (let step = 0; step < 5 && !clickButton("Start Import"); step += 1) {
+ expect(clickButton("Next"), `wizard step ${step}`).toBe(true);
+ await flushAsyncUpdates();
+ }
+ await flushAsyncUpdates();
+}
+
+beforeEach(() => {
+ mocks.listTables.mockReset().mockResolvedValue([]);
+ mocks.previewTableImportFile.mockReset().mockImplementation(() => Promise.resolve(delimitedPreview()));
+ mocks.importTableFile.mockReset().mockImplementation(async (request, onProgress) => {
+ onProgress({ importId: request.importId, status: "done", phase: "done", rowsImported: 2, totalRows: 3, elapsedMs: 1, rowsFailed: 0, failures: [], failuresTruncated: false });
+ return { importId: request.importId, rowsImported: 2, totalRows: 3, elapsedMs: 1, rowsFailed: 0, failures: [], failuresTruncated: false };
+ });
+ mocks.releaseTableImportSource.mockReset().mockResolvedValue(undefined);
+ mocks.toast.mockReset();
+});
+
+afterEach(() => {
+ for (const app of mountedApps.splice(0)) app.unmount();
+ document.body.textContent = "";
+ vi.clearAllMocks();
+});
+
+describe("TableImportDialog continue on error (#13683)", () => {
+ it("passes continueOnError to the backend only after the toggle is enabled", async () => {
+ await mountDialog();
+ await enableContinueOnErrorAndStart();
+
+ expect(mocks.importTableFile).toHaveBeenCalledTimes(1);
+ expect(mocks.importTableFile.mock.calls[0]![0].continueOnError).toBe(true);
+ });
+
+ it("lists failed rows after a continue-on-error import finishes", async () => {
+ mocks.importTableFile.mockImplementation(async (request, onProgress) => {
+ const failures = [{ table: "items", row: 2, error: "UNIQUE constraint failed: items.id" }];
+ onProgress({ importId: request.importId, status: "done", phase: "done", rowsImported: 2, totalRows: 3, elapsedMs: 1, rowsFailed: 1, failures, failuresTruncated: false });
+ return { importId: request.importId, rowsImported: 2, totalRows: 3, elapsedMs: 1, rowsFailed: 1, failures, failuresTruncated: false };
+ });
+
+ await mountDialog();
+ await enableContinueOnErrorAndStart();
+
+ const failures = document.body.querySelector('[data-testid="table-import-failures"]');
+ expect(failures, "failed rows list").toBeTruthy();
+ expect(failures!.textContent).toContain("Data row 2");
+ expect(failures!.textContent).toContain("UNIQUE constraint failed: items.id");
+ expect(mocks.toast).toHaveBeenCalledWith(expect.stringContaining("failed"), 2500);
+ });
+
+ it("keeps failed rows visible when a fatal error stops the import", async () => {
+ mocks.importTableFile.mockImplementation(async (request, onProgress) => {
+ const failures = [{ table: "items", row: 3, error: "value too long for column" }];
+ onProgress({
+ importId: request.importId,
+ status: "error",
+ phase: "done",
+ rowsImported: 1,
+ totalRows: 3,
+ elapsedMs: 1,
+ error: "connection lost",
+ rowsFailed: 1,
+ failures,
+ failuresTruncated: false,
+ });
+ throw new Error("connection lost");
+ });
+
+ await mountDialog();
+ await enableContinueOnErrorAndStart();
+
+ const failures = document.body.querySelector('[data-testid="table-import-failures"]');
+ expect(failures, "failed rows list survives the error").toBeTruthy();
+ expect(failures!.textContent).toContain("Data row 3");
+ expect(failures!.textContent).toContain("value too long for column");
+ });
+});
diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts
index 898aa89af6..4907415dd6 100644
--- a/apps/desktop/src/i18n/locales/en.ts
+++ b/apps/desktop/src/i18n/locales/en.ts
@@ -8923,6 +8923,16 @@ export default {
conflictPolicy: "On primary-key conflict",
conflictError: "Stop with an error",
skipDuplicateRows: "Skip duplicate rows",
+ continueOnError: "Continue after failed rows",
+ continueOnErrorHint: "Rows that fail are reported and skipped instead of stopping the import. Retrying a failed batch row by row makes imports with errors slower.",
+ failedRows: "{count} failed",
+ failedRowsTitle: "Failed rows",
+ failureRow: "Data row {row}",
+ failureTable: "Table {table}",
+ failureListTruncated: "Showing the first {count} failed rows.",
+ copyFailedRows: "Copy failed rows",
+ failedRowsCopied: "Failed rows copied",
+ successWithFailures: "Imported {count} rows, {failed} failed",
updateExistingRows: "Update the existing row",
updateExistingRequiresExistingTable: "Updating conflicts requires an existing target table.",
updateExistingUnsupportedDialect: "Updating conflicts is not supported for this database.",
diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts
index 16ac906499..b1ff5687ef 100644
--- a/apps/desktop/src/i18n/locales/es.ts
+++ b/apps/desktop/src/i18n/locales/es.ts
@@ -7922,6 +7922,16 @@ export default withEnglishFallback({
generatedColumns: "Columnas generadas automáticamente",
autoGenerateRequired: "Generar automáticamente columnas obligatorias",
selectTargetColumn: "Seleccionar columna de destino",
+ continueOnError: "Continuar en caso de error",
+ continueOnErrorHint: "Las filas que dan error se reintentan y registran por separado, y el resto de las filas se siguen importando; cuando hay errores, la escritura se hace fila por fila, lo que ralentiza el proceso.",
+ failedRows: "{count} filas fallidas",
+ failedRowsTitle: "Filas fallidas",
+ failureRow: "Fila de datos {row}",
+ failureTable: "Tabla {table}",
+ failureListTruncated: "Solo se muestran los primeros {count} registros fallidos.",
+ copyFailedRows: "Copiar filas fallidas",
+ failedRowsCopied: "Filas fallidas copiadas",
+ successWithFailures: "Se importaron {count} filas; {failed} fallidas",
},
dataGenerate: {
title: "Generación de datos",
diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts
index a02dc4e0f3..0c004ad8e8 100644
--- a/apps/desktop/src/i18n/locales/it.ts
+++ b/apps/desktop/src/i18n/locales/it.ts
@@ -7715,6 +7715,16 @@ export default withEnglishFallback({
generatedColumns: "Colonne generate automaticamente",
autoGenerateRequired: "Genera automaticamente le colonne obbligatorie",
selectTargetColumn: "Seleziona colonna di destinazione",
+ continueOnError: "Continua in caso di errore",
+ continueOnErrorHint: "Le righe che presentano errori vengono ritentate singolarmente e registrate, mentre le altre righe continuano l'importazione; in caso di errore, la scrittura avviene riga per riga e la velocità rallenta.",
+ failedRows: "{count} righe non riuscite",
+ failedRowsTitle: "Righe non riuscite",
+ failureRow: "Riga dati {row}",
+ failureTable: "Tabella {table}",
+ failureListTruncated: "Vengono mostrati solo i primi {count} record non riusciti.",
+ copyFailedRows: "Copia righe non riuscite",
+ failedRowsCopied: "Righe non riuscite copiate",
+ successWithFailures: "Importate {count} righe, {failed} righe non riuscite",
},
dataGenerate: {
title: "Generazione dati",
diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts
index 99bb712d1f..207da44c23 100644
--- a/apps/desktop/src/i18n/locales/ja.ts
+++ b/apps/desktop/src/i18n/locales/ja.ts
@@ -7733,6 +7733,16 @@ export default withEnglishFallback({
generatedColumns: "自動生成列",
autoGenerateRequired: "必須列を自動生成",
selectTargetColumn: "対象列を選択",
+ continueOnError: "失敗時は続行",
+ continueOnErrorHint: "エラーが発生した行は個別に再試行して記録し、残りの行のインポートを続行します。エラー発生時は1行ずつ書き込むため、処理速度が遅くなります。",
+ failedRows: "失敗 {count} 行",
+ failedRowsTitle: "失敗した行",
+ failureRow: "データ行 {row}",
+ failureTable: "テーブル {table}",
+ failureListTruncated: "失敗レコードの先頭 {count} 件のみ表示しています。",
+ copyFailedRows: "失敗した行をコピー",
+ failedRowsCopied: "失敗した行をコピーしました",
+ successWithFailures: "{count} 行をインポートし、{failed} 行が失敗しました",
},
dataGenerate: {
title: "データ生成",
diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts
index db1eddd3e1..80aa06b78d 100644
--- a/apps/desktop/src/i18n/locales/ko.ts
+++ b/apps/desktop/src/i18n/locales/ko.ts
@@ -7747,6 +7747,16 @@ export default withEnglishFallback({
generatedColumns: "자동 생성 열",
autoGenerateRequired: "필수 열 자동 생성",
selectTargetColumn: "대상 열 선택",
+ continueOnError: "실패 시 계속",
+ continueOnErrorHint: "오류가 발생한 행은 별도로 재시도하고 기록하며, 나머지 행은 계속 가져옵니다. 오류 발생 시 행별로 기록하므로 속도가 느려질 수 있습니다.",
+ failedRows: "실패 {count}행",
+ failedRowsTitle: "실패한 행",
+ failureRow: "데이터 행 {row}",
+ failureTable: "테이블 {table}",
+ failureListTruncated: "처음 {count}개의 실패 기록만 표시합니다.",
+ copyFailedRows: "실패한 행 복사",
+ failedRowsCopied: "실패한 행을 복사했습니다",
+ successWithFailures: "{count}개 행을 가져왔고 {failed}개 행이 실패했습니다",
},
dataGenerate: {
title: "데이터 생성",
diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts
index 03dfa0748d..f2f23f3f54 100644
--- a/apps/desktop/src/i18n/locales/pt-BR.ts
+++ b/apps/desktop/src/i18n/locales/pt-BR.ts
@@ -7712,6 +7712,16 @@ export default withEnglishFallback({
generatedColumns: "Colunas geradas automaticamente",
autoGenerateRequired: "Gerar automaticamente colunas obrigatórias",
selectTargetColumn: "Selecionar coluna de destino",
+ continueOnError: "Continuar após falha",
+ continueOnErrorHint: "As linhas com erro são repetidas individualmente e registradas, enquanto as demais linhas continuam a ser importadas; quando ocorrem erros, a gravação é feita linha por linha, o que torna o processo mais lento.",
+ failedRows: "{count} linhas com falha",
+ failedRowsTitle: "Linhas com falha",
+ failureRow: "Linha de dados {row}",
+ failureTable: "Tabela {table}",
+ failureListTruncated: "Exibindo apenas os primeiros {count} registros com falha.",
+ copyFailedRows: "Copiar linhas com falha",
+ failedRowsCopied: "Linhas com falha copiadas",
+ successWithFailures: "{count} linhas importadas, {failed} linhas com falha",
},
dataGenerate: {
title: "Geração de dados",
diff --git a/apps/desktop/src/i18n/locales/ru.ts b/apps/desktop/src/i18n/locales/ru.ts
index 3bf2def919..df909e58b4 100644
--- a/apps/desktop/src/i18n/locales/ru.ts
+++ b/apps/desktop/src/i18n/locales/ru.ts
@@ -9188,6 +9188,16 @@ export default withEnglishFallback({
generatedColumns: "Автоматически генерируемые столбцы",
autoGenerateRequired: "Автоматически генерировать обязательные столбцы",
selectTargetColumn: "Выбрать целевой столбец",
+ continueOnError: "Продолжать при ошибке",
+ continueOnErrorHint: "Строки с ошибками повторяются отдельно и записываются, остальные строки продолжают импортироваться; при ошибках запись выполняется построчно, поэтому скорость снижается.",
+ failedRows: "Неудачных строк: {count}",
+ failedRowsTitle: "Строки с ошибками",
+ failureRow: "Строка данных {row}",
+ failureTable: "Таблица {table}",
+ failureListTruncated: "Показаны только первые {count} записей об ошибках.",
+ copyFailedRows: "Копировать строки с ошибками",
+ failedRowsCopied: "Строки с ошибками скопированы",
+ successWithFailures: "Импортировано строк: {count}, с ошибками: {failed}",
},
dataGenerate: {
title: "Генерация данных",
diff --git a/apps/desktop/src/i18n/locales/th.ts b/apps/desktop/src/i18n/locales/th.ts
index e3261454f5..8ab03402da 100644
--- a/apps/desktop/src/i18n/locales/th.ts
+++ b/apps/desktop/src/i18n/locales/th.ts
@@ -9530,6 +9530,16 @@ export default withEnglishFallback({
generatedColumns: "คอลัมน์ที่สร้างอัตโนมัติ",
autoGenerateRequired: "สร้างคอลัมน์ที่จำเป็นโดยอัตโนมัติ",
selectTargetColumn: "เลือกคอลัมน์เป้าหมาย",
+ continueOnError: "ดำเนินการต่อเมื่อเกิดข้อผิดพลาด",
+ continueOnErrorHint: "แถวที่เกิดข้อผิดพลาดจะถูกนำไปลองใหม่และบันทึกแยกไว้ ส่วนแถวที่เหลือจะนำเข้าต่อไป เมื่อเกิดข้อผิดพลาดจะเขียนทีละแถว ทำให้ช้าลง",
+ failedRows: "ล้มเหลว {count} แถว",
+ failedRowsTitle: "แถวที่ล้มเหลว",
+ failureRow: "แถวข้อมูล {row}",
+ failureTable: "ตาราง {table}",
+ failureListTruncated: "แสดงเฉพาะรายการที่ล้มเหลว {count} รายการแรกเท่านั้น",
+ copyFailedRows: "คัดลอกแถวที่ล้มเหลว",
+ failedRowsCopied: "คัดลอกแถวที่ล้มเหลวแล้ว",
+ successWithFailures: "นำเข้าแล้ว {count} แถว ล้มเหลว {failed} แถว",
},
dataGenerate: {
title: "การสร้างข้อมูล",
diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts
index e5618bd810..476ae25020 100644
--- a/apps/desktop/src/i18n/locales/zh-CN.ts
+++ b/apps/desktop/src/i18n/locales/zh-CN.ts
@@ -8890,6 +8890,16 @@ export default withEnglishFallback({
conflictPolicy: "主键冲突时",
conflictError: "停止并报错",
skipDuplicateRows: "跳过重复行",
+ continueOnError: "失败继续",
+ continueOnErrorHint: "出错的行走单独重试并记录,其余行继续导入;出错时逐行写入,速度会变慢。",
+ failedRows: "失败 {count} 行",
+ failedRowsTitle: "失败行",
+ failureRow: "数据行 {row}",
+ failureTable: "表 {table}",
+ failureListTruncated: "仅显示前 {count} 条失败记录。",
+ copyFailedRows: "复制失败行",
+ failedRowsCopied: "已复制失败行",
+ successWithFailures: "已导入 {count} 行,失败 {failed} 行",
updateExistingRows: "更新已有行",
updateExistingRequiresExistingTable: "冲突更新仅适用于已有目标表。",
updateExistingUnsupportedDialect: "当前数据库不支持冲突更新。",
diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts
index f5bae97424..d3cddbe092 100644
--- a/apps/desktop/src/i18n/locales/zh-TW.ts
+++ b/apps/desktop/src/i18n/locales/zh-TW.ts
@@ -6761,6 +6761,16 @@ export default withEnglishFallback({
conflictPolicy: "主鍵衝突時",
conflictError: "停止並回報錯誤",
skipDuplicateRows: "略過重複列",
+ continueOnError: "失敗繼續",
+ continueOnErrorHint: "出錯的列會單獨重試並記錄,其餘列繼續匯入;出錯時逐列寫入,速度會變慢。",
+ failedRows: "失敗 {count} 列",
+ failedRowsTitle: "失敗列",
+ failureRow: "資料列 {row}",
+ failureTable: "表 {table}",
+ failureListTruncated: "僅顯示前 {count} 筆失敗記錄。",
+ copyFailedRows: "複製失敗列",
+ failedRowsCopied: "已複製失敗列",
+ successWithFailures: "已匯入 {count} 列,失敗 {failed} 列",
updateExistingRows: "更新既有列",
updateExistingRequiresExistingTable: "衝突更新僅適用於既有目標資料表。",
updateExistingUnsupportedDialect: "目前資料庫不支援衝突更新。",
diff --git a/apps/desktop/src/lib/backend/api.ts b/apps/desktop/src/lib/backend/api.ts
index 24a516927b..d4fcc07994 100644
--- a/apps/desktop/src/lib/backend/api.ts
+++ b/apps/desktop/src/lib/backend/api.ts
@@ -1229,6 +1229,7 @@ export type {
TableImportRequest,
TableImportSummary,
TableImportProgress,
+ TableImportFailure,
MongoImportFormat,
MongoImportTypeMode,
MongoImportInferredType,
diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts
index 39ce0b602a..66c00f8f80 100644
--- a/apps/desktop/src/lib/backend/http.ts
+++ b/apps/desktop/src/lib/backend/http.ts
@@ -3449,6 +3449,9 @@ export async function importTableFile(request: TableImportRequest, onProgress: (
rowsImported: progress.rowsImported,
totalRows: progress.totalRows,
elapsedMs: progress.elapsedMs,
+ rowsFailed: progress.rowsFailed ?? 0,
+ failures: progress.failures ?? [],
+ failuresTruncated: progress.failuresTruncated ?? false,
};
es.close();
resolve(summary);
diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts
index d721a2129b..186f60638e 100644
--- a/apps/desktop/src/lib/backend/tauri.ts
+++ b/apps/desktop/src/lib/backend/tauri.ts
@@ -6375,6 +6375,15 @@ export interface TableImportRequest {
retainSource?: boolean;
conflictPolicy?: TableImportConflictPolicy;
skipDuplicateRows?: boolean;
+ continueOnError?: boolean;
+}
+
+export interface TableImportFailure {
+ /** Target table, so a batch import can label each failed row. */
+ table: string;
+ /** 1-based data row number in the source (the header row is not counted). */
+ row: number;
+ error: string;
}
export interface TableImportSummary {
@@ -6382,6 +6391,10 @@ export interface TableImportSummary {
rowsImported: number;
totalRows: number;
elapsedMs: number;
+ // Absent from older web backends, so the dialog treats them as "no failures".
+ rowsFailed?: number;
+ failures?: TableImportFailure[];
+ failuresTruncated?: boolean;
}
export interface TableImportProgress {
@@ -6395,6 +6408,9 @@ export interface TableImportProgress {
totalBytes?: number;
elapsedMs: number;
error?: string | null;
+ rowsFailed?: number;
+ failures?: TableImportFailure[];
+ failuresTruncated?: boolean;
}
export async function previewTableImportFile(filePathOrRequest: string | File | TableImportPreviewRequest, options: Partial<TableImportPreviewRequest> = {}): Promise<TableImportPreview> {
diff --git a/crates/dbx-core/examples/table_import_live_bench.rs b/crates/dbx-core/examples/table_import_live_bench.rs
index 4ef257cfb5..e171ce89b1 100644
--- a/crates/dbx-core/examples/table_import_live_bench.rs
+++ b/crates/dbx-core/examples/table_import_live_bench.rs
@@ -421,6 +421,7 @@ fn import_request(
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
+ continue_on_error: false,
}
}
diff --git a/crates/dbx-core/src/data/csv_export.rs b/crates/dbx-core/src/data/csv_export.rs
index e574aed186..77ea0f7fe6 100644
--- a/crates/dbx-core/src/data/csv_export.rs
+++ b/crates/dbx-core/src/data/csv_export.rs
@@ -553,6 +553,7 @@ mod tests {
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
+ continue_on_error: false,
};
let summary = import_table_file_core(
&state,
diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs
index 0d6ab17e41..d303f1f2f6 100644
--- a/crates/dbx-core/src/data/table_import.rs
+++ b/crates/dbx-core/src/data/table_import.rs
@@ -77,6 +77,10 @@ const POSTGRES_COPY_MAX_ROWS: usize = 50_000;
const SQLSERVER_BULK_ROW_MEMORY_BYTES: usize = 16 * 1024 * 1024;
const SQL_IMPORT_EXPRESSION_KEY: &str = "$dbxSqlImportExpression";
const SQL_IMPORT_CURRENT_TIMESTAMP: &str = "currentTimestamp";
+/// Upper bound on the failed rows reported back to the UI. A pathological import
+/// can fail on every row, and shipping millions of records through one progress
+/// payload would stall the backend without helping the user.
+const IMPORT_FAILURE_RECORD_LIMIT: usize = 1000;
pub fn table_import_client_session_id(import_id: &str) -> String {
task_client_session_id("table-import", import_id)
@@ -387,6 +391,58 @@ pub struct TableImportRequest {
pub conflict_policy: Option<TableImportConflictPolicy>,
#[serde(default)]
pub skip_duplicate_rows: bool,
+ /// Keep importing after a row fails (issue #13683). Off by default so a
+ /// failed statement still aborts the whole import as before. When enabled,
+ /// a failed multi-row batch is retried row by row, and the failed rows are
+ /// reported in the summary/progress instead of silently disappearing.
+ #[serde(default)]
+ pub continue_on_error: bool,
+}
+
+/// One source row that failed to import while `continue_on_error` is enabled.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct TableImportFailure {
+ /// Target table, so a multi-table (batch) import can label each failure.
+ pub table: String,
+ /// 1-based data row number in the source (the header row is not counted).
+ pub row: usize,
+ pub error: String,
+}
+
+/// Collects failed rows for one import. The full list is capped so a broken
+/// source cannot exhaust memory; `failed` still counts every failed row.
+#[derive(Debug)]
+struct ImportFailureSink {
+ table: String,
+ failures: Vec<TableImportFailure>,
+ failed: usize,
+ omitted: usize,
+}
+
+impl ImportFailureSink {
+ fn new(table: &str) -> Self {
+ Self { table: table.to_string(), failures: Vec::new(), failed: 0, omitted: 0 }
+ }
+
+ fn record(&mut self, row: usize, error: impl Into<String>) {
+ self.failed = self.failed.saturating_add(1);
+ if self.failures.len() < IMPORT_FAILURE_RECORD_LIMIT {
+ self.failures.push(TableImportFailure { table: self.table.clone(), row, error: error.into() });
+ } else {
+ self.omitted = self.omitted.saturating_add(1);
+ }
+ }
+
+ fn failures_truncated(&self) -> bool {
+ self.omitted > 0
+ }
+}
+
+impl Default for ImportFailureSink {
+ fn default() -> Self {
+ Self::new("")
+ }
}
impl TableImportRequest {
@@ -503,6 +559,11 @@ pub struct TableImportSummary {
pub rows_imported: usize,
pub total_rows: usize,
pub elapsed_ms: u128,
+ /// Rows skipped because they failed while `continue_on_error` was enabled.
+ pub rows_failed: usize,
+ pub failures: Vec<TableImportFailure>,
+ /// True when `failures` is capped and omits further failed rows.
+ pub failures_truncated: bool,
}
#[derive(Debug, Clone, Serialize)]
@@ -518,6 +579,11 @@ pub struct TableImportProgress {
pub total_bytes: u64,
pub elapsed_ms: u128,
pub error: Option<String>,
+ /// Failed-row details are carried on the terminal event; running events
+ /// keep the counters but leave the list empty.
+ pub rows_failed: usize,
+ pub failures: Vec<TableImportFailure>,
+ pub failures_truncated: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
@@ -5814,6 +5880,17 @@ fn import_error_message(request: &TableImportRequest, rows_imported: usize, erro
format!("Import into table '{}' failed after {} imported rows: {}", request.table, rows_imported, error.as_ref())
}
+/// A standalone truncate (continue-on-error, or an engine whose truncate cannot join the
+/// chunk transaction) commits before any row is written, so a later fatal error cannot roll
+/// it back and the target table stays emptied. The terminal error must say so explicitly.
+fn append_truncate_note(message: String, table_truncated: bool) -> String {
+ if table_truncated {
+ format!("{message}; the table was already truncated before the failure")
+ } else {
+ message
+ }
+}
+
fn import_progress(
import_id: &str,
status: TableImportStatus,
@@ -5853,15 +5930,40 @@ fn import_progress_with_details(
total_bytes,
elapsed_ms: started_at.elapsed().as_millis(),
error,
+ rows_failed: 0,
+ failures: Vec::new(),
+ failures_truncated: false,
}
}
+/// Attaches the accumulated failed-row report to a terminal progress event.
+fn with_import_failures(mut progress: TableImportProgress, sink: &ImportFailureSink) -> TableImportProgress {
+ progress.rows_failed = sink.failed;
+ progress.failures = sink.failures.clone();
+ progress.failures_truncated = sink.failures_truncated();
+ progress
+}
+
+#[cfg(test)]
fn import_summary(import_id: &str, rows_imported: usize, total_rows: usize, started_at: Instant) -> TableImportSummary {
+ import_summary_with_failures(import_id, rows_imported, total_rows, started_at, &ImportFailureSink::default())
+}
+
+fn import_summary_with_failures(
+ import_id: &str,
+ rows_imported: usize,
+ total_rows: usize,
+ started_at: Instant,
+ sink: &ImportFailureSink,
+) -> TableImportSummary {
TableImportSummary {
import_id: import_id.to_string(),
rows_imported,
total_rows,
elapsed_ms: started_at.elapsed().as_millis(),
+ rows_failed: sink.failed,
+ failures: sink.failures.clone(),
+ failures_truncated: sink.failures_truncated(),
}
}
@@ -6283,8 +6385,13 @@ impl SqliteAppendTransaction {
fn sqlite_append_transaction_for_import(
mode: &TableImportMode,
db_type: &DatabaseType,
+ continue_on_error: bool,
) -> Option<SqliteAppendTransaction> {
- (matches!(mode, TableImportMode::Append) && *db_type == DatabaseType::Sqlite).then(SqliteAppendTransaction::new)
+ // The buffered transaction commits a whole chunk at once, which turns one
+ // bad row into a failure for the entire chunk. Continue-on-error must keep
+ // per-row attribution, so it writes through the ordinary statement path.
+ (!continue_on_error && matches!(mode, TableImportMode::Append) && *db_type == DatabaseType::Sqlite)
+ .then(SqliteAppendTransaction::new)
}
fn supports_transactional_import_truncate(db_type: &DatabaseType) -> bool {
@@ -6472,11 +6579,16 @@ async fn execute_import_rows_batch(
date_time_format: Option<&str>,
hard_sql_bytes: Option<usize>,
start_row_offset: usize,
+ continue_on_error: bool,
+ failures: &mut ImportFailureSink,
db_write_ms: &mut u128,
statement_count: &mut usize,
) -> Result<usize, ImportRowsBatchError> {
let execution_policy = import_batch_execution_policy(mode, pending_truncate, db_type);
- if conflict_policy == TableImportConflictPolicy::Error {
+ // Both fast paths write a whole chunk through one backend operation, so a
+ // single bad row fails the batch with no way to tell which row it was.
+ // Continue-on-error has to fall back to the portable INSERT generator.
+ if !continue_on_error && conflict_policy == TableImportConflictPolicy::Error {
if let Some((import_plan, bulk_plan)) = sqlserver_bulk_plans_for_rows(db_type, plan, sqlserver_bulk_plan, rows)
{
return execute_sqlserver_bulk_rows_batch(
@@ -6498,7 +6610,8 @@ async fn execute_import_rows_batch(
}
// COPY is used only for plain scalar PostgreSQL rows and ordinary tables. Any unsupported
// value or table feature falls through to the portable INSERT generator below.
- if conflict_policy == TableImportConflictPolicy::Error
+ if !continue_on_error
+ && conflict_policy == TableImportConflictPolicy::Error
&& execution_policy.allow_postgres_copy
&& *db_type == DatabaseType::Postgres
&& !rows
@@ -6551,77 +6664,208 @@ async fn execute_import_rows_batch(
start_row_offset,
)
.map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
- if let Some(transaction) = sqlite_append_transaction.as_mut() {
- for batch in batches {
+ if !continue_on_error {
+ if let Some(transaction) = sqlite_append_transaction.as_mut() {
+ for batch in batches {
+ ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
+ if transaction.should_flush_before(&batch) {
+ rows_imported = flush_sqlite_append_transaction(
+ state,
+ pool_key,
+ import_id,
+ is_cancelled,
+ connection_id,
+ database,
+ schema,
+ transaction,
+ rows_imported,
+ db_write_ms,
+ statement_count,
+ )
+ .await?;
+ }
+ transaction.push(batch);
+ if transaction.is_ready() {
+ rows_imported = flush_sqlite_append_transaction(
+ state,
+ pool_key,
+ import_id,
+ is_cancelled,
+ connection_id,
+ database,
+ schema,
+ transaction,
+ rows_imported,
+ db_write_ms,
+ statement_count,
+ )
+ .await?;
+ }
+ }
+ return Ok(rows_imported);
+ }
+ if execution_policy.transactional {
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
- if transaction.should_flush_before(&batch) {
- rows_imported = flush_sqlite_append_transaction(
- state,
- pool_key,
- import_id,
- is_cancelled,
- connection_id,
- database,
- schema,
- transaction,
- rows_imported,
- db_write_ms,
- statement_count,
- )
- .await?;
- }
- transaction.push(batch);
- if transaction.is_ready() {
- rows_imported = flush_sqlite_append_transaction(
- state,
- pool_key,
- import_id,
- is_cancelled,
- connection_id,
- database,
- schema,
- transaction,
- rows_imported,
- db_write_ms,
- statement_count,
- )
- .await?;
+ let mut statements = Vec::with_capacity(batches.len() + usize::from(execution_policy.include_truncate));
+ if execution_policy.include_truncate {
+ statements.push(truncate_sql(table, schema, db_type));
}
+ statements.extend(batches.into_iter().map(|batch| batch.sql));
+ let result = execute_import_transaction(
+ state,
+ pool_key,
+ connection_id,
+ database,
+ schema,
+ &statements,
+ db_write_ms,
+ statement_count,
+ )
+ .await
+ .map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
+ return Ok(rows_imported.saturating_add(completed_import_rows(
+ conflict_policy,
+ rows.len(),
+ result.affected_rows,
+ )));
+ }
+ for batch in batches {
+ ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
+ let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count)
+ .await
+ .map_err(|error| ImportRowsBatchError::with_rows_imported(rows_imported, error))?;
+ rows_imported = rows_imported.saturating_add(completed_import_rows(
+ conflict_policy,
+ batch.row_count,
+ result.affected_rows,
+ ));
}
return Ok(rows_imported);
}
- if execution_policy.transactional {
+ // Continue-on-error runs one statement per batch and retries a failed batch
+ // row by row. It deliberately avoids the chunk-wide transaction above: a
+ // rollback there would also undo batches that already succeeded, and
+ // replaying those rows would insert them twice.
+ let mut batch_start = 0usize;
+ for batch in batches {
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
- let mut statements = Vec::with_capacity(batches.len() + usize::from(execution_policy.include_truncate));
- if execution_policy.include_truncate {
- statements.push(truncate_sql(table, schema, db_type));
+ let batch_end = (batch_start + batch.row_count).min(rows.len());
+ let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count).await;
+ match result {
+ Ok(result) => {
+ rows_imported = rows_imported.saturating_add(completed_import_rows(
+ conflict_policy,
+ batch.row_count,
+ result.affected_rows,
+ ));
+ }
+ Err(_) => {
+ // Keep the batch error private: each retried row gets the
+ // backend's own per-row message, which is what the user needs.
+ rows_imported = rows_imported.saturating_add(
+ import_failed_batch_rows(
+ state,
+ pool_key,
+ import_id,
+ is_cancelled,
+ &rows[batch_start..batch_end],
+ plan,
+ columns,
+ mappings,
+ target_column_types,
+ table,
+ schema,
+ db_type,
+ kingbase_oracle_mode,
+ identifier_quote,
+ conflict_policy,
+ primary_key_columns,
+ date_time_format,
+ hard_sql_bytes,
+ start_row_offset + batch_start,
+ failures,
+ db_write_ms,
+ statement_count,
+ )
+ .await?,
+ );
+ }
}
- statements.extend(batches.into_iter().map(|batch| batch.sql));
- let result = execute_import_transaction(
- state,
- pool_key,
- connection_id,
- database,
- schema,
- &statements,
- db_write_ms,
- statement_count,
- )
- .await
- .map_err(|message| ImportRowsBatchError::with_rows_imported(rows_imported, message))?;
- return Ok(rows_imported.saturating_add(completed_import_rows(
- conflict_policy,
- rows.len(),
- result.affected_rows,
- )));
+ batch_start = batch_end;
}
- for batch in batches {
+ Ok(rows_imported)
+}
+
+/// Retries the rows of a failed batch one statement at a time, recording each
+/// row that still fails instead of aborting the import.
+#[allow(clippy::too_many_arguments)]
+async fn import_failed_batch_rows(
+ state: &AppState,
+ pool_key: &str,
+ import_id: &str,
+ is_cancelled: &impl Fn(&str) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send + '_>>,
+ rows: &[Vec<serde_json::Value>],
+ plan: Option<&CompiledImportPlan>,
+ columns: &[String],
+ mappings: &[TableImportColumnMapping],
+ target_column_types: &[(String, String)],
+ table: &str,
+ schema: &str,
+ db_type: &DatabaseType,
+ kingbase_oracle_mode: bool,
+ identifier_quote: Option<&str>,
+ conflict_policy: TableImportConflictPolicy,
+ primary_key_columns: &[String],
+ date_time_format: Option<&str>,
+ hard_sql_bytes: Option<usize>,
+ start_row_offset: usize,
+ failures: &mut ImportFailureSink,
+ db_write_ms: &mut u128,
+ statement_count: &mut usize,
+) -> Result<usize, ImportRowsBatchError> {
+ let mut rows_imported = 0usize;
+ for (local_index, row) in rows.iter().enumerate() {
ensure_import_write_allowed(import_id, is_cancelled, rows_imported).await?;
- let result = execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count)
- .await
- .map_err(|error| ImportRowsBatchError::with_rows_imported(rows_imported, error))?;
- rows_imported =
- rows_imported.saturating_add(completed_import_rows(conflict_policy, batch.row_count, result.affected_rows));
+ let source_row = start_row_offset.saturating_add(local_index).saturating_add(1);
+ let row_start_offset = start_row_offset.saturating_add(local_index);
+ let row_batches = match build_import_execution_batches(
+ std::slice::from_ref(row),
+ plan,
+ columns,
+ mappings,
+ target_column_types,
+ table,
+ schema,
+ db_type,
+ kingbase_oracle_mode,
+ identifier_quote,
+ conflict_policy,
+ primary_key_columns,
+ date_time_format,
+ hard_sql_bytes,
+ row_start_offset,
+ ) {
+ Ok(batches) => batches,
+ Err(error) => {
+ failures.record(source_row, error);
+ continue;
+ }
+ };
+ for batch in row_batches {
+ match execute_import_statement(state, pool_key, &batch.sql, db_write_ms, statement_count).await {
+ Ok(result) => {
+ rows_imported = rows_imported.saturating_add(completed_import_rows(
+ conflict_policy,
+ batch.row_count,
+ result.affected_rows,
+ ));
+ }
+ Err(error) => {
+ failures.record(source_row, error);
+ break;
+ }
+ }
+ }
}
Ok(rows_imported)
}
@@ -6685,11 +6929,89 @@ fn emit_import_error_with_total_rows_exact<F>(
started_at: Instant,
error: impl AsRef<str>,
) -> String
+where
+ F: FnMut(TableImportProgress),
+{
+ emit_import_error_and_sink(
+ progress_callback,
+ request,
+ rows_imported,
+ total_rows,
+ total_rows_exact,
+ started_at,
+ error,
+ None,
+ )
+}
+
+/// Error terminal for an import that may already have skipped failed rows.
+#[allow(clippy::too_many_arguments)]
+fn emit_import_error_with_failures<F>(
+ progress_callback: &mut F,
+ request: &TableImportRequest,
+ rows_imported: usize,
+ total_rows: usize,
+ started_at: Instant,
+ error: impl AsRef<str>,
+ failures: &ImportFailureSink,
+) -> String
+where
+ F: FnMut(TableImportProgress),
+{
+ emit_import_error_and_sink(
+ progress_callback,
+ request,
+ rows_imported,
+ total_rows,
+ true,
+ started_at,
+ error,
+ Some(failures),
+ )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn emit_import_error_with_total_rows_exact_with_failures<F>(
+ progress_callback: &mut F,
+ request: &TableImportRequest,
+ rows_imported: usize,
+ total_rows: usize,
+ total_rows_exact: bool,
+ started_at: Instant,
+ error: impl AsRef<str>,
+ failures: &ImportFailureSink,
+) -> String
+where
+ F: FnMut(TableImportProgress),
+{
+ emit_import_error_and_sink(
+ progress_callback,
+ request,
+ rows_imported,
+ total_rows,
+ total_rows_exact,
+ started_at,
+ error,
+ Some(failures),
+ )
+}
+
+#[allow(clippy::too_many_arguments)]
+fn emit_import_error_and_sink<F>(
+ progress_callback: &mut F,
+ request: &TableImportRequest,
+ rows_imported: usize,
+ total_rows: usize,
+ total_rows_exact: bool,
+ started_at: Instant,
+ error: impl AsRef<str>,
+ failures: Option<&ImportFailureSink>,
+) -> String
where
F: FnMut(TableImportProgress),
{
let message = import_error_message(request, rows_imported, error);
- progress_callback(import_progress_with_details(
+ let progress = import_progress_with_details(
&request.import_id,
TableImportStatus::Error,
TableImportPhase::Done,
@@ -6700,7 +7022,11 @@ where
0,
started_at,
Some(message.clone()),
- ));
+ );
+ progress_callback(match failures {
+ Some(failures) => with_import_failures(progress, failures),
+ None => progress,
+ });
message
}
@@ -6726,6 +7052,21 @@ fn import_cancelled_progress(
)
}
+/// Cancellation terminal for an import that may already have skipped failed rows.
+fn import_cancelled_progress_with_failures(
+ import_id: &str,
+ rows_imported: usize,
+ total_rows: usize,
+ total_rows_exact: bool,
+ started_at: Instant,
+ failures: &ImportFailureSink,
+) -> TableImportProgress {
+ with_import_failures(
+ import_cancelled_progress(import_id, rows_imported, total_rows, total_rows_exact, started_at),
+ failures,
+ )
+}
+
fn delimited_record_to_row(
record: &csv::StringRecord,
columns_len: usize,
@@ -7830,9 +8171,14 @@ where
&request.table,
&request.schema,
);
- let mut sqlite_append_transaction = sqlite_append_transaction_for_import(&request.mode, db_type);
- let mut pending_truncate =
- matches!(request.mode, TableImportMode::Truncate) && supports_transactional_import_truncate(db_type);
+ let mut sqlite_append_transaction =
+ sqlite_append_transaction_for_import(&request.mode, db_type, request.continue_on_error);
+ // Continue-on-error skips the chunk-wide transaction, so the truncate
+ // must run on its own (below) or it would never execute.
+ let mut pending_truncate = matches!(request.mode, TableImportMode::Truncate)
+ && supports_transactional_import_truncate(db_type)
+ && !request.continue_on_error;
+ let mut table_truncated = false;
if matches!(request.mode, TableImportMode::Truncate) && !pending_truncate {
let sql = truncate_sql(&request.table, &request.schema, db_type);
if let Err(error) =
@@ -7842,8 +8188,11 @@ where
let _ = producer.await;
return Err(emit_import_error(&mut progress_callback, request, 0, total_rows, started_at, error));
}
+ // The truncate committed on its own, so no later failure can roll it back.
+ table_truncated = true;
}
let mut rows_imported = 0usize;
+ let mut import_failures = ImportFailureSink::new(&request.table);
let mut sequence_offset = 0usize;
let mut last_bytes_read = 0u64;
let mut last_progress_emit = Instant::now();
@@ -7859,17 +8208,20 @@ where
if is_cancelled(&request.import_id).await {
drop(receiver);
let _ = producer.await;
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- total_rows,
- total_rows_exact,
- last_bytes_read.min(total_bytes),
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ total_rows,
+ total_rows_exact,
+ last_bytes_read.min(total_bytes),
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err("Import cancelled".to_string());
}
@@ -7900,6 +8252,8 @@ where
request.date_time_format.as_deref(),
import_sql_hard_limit,
sequence_offset,
+ request.continue_on_error,
+ &mut import_failures,
&mut db_write_ms,
&mut statement_count,
)
@@ -7914,27 +8268,31 @@ where
let _ = producer.await;
rows_imported = rows_imported.saturating_add(error.rows_imported);
if error.cancelled {
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- total_rows,
- total_rows_exact,
- last_bytes_read.min(total_bytes),
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ total_rows,
+ total_rows_exact,
+ last_bytes_read.min(total_bytes),
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err(error.message);
}
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
@@ -7963,13 +8321,14 @@ where
Err(error) => {
drop(receiver);
let _ = producer.await;
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
started_at,
- error,
+ append_truncate_note(error, table_truncated),
+ &import_failures,
));
}
}
@@ -7977,23 +8336,25 @@ where
match producer.await {
Ok(Ok(())) => {}
Ok(Err(error)) => {
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
started_at,
- error,
+ append_truncate_note(error, table_truncated),
+ &import_failures,
));
}
Err(error) => {
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
started_at,
- error.to_string(),
+ append_truncate_note(error.to_string(), table_truncated),
+ &import_failures,
));
}
}
@@ -8024,28 +8385,32 @@ where
{
Ok(rows) => rows,
Err(error) if error.cancelled => {
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- total_rows,
- total_rows_exact,
- last_bytes_read.min(total_bytes),
- total_bytes,
- started_at,
- None,
- ));
- return Err(error.message);
- }
- Err(error) => {
- return Err(emit_import_error(
- &mut progress_callback,
- request,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ total_rows,
+ total_rows_exact,
+ last_bytes_read.min(total_bytes),
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
+ ));
+ return Err(error.message);
+ }
+ Err(error) => {
+ return Err(emit_import_error_with_failures(
+ &mut progress_callback,
+ request,
rows_imported,
total_rows,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
@@ -8054,21 +8419,30 @@ where
rows_imported = rows_imported.min(known_total_rows);
}
- progress_callback(import_progress_with_details(
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Done,
+ TableImportPhase::Done,
+ rows_imported,
+ rows_imported,
+ true,
+ total_bytes,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
+ ));
+ log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
+
+ return Ok(import_summary_with_failures(
&request.import_id,
- TableImportStatus::Done,
- TableImportPhase::Done,
rows_imported,
rows_imported,
- true,
- total_bytes,
- total_bytes,
started_at,
- None,
+ &import_failures,
));
- log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
-
- return Ok(import_summary(&request.import_id, rows_imported, rows_imported, started_at));
}
if source_format == TableImportSourceFormat::Excel && excel_path_streams_incrementally(&request.file_path) {
@@ -8296,9 +8670,14 @@ where
&request.table,
&request.schema,
);
- let mut sqlite_append_transaction = sqlite_append_transaction_for_import(&request.mode, db_type);
- let mut pending_truncate =
- matches!(request.mode, TableImportMode::Truncate) && supports_transactional_import_truncate(db_type);
+ let mut sqlite_append_transaction =
+ sqlite_append_transaction_for_import(&request.mode, db_type, request.continue_on_error);
+ // Continue-on-error skips the chunk-wide transaction, so the truncate
+ // must run on its own (below) or it would never execute.
+ let mut pending_truncate = matches!(request.mode, TableImportMode::Truncate)
+ && supports_transactional_import_truncate(db_type)
+ && !request.continue_on_error;
+ let mut table_truncated = false;
if matches!(request.mode, TableImportMode::Truncate) && !pending_truncate {
let sql = truncate_sql(&request.table, &request.schema, db_type);
if let Err(error) =
@@ -8309,8 +8688,11 @@ where
let _ = producer.await;
return Err(emit_import_error(&mut progress_callback, request, 0, 0, started_at, error));
}
+ // The truncate committed on its own, so no later failure can roll it back.
+ table_truncated = true;
}
let mut rows_imported = 0usize;
+ let mut import_failures = ImportFailureSink::new(&request.table);
let mut sequence_offset = 0usize;
loop {
let message = match receive_xlsx_stream_message(
@@ -8326,17 +8708,20 @@ where
Err(()) => {
drop(receiver);
let _ = producer.await;
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- 0,
- false,
- last_xlsx_read_bytes,
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ 0,
+ false,
+ last_xlsx_read_bytes,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err("Import cancelled".to_string());
}
@@ -8348,17 +8733,20 @@ where
producer_cancelled.store(true, Ordering::Release);
drop(receiver);
let _ = producer.await;
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- 0,
- false,
- last_xlsx_read_bytes,
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ 0,
+ false,
+ last_xlsx_read_bytes,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err("Import cancelled".to_string());
}
@@ -8389,6 +8777,8 @@ where
request.date_time_format.as_deref(),
import_sql_hard_limit,
sequence_offset,
+ request.continue_on_error,
+ &mut import_failures,
&mut db_write_ms,
&mut statement_count,
)
@@ -8404,27 +8794,31 @@ where
let _ = producer.await;
rows_imported = rows_imported.saturating_add(error.rows_imported);
if error.cancelled {
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- 0,
- false,
- 0,
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ 0,
+ false,
+ 0,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err(error.message);
}
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
0,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
@@ -8464,13 +8858,14 @@ where
producer_cancelled.store(true, Ordering::Release);
drop(receiver);
let _ = producer.await;
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
0,
started_at,
- error,
+ append_truncate_note(error, table_truncated),
+ &import_failures,
));
}
}
@@ -8478,16 +8873,25 @@ where
match producer.await {
Ok(Ok(())) => {}
Ok(Err(error)) => {
- return Err(emit_import_error(&mut progress_callback, request, rows_imported, 0, started_at, error));
+ return Err(emit_import_error_with_failures(
+ &mut progress_callback,
+ request,
+ rows_imported,
+ 0,
+ started_at,
+ append_truncate_note(error, table_truncated),
+ &import_failures,
+ ));
}
Err(error) => {
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
0,
started_at,
- error.to_string(),
+ append_truncate_note(error.to_string(), table_truncated),
+ &import_failures,
));
}
}
@@ -8518,46 +8922,59 @@ where
{
Ok(rows) => rows,
Err(error) if error.cancelled => {
- progress_callback(import_progress_with_details(
- &request.import_id,
- TableImportStatus::Cancelled,
- TableImportPhase::Done,
- rows_imported,
- 0,
- false,
- total_bytes,
- total_bytes,
- started_at,
- None,
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Cancelled,
+ TableImportPhase::Done,
+ rows_imported,
+ 0,
+ false,
+ total_bytes,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
));
return Err(error.message);
}
Err(error) => {
- return Err(emit_import_error(
+ return Err(emit_import_error_with_failures(
&mut progress_callback,
request,
rows_imported,
0,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
rows_imported = rows_imported.saturating_add(flushed_rows);
- progress_callback(import_progress_with_details(
+ progress_callback(with_import_failures(
+ import_progress_with_details(
+ &request.import_id,
+ TableImportStatus::Done,
+ TableImportPhase::Done,
+ rows_imported,
+ rows_imported,
+ true,
+ total_bytes,
+ total_bytes,
+ started_at,
+ None,
+ ),
+ &import_failures,
+ ));
+ log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
+ return Ok(import_summary_with_failures(
&request.import_id,
- TableImportStatus::Done,
- TableImportPhase::Done,
rows_imported,
rows_imported,
- true,
- total_bytes,
- total_bytes,
started_at,
- None,
+ &import_failures,
));
- log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
- return Ok(import_summary(&request.import_id, rows_imported, rows_imported, started_at));
}
let total_bytes = tokio::fs::metadata(&request.file_path).await.map(|metadata| metadata.len()).unwrap_or(0);
@@ -8660,10 +9077,15 @@ where
&request.table,
&request.schema,
);
- let mut sqlite_append_transaction = sqlite_append_transaction_for_import(&request.mode, db_type);
-
- let mut pending_truncate =
- matches!(request.mode, TableImportMode::Truncate) && supports_transactional_import_truncate(db_type);
+ let mut sqlite_append_transaction =
+ sqlite_append_transaction_for_import(&request.mode, db_type, request.continue_on_error);
+
+ // Continue-on-error skips the chunk-wide transaction, so the truncate must
+ // run on its own (below) or it would never execute.
+ let mut pending_truncate = matches!(request.mode, TableImportMode::Truncate)
+ && supports_transactional_import_truncate(db_type)
+ && !request.continue_on_error;
+ let mut table_truncated = false;
if matches!(request.mode, TableImportMode::Truncate) && !pending_truncate {
let sql = truncate_sql(&request.table, &request.schema, db_type);
if let Err(error) =
@@ -8671,33 +9093,38 @@ where
{
return Err(emit_import_error(&mut progress_callback, request, 0, total_rows, started_at, error));
}
+ // The truncate committed on its own, so no later failure can roll it back.
+ table_truncated = true;
}
let mut rows_imported = 0;
+ let mut import_failures = ImportFailureSink::new(&request.table);
let mut sequence_offset = 0usize;
loop {
let rows = match row_source.next_batch(effective_batch_size).await {
Ok(Some(rows)) => rows,
Ok(None) => break,
Err(error) => {
- return Err(emit_import_error_with_total_rows_exact(
+ return Err(emit_import_error_with_total_rows_exact_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
total_rows_exact,
started_at,
- error,
+ append_truncate_note(error, table_truncated),
+ &import_failures,
));
}
};
if is_cancelled(&request.import_id).await {
- progress_callback(import_cancelled_progress(
+ progress_callback(import_cancelled_progress_with_failures(
&request.import_id,
rows_imported,
total_rows,
total_rows_exact,
started_at,
+ &import_failures,
));
return Err("Import cancelled".to_string());
}
@@ -8729,6 +9156,8 @@ where
request.date_time_format.as_deref(),
import_sql_hard_limit,
sequence_offset,
+ request.continue_on_error,
+ &mut import_failures,
&mut db_write_ms,
&mut statement_count,
)
@@ -8741,23 +9170,25 @@ where
Err(error) => {
rows_imported = (rows_imported + error.rows_imported).min(total_rows_cap);
if error.cancelled {
- progress_callback(import_cancelled_progress(
+ progress_callback(import_cancelled_progress_with_failures(
&request.import_id,
rows_imported,
total_rows,
total_rows_exact,
started_at,
+ &import_failures,
));
return Err(error.message);
}
- return Err(emit_import_error_with_total_rows_exact(
+ return Err(emit_import_error_with_total_rows_exact_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
total_rows_exact,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
@@ -8820,24 +9251,26 @@ where
let flushed_rows = match flushed_rows {
Ok(rows) => rows,
Err(error) if error.cancelled => {
- progress_callback(import_cancelled_progress(
+ progress_callback(import_cancelled_progress_with_failures(
&request.import_id,
rows_imported,
total_rows,
total_rows_exact,
started_at,
+ &import_failures,
));
return Err(error.message);
}
Err(error) => {
- return Err(emit_import_error_with_total_rows_exact(
+ return Err(emit_import_error_with_total_rows_exact_with_failures(
&mut progress_callback,
request,
rows_imported,
total_rows,
total_rows_exact,
started_at,
- error.message,
+ append_truncate_note(error.message, table_truncated),
+ &import_failures,
));
}
};
@@ -8845,17 +9278,26 @@ where
// 流式脚本到 EOF 才能确认总行数;写入全部成功时它就是已导入的行数。
let reported_total_rows = if total_rows_exact { total_rows } else { rows_imported };
- progress_callback(import_progress(
+ progress_callback(with_import_failures(
+ import_progress(
+ &request.import_id,
+ TableImportStatus::Done,
+ rows_imported,
+ reported_total_rows,
+ started_at,
+ None,
+ ),
+ &import_failures,
+ ));
+ log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
+
+ Ok(import_summary_with_failures(
&request.import_id,
- TableImportStatus::Done,
rows_imported,
reported_total_rows,
started_at,
- None,
- ));
- log_import_metrics(request, source_format, rows_imported, started_at, db_write_ms, statement_count);
-
- Ok(import_summary(&request.import_id, rows_imported, reported_total_rows, started_at))
+ &import_failures,
+ ))
}
#[cfg(test)]
@@ -8916,6 +9358,51 @@ mod tests {
assert_eq!(progress_json["totalRowsExact"], serde_json::json!(true));
}
+ #[test]
+ fn error_and_cancel_terminals_carry_recorded_failures() {
+ let request: TableImportRequest = serde_json::from_value(serde_json::json!({
+ "importId": "import-1",
+ "connectionId": "connection-1",
+ "database": "db",
+ "schema": "public",
+ "table": "items",
+ "filePath": "items.csv",
+ "mappings": [],
+ "mode": "append",
+ "batchSize": 500
+ }))
+ .unwrap();
+ let mut failures = ImportFailureSink::new("items");
+ failures.record(2, "value too long");
+ let mut events = Vec::new();
+ {
+ let mut callback = |progress: TableImportProgress| events.push(progress);
+ let message =
+ emit_import_error_with_failures(&mut callback, &request, 1, 3, Instant::now(), "fatal", &failures);
+ assert!(message.contains("fatal"));
+ events.push(import_cancelled_progress_with_failures(
+ &request.import_id,
+ 1,
+ 3,
+ true,
+ Instant::now(),
+ &failures,
+ ));
+ }
+
+ assert_eq!(events.len(), 2);
+ assert_eq!(events[0].status, TableImportStatus::Error);
+ assert_eq!(events[1].status, TableImportStatus::Cancelled);
+ for event in events {
+ assert_eq!(event.rows_failed, 1);
+ assert_eq!(event.failures.len(), 1);
+ assert_eq!(event.failures[0].table, "items");
+ assert_eq!(event.failures[0].row, 2);
+ assert_eq!(event.failures[0].error, "value too long");
+ assert!(!event.failures_truncated);
+ }
+ }
+
#[test]
fn table_import_conflict_policy_preserves_legacy_skip_requests() {
let request_json = serde_json::json!({
@@ -9122,6 +9609,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let prepared = validated_prepared_import_source(&request, TableImportSourceFormat::Csv).unwrap();
@@ -11323,6 +11811,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let started_at = Instant::now();
let mut events = vec![import_cancelled_progress(&request.import_id, 19_000, 2_771, false, started_at)];
@@ -11748,6 +12237,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let error = import_table_file_core(
@@ -11841,6 +12331,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let error = import_table_file_core(
@@ -11925,6 +12416,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let error = import_table_file_core(
@@ -14193,6 +14685,7 @@ mod tests {
let checks_for_import = cancellation_checks.clone();
let mut postgres_copy_accumulator = None;
let mut sqlite_append_transaction = None;
+ let mut import_failures = ImportFailureSink::default();
let mut db_write_ms = 0;
let mut statement_count = 0;
@@ -14228,6 +14721,8 @@ mod tests {
// batches and the cancellation check lands between writes.
Some(512 * 1024),
0,
+ false,
+ &mut import_failures,
&mut db_write_ms,
&mut statement_count,
)
@@ -14416,6 +14911,7 @@ mod tests {
async fn append(&mut self, ids: &[i64]) -> Result<usize, ImportRowsBatchError> {
let rows = ids.iter().map(|id| vec![serde_json::json!(id)]).collect::<Vec<_>>();
+ let mut import_failures = ImportFailureSink::default();
execute_import_rows_batch(
&self.state,
&self.pool_key,
@@ -14443,6 +14939,8 @@ mod tests {
None,
None,
0,
+ false,
+ &mut import_failures,
&mut self.db_write_ms,
&mut self.statement_count,
)
@@ -14570,6 +15068,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let summary = import_table_file_core(
@@ -14665,6 +15164,7 @@ mod tests {
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
+ continue_on_error: false,
};
let mut active_during_import = Vec::new();
@@ -14756,6 +15256,7 @@ mod tests {
retain_source: false,
conflict_policy: Some(TableImportConflictPolicy::UpdateExisting),
skip_duplicate_rows: false,
+ continue_on_error: false,
};
let summary = import_table_file_core(
@@ -14784,6 +15285,347 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn sqlite_continue_on_error_skips_failed_rows_and_reports_them() {
+ let dir = tempfile::tempdir().unwrap();
+ let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let connection_id = "sqlite-continue-on-error";
+ let pool_key = format!("{connection_id}:session:import");
+ let database_path = dir.path().join("target.db");
+ let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
+ .await
+ .unwrap();
+ state
+ .update_connection_pools(|connections| {
+ connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
+ })
+ .await;
+ let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+ "id": connection_id,
+ "name": "SQLite continue-on-error test",
+ "db_type": "sqlite",
+ "host": "",
+ "port": 0,
+ "username": "",
+ "password": "",
+ "database": database_path.to_string_lossy()
+ }))
+ .unwrap();
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ let data_path = dir.path().join("rows.csv");
+ // Row 2 duplicates the primary key, so it fails while rows 1 and 3 must land.
+ std::fs::write(&data_path, b"id,name\n1,Ada\n1,Duplicate\n2,Bob\n").unwrap();
+ let request = TableImportRequest {
+ import_id: "sqlite-continue-on-error".to_string(),
+ connection_id: connection_id.to_string(),
+ database: String::new(),
+ schema: String::new(),
+ table: "items".to_string(),
+ file_path: data_path.to_string_lossy().to_string(),
+ source_ref: None,
+ source_format: Some(TableImportSourceFormat::Csv),
+ parse_options: TableImportParseOptions::default(),
+ mappings: ["id", "name"]
+ .into_iter()
+ .map(|column| TableImportColumnMapping {
+ source_column: column.to_string(),
+ target_column: column.to_string(),
+ target_data_type: None,
+ generator: None,
+ })
+ .collect(),
+ mode: TableImportMode::Append,
+ create_table: false,
+ batch_size: 500,
+ date_time_format: None,
+ prepared_source: None,
+ retain_source: false,
+ conflict_policy: Some(TableImportConflictPolicy::Error),
+ skip_duplicate_rows: false,
+ continue_on_error: true,
+ };
+
+ let summary = import_table_file_core(
+ &state,
+ &request,
+ &DatabaseType::Sqlite,
+ &pool_key,
+ |_| Box::pin(async { false }),
+ |_| {},
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(summary.rows_imported, 2);
+ assert_eq!(summary.rows_failed, 1);
+ assert!(!summary.failures_truncated);
+ assert_eq!(summary.failures.len(), 1);
+ assert_eq!(summary.failures[0].row, 2);
+ assert_eq!(summary.failures[0].table, "items");
+ assert!(
+ summary.failures[0].error.to_ascii_lowercase().contains("unique")
+ || summary.failures[0].error.to_ascii_lowercase().contains("constraint"),
+ "unexpected failure message: {}",
+ summary.failures[0].error
+ );
+ let rows = crate::db::sqlite::execute_query(&sqlite, "SELECT id FROM items ORDER BY id").await.unwrap().rows;
+ assert_eq!(rows, vec![vec![serde_json::json!(1)], vec![serde_json::json!(2)]]);
+ }
+
+ #[tokio::test]
+ async fn sqlite_import_still_aborts_on_a_failed_batch_by_default() {
+ let dir = tempfile::tempdir().unwrap();
+ let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let connection_id = "sqlite-stop-on-error";
+ let pool_key = format!("{connection_id}:session:import");
+ let database_path = dir.path().join("target.db");
+ let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
+ .await
+ .unwrap();
+ state
+ .update_connection_pools(|connections| {
+ connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
+ })
+ .await;
+ let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+ "id": connection_id,
+ "name": "SQLite stop-on-error test",
+ "db_type": "sqlite",
+ "host": "",
+ "port": 0,
+ "username": "",
+ "password": "",
+ "database": database_path.to_string_lossy()
+ }))
+ .unwrap();
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ let data_path = dir.path().join("rows.csv");
+ std::fs::write(&data_path, b"id,name\n1,Ada\n1,Duplicate\n2,Bob\n").unwrap();
+ let request = TableImportRequest {
+ import_id: "sqlite-stop-on-error".to_string(),
+ connection_id: connection_id.to_string(),
+ database: String::new(),
+ schema: String::new(),
+ table: "items".to_string(),
+ file_path: data_path.to_string_lossy().to_string(),
+ source_ref: None,
+ source_format: Some(TableImportSourceFormat::Csv),
+ parse_options: TableImportParseOptions::default(),
+ mappings: ["id", "name"]
+ .into_iter()
+ .map(|column| TableImportColumnMapping {
+ source_column: column.to_string(),
+ target_column: column.to_string(),
+ target_data_type: None,
+ generator: None,
+ })
+ .collect(),
+ mode: TableImportMode::Append,
+ create_table: false,
+ batch_size: 500,
+ date_time_format: None,
+ prepared_source: None,
+ retain_source: false,
+ conflict_policy: Some(TableImportConflictPolicy::Error),
+ skip_duplicate_rows: false,
+ continue_on_error: false,
+ };
+
+ let error = import_table_file_core(
+ &state,
+ &request,
+ &DatabaseType::Sqlite,
+ &pool_key,
+ |_| Box::pin(async { false }),
+ |_| {},
+ )
+ .await
+ .unwrap_err();
+
+ assert!(
+ error.to_ascii_lowercase().contains("unique") || error.to_ascii_lowercase().contains("constraint"),
+ "unexpected error: {error}"
+ );
+ let rows = crate::db::sqlite::execute_query(&sqlite, "SELECT COUNT(*) FROM items").await.unwrap().rows;
+ assert_eq!(rows, vec![vec![serde_json::json!(0)]]);
+ }
+
+ #[tokio::test]
+ async fn sqlite_truncate_import_with_continue_on_error_notes_truncated_table_on_fatal_error() {
+ let dir = tempfile::tempdir().unwrap();
+ let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let connection_id = "sqlite-truncate-note-continue";
+ let pool_key = format!("{connection_id}:session:import");
+ let database_path = dir.path().join("target.db");
+ let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
+ .await
+ .unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "INSERT INTO items VALUES (999, 'old')").await.unwrap();
+ state
+ .update_connection_pools(|connections| {
+ connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
+ })
+ .await;
+ let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+ "id": connection_id,
+ "name": "SQLite truncate note continue-on-error test",
+ "db_type": "sqlite",
+ "host": "",
+ "port": 0,
+ "username": "",
+ "password": "",
+ "database": database_path.to_string_lossy()
+ }))
+ .unwrap();
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ // The valid INSERTs are prefetched before the truncate; the trailing unterminated
+ // literal only surfaces at EOF, which is a fatal error even under continue-on-error
+ // (it kills the source stream, not a single row).
+ let data_path = dir.path().join("rows.sql");
+ std::fs::write(
+ &data_path,
+ b"INSERT INTO items (id, name) VALUES (1, 'Ada');\nINSERT INTO items (id, name) VALUES (2, 'Grace');\nINSERT INTO items (id, name) VALUES (3, 'Linus');\nINSERT INTO items (id, name) VALUES (4, 'Broken\n",
+ )
+ .unwrap();
+ let request = TableImportRequest {
+ import_id: "sqlite-truncate-note-continue".to_string(),
+ connection_id: connection_id.to_string(),
+ database: String::new(),
+ schema: String::new(),
+ table: "items".to_string(),
+ file_path: data_path.to_string_lossy().to_string(),
+ source_ref: None,
+ source_format: Some(TableImportSourceFormat::Sql),
+ parse_options: TableImportParseOptions::default(),
+ mappings: ["id", "name"]
+ .into_iter()
+ .map(|column| TableImportColumnMapping {
+ source_column: column.to_string(),
+ target_column: column.to_string(),
+ target_data_type: None,
+ generator: None,
+ })
+ .collect(),
+ mode: TableImportMode::Truncate,
+ create_table: false,
+ batch_size: 2,
+ date_time_format: None,
+ prepared_source: None,
+ retain_source: false,
+ conflict_policy: Some(TableImportConflictPolicy::Error),
+ skip_duplicate_rows: false,
+ continue_on_error: true,
+ };
+
+ let error = import_table_file_core(
+ &state,
+ &request,
+ &DatabaseType::Sqlite,
+ &pool_key,
+ |_| Box::pin(async { false }),
+ |_| {},
+ )
+ .await
+ .unwrap_err();
+
+ assert!(error.contains("Import into table 'items' failed"), "unexpected terminal error: {error}");
+ assert!(error.contains("the table was already truncated"), "terminal error must explain the truncate: {error}");
+ // The standalone truncate cannot be rolled back; only rows delivered before the
+ // fatal error remain, which is exactly why the error message must mention it.
+ // (Row 3 was still buffered in the stream when the EOF error surfaced, so the
+ // stream drops it together with the failure — pre-existing behavior.)
+ let rows = crate::db::sqlite::execute_query(&sqlite, "SELECT id FROM items ORDER BY id").await.unwrap().rows;
+ assert_eq!(rows, vec![vec![serde_json::json!(1)], vec![serde_json::json!(2)]]);
+ }
+
+ #[tokio::test]
+ async fn sqlite_truncate_import_without_continue_on_error_keeps_error_untouched() {
+ let dir = tempfile::tempdir().unwrap();
+ let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
+ let state = AppState::new(storage);
+ let connection_id = "sqlite-truncate-note-default";
+ let pool_key = format!("{connection_id}:session:import");
+ let database_path = dir.path().join("target.db");
+ let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT)")
+ .await
+ .unwrap();
+ crate::db::sqlite::execute_query(&sqlite, "INSERT INTO items VALUES (999, 'old')").await.unwrap();
+ state
+ .update_connection_pools(|connections| {
+ connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
+ })
+ .await;
+ let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
+ "id": connection_id,
+ "name": "SQLite truncate note default test",
+ "db_type": "sqlite",
+ "host": "",
+ "port": 0,
+ "username": "",
+ "password": "",
+ "database": database_path.to_string_lossy()
+ }))
+ .unwrap();
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ // The transactional path rolls the truncate back with the failed batch, so the
+ // pre-existing row survives and the error must not claim the table was truncated.
+ let data_path = dir.path().join("rows.csv");
+ std::fs::write(&data_path, b"id,name\n1,Ada\n1,Duplicate\n2,Bob\n").unwrap();
+ let request = TableImportRequest {
+ import_id: "sqlite-truncate-note-default".to_string(),
+ connection_id: connection_id.to_string(),
+ database: String::new(),
+ schema: String::new(),
+ table: "items".to_string(),
+ file_path: data_path.to_string_lossy().to_string(),
+ source_ref: None,
+ source_format: Some(TableImportSourceFormat::Csv),
+ parse_options: TableImportParseOptions::default(),
+ mappings: ["id", "name"]
+ .into_iter()
+ .map(|column| TableImportColumnMapping {
+ source_column: column.to_string(),
+ target_column: column.to_string(),
+ target_data_type: None,
+ generator: None,
+ })
+ .collect(),
+ mode: TableImportMode::Truncate,
+ create_table: false,
+ batch_size: 500,
+ date_time_format: None,
+ prepared_source: None,
+ retain_source: false,
+ conflict_policy: Some(TableImportConflictPolicy::Error),
+ skip_duplicate_rows: false,
+ continue_on_error: false,
+ };
+
+ let error = import_table_file_core(
+ &state,
+ &request,
+ &DatabaseType::Sqlite,
+ &pool_key,
+ |_| Box::pin(async { false }),
+ |_| {},
+ )
+ .await
+ .unwrap_err();
+
+ assert!(error.contains("Import into table 'items' failed"), "unexpected terminal error: {error}");
+ assert!(!error.contains("truncated"), "error must not mention a truncate that rolled back: {error}");
+ let rows =
+ crate::db::sqlite::execute_query(&sqlite, "SELECT id, name FROM items ORDER BY id").await.unwrap().rows;
+ assert_eq!(rows, vec![vec![serde_json::json!(999), serde_json::json!("old")]]);
+ }
+
#[tokio::test]
async fn preview_missing_source_fails_before_parsing() {
let path = std::env::temp_dir().join(format!("dbx-missing-import-{}.csv", uuid::Uuid::new_v4()));
diff --git a/crates/dbx-core/tests/live_opengauss_table_import.rs b/crates/dbx-core/tests/live_opengauss_table_import.rs
index 0a3d7769ca..84fd26cd8c 100644
--- a/crates/dbx-core/tests/live_opengauss_table_import.rs
+++ b/crates/dbx-core/tests/live_opengauss_table_import.rs
@@ -177,6 +177,7 @@ fn base_request(connection_id: &str, database: &str, file_path: &str, table: &st
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
+ continue_on_error: false,
}
}
diff --git a/crates/dbx-core/tests/live_sqlserver_completion.rs b/crates/dbx-core/tests/live_sqlserver_completion.rs
index 7e6e7d7f36..2078277d66 100644
--- a/crates/dbx-core/tests/live_sqlserver_completion.rs
+++ b/crates/dbx-core/tests/live_sqlserver_completion.rs
@@ -146,6 +146,7 @@ fn live_sqlserver_import_request(
retain_source: false,
conflict_policy: None,
skip_duplicate_rows: false,
+ continue_on_error: false,
}
}
diff --git a/crates/dbx-web/src/routes/table_import.rs b/crates/dbx-web/src/routes/table_import.rs
index b17acdcb3f..155f55c612 100644
--- a/crates/dbx-web/src/routes/table_import.rs
+++ b/crates/dbx-web/src/routes/table_import.rs
@@ -38,6 +38,9 @@ fn initial_import_progress(import_id: &str, started_at: Instant) -> TableImportP
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: None,
+ rows_failed: 0,
+ failures: Vec::new(),
+ failures_truncated: false,
}
}
@@ -53,6 +56,9 @@ fn failed_import_progress(import_id: &str, started_at: Instant, error: String) -
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: Some(error),
+ rows_failed: 0,
+ failures: Vec::new(),
+ failures_truncated: false,
}
}
@@ -558,6 +564,9 @@ mod tests {
total_bytes: 20,
elapsed_ms: 10,
error: None,
+ rows_failed: 0,
+ failures: Vec::new(),
+ failures_truncated: false,
};
send_import_progress(&tx, &terminal);
diff --git a/src-tauri/src/commands/table_import.rs b/src-tauri/src/commands/table_import.rs
index 128a971d5f..2509a28f58 100644
--- a/src-tauri/src/commands/table_import.rs
+++ b/src-tauri/src/commands/table_import.rs
@@ -147,6 +147,9 @@ mod tests {
total_bytes: 100,
elapsed_ms: 25,
error: None,
+ rows_failed: 0,
+ failures: Vec::new(),
+ failures_truncated: false,
}
}