Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(sql-file): preserve statements across split ZIP parts

КГБ Lieu2384a2e2c2a7
diff --git a/crates/dbx-core/src/data/database_export.rs b/crates/dbx-core/src/data/database_export.rs
index 0facacdb0b..4cd3dd33d1 100644
--- a/crates/dbx-core/src/data/database_export.rs
+++ b/crates/dbx-core/src/data/database_export.rs
@@ -15,6 +15,7 @@ use crate::connection::task_client_session_id;
use crate::models::connection::DatabaseType;
use crate::mysql_ddl_normalize::DdlNormalizeOptions;
use crate::object_source_sql::build_export_object_source_sql;
+use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
@@ -166,6 +167,16 @@ enum DatabaseExportWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
+impl DatabaseExportWriter {
+ fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
+ match self {
+ Self::Plain(writer) => writer.write_all(unit),
+ Self::Gzip(writer) => writer.write_all(unit),
+ Self::SplitZip(writer) => writer.write_sql_unit(unit),
+ }
+ }
+}
+
impl Write for DatabaseExportWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter {
}
}
+ fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
+ let mut unit = String::new();
+ std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
+ self.write_sql_unit(unit.as_bytes())
+ }
+
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
}
-// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
-// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
- let mut line = Vec::new();
+ let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres));
+ let mut chunk = String::new();
loop {
- line.clear();
- let bytes_read = source.read_until(b'\n', &mut line)?;
+ chunk.clear();
+ let bytes_read = source.read_line(&mut chunk)?;
if bytes_read == 0 {
break;
}
- destination.write_all(&line)?;
+ for statement in splitter.push_chunk(&chunk) {
+ destination.write_all(format!("{statement};\n").as_bytes())?;
+ }
+ }
+ for statement in splitter.finish() {
+ destination.write_all(format!("{statement};\n").as_bytes())?;
}
Ok(())
}
@@ -6706,6 +6727,75 @@ mod tests {
assert_eq!(combined.matches("INSERT INTO").count(), 20);
}
+ #[test]
+ fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() {
+ // Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE
+ // DDL statement, an INSERT, and a SQL Server-style multi-statement
+ // routine body must each land in exactly one part, never split at an
+ // arbitrary byte offset inside one, even when writeln! formats the
+ // unit from several interpolated arguments (real call sites do this,
+ // e.g. `writeln!(file, "{source}\n")` for exported routine bodies).
+ let directory = tempfile::tempdir().unwrap();
+ let zip_path = directory.path().join("schema.zip");
+ let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
+ &zip_path,
+ crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
+ "schema",
+ "sql",
+ )
+ .unwrap();
+
+ let table_name = "dbo.orders";
+ // part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE
+ // TABLE unit alone exceed that -- it stays whole as the oversized-unit
+ // exception, but the *next* unit must then rotate into a new part.
+ let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::<Vec<_>>().join(", ");
+ let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});");
+ // A SQL Server routine body: multiple statements inside one BEGIN/END
+ // block plus a batch-separator comment, matching what
+ // build_export_object_source_sql emits for a procedure/trigger --
+ // the writer must never cut between the inner statements.
+ let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string();
+ let go_batch_separator = "GO";
+ let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string();
+
+ writeln!(writer, "{create_table_ddl}").unwrap();
+ writeln!(writer, "{procedure_source}").unwrap();
+ writeln!(writer, "{go_batch_separator}").unwrap();
+ writeln!(writer, "{insert}").unwrap();
+ writer.finish("schema.sql").unwrap();
+
+ let file = std::fs::File::open(&zip_path).unwrap();
+ let mut archive = zip::ZipArchive::new(file).unwrap();
+ let mut sql_parts = Vec::new();
+ for index in 0..archive.len() {
+ let mut entry = archive.by_index(index).unwrap();
+ if !entry.name().ends_with(".sql") {
+ continue;
+ }
+ let mut contents = String::new();
+ entry.read_to_string(&mut contents).unwrap();
+ sql_parts.push((entry.name().to_string(), contents));
+ }
+ sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
+
+ assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units");
+
+ let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
+ assert_eq!(
+ combined,
+ format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"),
+ "reassembling every part must reproduce the original stream byte for byte"
+ );
+
+ // Every unit must be found intact inside exactly one part -- proof
+ // that none of them was fragmented across a rotation boundary.
+ for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] {
+ let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count();
+ assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}");
+ }
+ }
+
#[test]
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
// Mirror of the per-schema temporary files that
@@ -6758,7 +6848,7 @@ mod tests {
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
assert!(
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
- "{name} has a malformed line from a mid-statement cut"
+ "{name} has a malformed line from a mid-statement cut: {line:?}"
);
}
}
diff --git a/crates/dbx-core/src/data/sql_file_import.rs b/crates/dbx-core/src/data/sql_file_import.rs
index 6ce84870ec..1ebe70272d 100644
--- a/crates/dbx-core/src/data/sql_file_import.rs
+++ b/crates/dbx-core/src/data/sql_file_import.rs
@@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner(
let mut prev_success_count = 0usize;
let mut prev_failure_count = 0usize;
let mut prev_affected_rows = 0u64;
+ let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
+ let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let import_result = async {
for (file_index, file_path) in file_paths.iter().enumerate() {
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
@@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner(
});
}
- let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
- let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
});
@@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner(
"",
None,
));
- let mut next_statements = splitter.push_chunk(&chunk);
+ let mut next_statements = splitter
+ .as_mut()
+ .expect("splitter is only taken by finish() on the last file, after this loop exits")
+ .push_chunk(&chunk);
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
@@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner(
.await?;
}
- let mut next_statements = splitter.finish();
- if let Some(filter) = restore_filter.as_mut() {
- if let Err(error) = filter_restore_statements(
- &mut next_statements,
- filter,
- request.selected_tables.as_deref().unwrap_or_default(),
- ) {
- emit(sql_file_execution_error_progress(
- &request.execution_id,
- started_at,
- &progress,
- error.clone(),
- ));
- return Err(error);
+ let is_last_file = file_index + 1 == file_count;
+ if is_last_file {
+ let mut next_statements =
+ splitter.take().expect("splitter is only taken once, on the last file").finish();
+ if let Some(filter) = restore_filter.as_mut() {
+ if let Err(error) = filter_restore_statements(
+ &mut next_statements,
+ filter,
+ request.selected_tables.as_deref().unwrap_or_default(),
+ ) {
+ emit(sql_file_execution_error_progress(
+ &request.execution_id,
+ started_at,
+ &progress,
+ error.clone(),
+ ));
+ return Err(error);
+ }
}
+ pending_statements.extend(next_statements);
}
- pending_statements.extend(next_statements);
execute_sql_file_statement_batch(
state,
request,
@@ -3183,6 +3190,20 @@ mod tests {
assert_eq!(statements[1], "SELECT 3;");
}
+ #[test]
+ fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
+ let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
+ let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
+ let expected = format!("{part_one}{part_two}").trim().to_string();
+ let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
+
+ assert!(splitter.push_chunk(part_one).is_empty());
+ assert!(splitter.push_chunk(part_two).is_empty());
+ let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
+
+ assert_eq!(statements, vec![expected]);
+ }
+
#[test]
fn streaming_sqlserver_splitter_handles_go_across_chunks() {
let mut splitter = SqlServerBatchSplitter::default();
diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rs
index 67f1b85040..0cfbe8c4c2 100644
--- a/crates/dbx-core/src/data/table_export.rs
+++ b/crates/dbx-core/src/data/table_export.rs
@@ -1068,6 +1068,15 @@ enum TableExportSqlWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
+impl TableExportSqlWriter {
+ fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
+ match self {
+ Self::Plain(writer) => writer.write_all(unit),
+ Self::SplitZip(writer) => writer.write_sql_unit(unit),
+ }
+ }
+}
+
impl Write for TableExportSqlWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter {
}
}
+ fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
+ let mut unit = String::new();
+ std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
+ self.write_sql_unit(unit.as_bytes())
+ }
+
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -1093,6 +1108,21 @@ impl TableExportSqlWriter {
}
}
+fn write_sql_export_statements(
+ file: &mut impl Write,
+ statements: Vec<String>,
+ wrote_statements: &mut bool,
+) -> Result<(), String> {
+ for statement in statements {
+ if *wrote_statements {
+ file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
+ }
+ file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?;
+ *wrote_statements = true;
+ }
+ Ok(())
+}
+
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
if let Some(max_mb) = request.split_max_mb {
let zip_path = std::path::Path::new(&request.file_path);
@@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream(
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
request.insert_dialect,
)?;
- if !statements.is_empty() {
- if wrote_statements {
- file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
- }
- file.write_all(statements.join("\n").as_bytes())
- .map_err(|e| format!("Failed to write SQL: {e}"))?;
- wrote_statements = true;
- }
+ write_sql_export_statements(file, statements, &mut wrote_statements)?;
Ok(())
};
let result = stream_native_table_rows(
@@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner(
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
request.insert_dialect,
)?;
- if !statements.is_empty() {
- if wrote_statements {
- file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
- }
- file.write_all(statements.join("\n").as_bytes())
- .map_err(|e| format!("Failed to write SQL: {e}"))?;
- wrote_statements = true;
- }
+ write_sql_export_statements(&mut file, statements, &mut wrote_statements)?;
rows_exported += row_count as u64;
if use_keyset {
@@ -2361,6 +2377,39 @@ mod tests {
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
+ #[test]
+ fn split_zip_sql_export_writes_batch_statements_separately() {
+ let dir = tempfile::tempdir().unwrap();
+ let zip_path = dir.path().join("export.zip");
+ let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
+ let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
+ &zip_path,
+ crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
+ "table",
+ "sql",
+ )
+ .unwrap();
+ let mut wrote_statements = false;
+
+ write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements)
+ .unwrap();
+ writer.finish("table.sql").unwrap();
+
+ let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap();
+ let mut sql_parts = Vec::new();
+ for index in 0..archive.len() {
+ let mut entry = archive.by_index(index).unwrap();
+ if entry.name().ends_with(".sql") {
+ let mut contents = String::new();
+ entry.read_to_string(&mut contents).unwrap();
+ sql_parts.push(contents);
+ }
+ }
+
+ assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]);
+ assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024));
+ }
+
#[test]
fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() {
let mut payload = json!({
diff --git a/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs
new file mode 100644
index 0000000000..8f889dc7e5
--- /dev/null
+++ b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs
@@ -0,0 +1,267 @@
+// E2E test executed manually against a real local SQL Server 2025 container
+// (`database-mssql-1`, database `default`) already loaded with a production-shaped
+// dataset (50 tables / 52 foreign keys), as part of task #01a0db6e-2276.
+//
+// Flow: export database `default` (with `split_max_mb` set, i.e. the split-ZIP
+// contract under review) -> verify manifest/parts -> re-import the resulting ZIP
+// via `execute_sql_file_paths` with `skip_relational_constraints = true` ("Skip
+// relationship") into a fresh scratch database, exactly like the desktop app does.
+//
+// Run manually with:
+// DBX_LIVE_SQLSERVER_PASSWORD='MSSQLStrongPassword123!' \
+// cargo test -p dbx-core --test live_sqlserver_split_zip_e2e -- --nocapture --test-threads=1
+
+use dbx_core::connection::AppState;
+use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest};
+use dbx_core::models::connection::DatabaseType;
+use dbx_core::sql::SqlFileRequest;
+use dbx_core::sql_file_import::execute_sql_file_paths;
+use std::sync::Arc;
+use std::time::Instant;
+use tokio_util::sync::CancellationToken;
+
+fn live_sqlserver_config(id: &str, database: &str) -> dbx_core::models::connection::ConnectionConfig {
+ dbx_core::models::connection::ConnectionConfig {
+ docs_notes_path: None,
+ id: id.to_string(),
+ name: id.to_string(),
+ note: String::new(),
+ db_type: DatabaseType::SqlServer,
+ driver_profile: None,
+ driver_label: None,
+ url_params: None,
+ agent_java_options: Vec::new(),
+ host: std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
+ port: std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
+ username: std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
+ password: std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
+ database: Some(database.to_string()),
+ default_schema: None,
+ visible_databases: None,
+ visible_database_patterns: None,
+ visible_schemas: None,
+ attached_databases: Vec::new(),
+ init_script: None,
+ color: None,
+ transport_layers: Vec::new(),
+ connect_timeout_secs: 10,
+ query_timeout_secs: 30,
+ idle_timeout_secs: 60,
+ keepalive_interval_secs: 0,
+ ssl: false,
+ ca_cert_path: String::new(),
+ client_cert_path: String::new(),
+ client_key_path: String::new(),
+ sysdba: false,
+ oracle_connection_type: None,
+ connection_string: None,
+ redis_connection_mode: None,
+ redis_sentinel_master: String::new(),
+ redis_sentinel_nodes: String::new(),
+ redis_sentinel_username: String::new(),
+ redis_sentinel_password: String::new(),
+ redis_sentinel_tls: false,
+ redis_cluster_nodes: String::new(),
+ redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
+ redis_scan_page_size: None,
+ redis_database_aliases: Default::default(),
+ redis_key_templates: Vec::new(),
+ redis_key_grouping: None,
+ etcd_endpoints: String::new(),
+ gbase_server: String::new(),
+ informix_server: String::new(),
+ external_config: None,
+ plugin_id: None,
+ plugin_connection_provider: None,
+ plugin_connection_type: None,
+ connection_secrets: Default::default(),
+ jdbc_driver_class: None,
+ jdbc_driver_paths: Vec::new(),
+ one_time: false,
+ save_password: true,
+ read_only: false,
+ is_production: false,
+ production_databases: vec![],
+ show_system_schemas: false,
+ database_info: None,
+ sidebar_auto_load_all_tables: false,
+ }
+}
+
+async fn live_sqlserver_state(connection_id: &str, database: &str, suffix: &str) -> (AppState, std::path::PathBuf) {
+ let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-splitzip-{suffix}"));
+ std::fs::create_dir_all(&dir).expect("create live test directory");
+ let storage =
+ dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open live test storage");
+ let state = AppState::new(storage);
+ let config = live_sqlserver_config(connection_id, database);
+ state.configs.write().await.insert(connection_id.to_string(), config);
+ (state, dir)
+}
+
+#[tokio::test]
+#[ignore = "manual E2E against a real local SQL Server instance -- see file header"]
+async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relationship() {
+ let export_connection_id = "e2e-mssql-export";
+ let import_connection_id = "e2e-mssql-import";
+ let source_database = "default";
+ // Import target is a separate scratch database, NOT the just-emptied
+ // `default`, per the task's own guidance to isolate the destructive
+ // re-import from the production-shaped database whenever possible.
+ let import_database = "dbx_e2e_split_zip_restore";
+
+ let (export_state, work_dir) = live_sqlserver_state(export_connection_id, source_database, "export").await;
+ let export_state = Arc::new(export_state);
+
+ let zip_path = work_dir.join("default-export.zip");
+ let export_request = DatabaseExportRequest {
+ export_id: format!("e2e-export-{}", uuid::Uuid::new_v4()),
+ connection_id: export_connection_id.to_string(),
+ database: source_database.to_string(),
+ schema: String::new(),
+ file_path: zip_path.to_string_lossy().to_string(),
+ selected_tables: Vec::new(),
+ excluded_tables: Vec::new(),
+ include_structure: true,
+ include_data: true,
+ include_objects: true,
+ include_create_database: false,
+ drop_table_if_exists: true,
+ omit_auto_increment: false,
+ fail_on_error: true,
+ prevent_overwrite: false,
+ output_compression: DatabaseExportOutputCompression::None,
+ snapshot_session_id: None,
+ batch_size: 1000,
+ split_max_mb: Some(1),
+ insert_dialect: Default::default(),
+ };
+
+ let progress_log = std::sync::Mutex::new(Vec::new());
+ let export_result = export_database_sql_core(&export_state, &export_request, |progress| {
+ progress_log.lock().unwrap().push(format!("{progress:?}"));
+ })
+ .await;
+
+ eprintln!("=== EXPORT PROGRESS LOG ===");
+ for line in progress_log.lock().unwrap().iter() {
+ eprintln!("{line}");
+ }
+ eprintln!("=== EXPORT RESULT ===\n{export_result:?}");
+ export_result.expect("split-ZIP database export should succeed");
+
+ assert!(zip_path.exists(), "expected export ZIP at {}", zip_path.display());
+ let zip_metadata = std::fs::metadata(&zip_path).expect("stat export zip");
+ eprintln!("Export ZIP size: {} bytes", zip_metadata.len());
+
+ // --- Verify manifest.json + parts against the dbx-sql-export-parts-v1 contract ---
+ let file = std::fs::File::open(&zip_path).expect("open export zip");
+ let mut archive = zip::ZipArchive::new(file).expect("read export zip as a zip archive");
+ let mut entry_names: Vec<String> =
+ (0..archive.len()).map(|index| archive.by_index(index).expect("zip entry").name().to_string()).collect();
+ entry_names.sort();
+ eprintln!("=== ZIP ENTRIES ===\n{entry_names:#?}");
+ assert!(entry_names.contains(&"manifest.json".to_string()), "manifest.json entry must be present");
+ let sql_part_count = entry_names.iter().filter(|name| name.ends_with(".sql")).count();
+ assert!(sql_part_count >= 1, "at least one .sql part expected");
+
+ let manifest_bytes = {
+ let mut entry = archive.by_name("manifest.json").expect("manifest.json entry");
+ let mut bytes = Vec::new();
+ std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read manifest.json");
+ bytes
+ };
+ let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes).expect("parse manifest.json");
+ eprintln!("=== MANIFEST ===\n{}", serde_json::to_string_pretty(&manifest).unwrap());
+ assert_eq!(manifest["format"], "dbx-sql-export-parts-v1", "manifest format must match the documented contract");
+ let manifest_parts = manifest["parts"].as_array().expect("manifest.parts array").len();
+ assert_eq!(manifest_parts, sql_part_count, "manifest part count must match actual .sql entries in the zip");
+
+ // --- Prepare a clean scratch database on the same SQL Server instance for import ---
+ let (import_state, _import_work_dir) =
+ live_sqlserver_state(import_connection_id, "master", "import-bootstrap").await;
+ let bootstrap_pool = import_state
+ .get_or_create_pool(import_connection_id, Some("master"))
+ .await
+ .expect("connect to master to bootstrap scratch database");
+ let drop_sql = format!(
+ "IF DB_ID(N'{import_database}') IS NOT NULL BEGIN ALTER DATABASE [{import_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{import_database}]; END"
+ );
+ dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &drop_sql, None, None)
+ .await
+ .expect("drop pre-existing scratch database if present");
+ let create_sql = format!("CREATE DATABASE [{import_database}]");
+ dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &create_sql, None, None)
+ .await
+ .expect("create scratch database for import");
+ let _ = bootstrap_pool;
+
+ // --- Extract the ZIP the same way the app does before feeding SQL files to the importer ---
+ let extraction_dir = work_dir.join("extracted-parts");
+ let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&zip_path, &extraction_dir)
+ .expect("extract split-ZIP package using the shared sql_file_zip_package contract");
+ let extracted_paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package);
+ eprintln!("=== EXTRACTED PART PATHS ===\n{extracted_paths:#?}");
+ assert_eq!(extracted_paths.len(), sql_part_count, "extracted part count must match the zip's .sql entries");
+
+ let (reimport_state, _reimport_work_dir) =
+ live_sqlserver_state(import_connection_id, import_database, "import-exec").await;
+ let import_request = SqlFileRequest {
+ execution_id: format!("e2e-import-{}", uuid::Uuid::new_v4()),
+ connection_id: import_connection_id.to_string(),
+ database: import_database.to_string(),
+ file_path: extracted_paths[0].to_string_lossy().to_string(),
+ continue_on_error: false,
+ txn_session_id: None,
+ selected_tables: None,
+ part_cooldown_ms: 0,
+ // This is the "Skip relationship" option from the task: SQL Server
+ // constraint bypass runs `ALTER TABLE ... NOCHECK CONSTRAINT ALL`
+ // before the import and `WITH NOCHECK CHECK CONSTRAINT ALL` after.
+ skip_relational_constraints: true,
+ };
+
+ let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect();
+ let import_progress_log = std::sync::Mutex::new(Vec::new());
+ let started_at = Instant::now();
+ let import_result = execute_sql_file_paths(
+ &reimport_state,
+ &import_request,
+ &file_path_refs,
+ CancellationToken::new(),
+ started_at,
+ |progress| import_progress_log.lock().unwrap().push(format!("{progress:?}")),
+ )
+ .await;
+
+ eprintln!("=== IMPORT PROGRESS LOG ===");
+ for line in import_progress_log.lock().unwrap().iter() {
+ eprintln!("{line}");
+ }
+ eprintln!("=== IMPORT RESULT ===\n{import_result:?}");
+
+ // Intentionally do not assert success here: the whole point of this manual
+ // E2E run is to surface whatever real error the split-ZIP + skip-relationship
+ // path produces (or confirm it succeeds), and report it verbatim, per the
+ // task's explicit instruction not to fabricate outcomes.
+ match &import_result {
+ Ok(()) => eprintln!("IMPORT SUCCEEDED"),
+ Err(error) => eprintln!("IMPORT FAILED WITH ERROR: {error}"),
+ }
+
+ // --- Post-import verification: table/row counts on the scratch database ---
+ let verify_sql = "SELECT COUNT(*) AS tbl_count FROM sys.tables WHERE is_ms_shipped = 0";
+ match dbx_core::query::execute_sql_statement(
+ &reimport_state,
+ import_connection_id,
+ import_database,
+ verify_sql,
+ None,
+ None,
+ )
+ .await
+ {
+ Ok(result) => eprintln!("=== POST-IMPORT TABLE COUNT ===\n{result:?}"),
+ Err(error) => eprintln!("=== POST-IMPORT VERIFY QUERY FAILED ===\n{error}"),
+ }
+}
diff --git a/crates/dbx-formats/src/export_split_zip.rs b/crates/dbx-formats/src/export_split_zip.rs
index 15626b083c..2c73fe1895 100644
--- a/crates/dbx-formats/src/export_split_zip.rs
+++ b/crates/dbx-formats/src/export_split_zip.rs
@@ -122,6 +122,21 @@ impl SplitZipExportWriter {
Ok(())
}
+ /// Writes one complete SQL export unit into a single part. A unit may be
+ /// a DDL statement, routine body, INSERT, comment, or client-side batch
+ /// delimiter such as SQL Server `GO`; it must never be assembled from
+ /// multiple calls. A unit that fits starts a new part before it would
+ /// exceed the configured size; an oversized individual unit remains whole
+ /// as the only unavoidable exception.
+ pub fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
+ if self.current_part_bytes > 0 && self.current_part_bytes + unit.len() as u64 > self.part_max_bytes {
+ self.start_next_part().map_err(std::io::Error::other)?;
+ }
+ self.zip.write_all(unit)?;
+ self.current_part_bytes += unit.len() as u64;
+ Ok(())
+ }
+
/// Writes the trailing `manifest.json` entry (listing every part in
/// order) and finalizes the zip archive. Consumes the writer because the
/// underlying `zip::ZipWriter::finish` does.
@@ -148,12 +163,14 @@ impl SplitZipExportWriter {
impl Write for SplitZipExportWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
- if self.current_part_bytes >= self.part_max_bytes {
- self.start_next_part().map_err(std::io::Error::other)?;
- }
- let written = self.zip.write(buffer)?;
- self.current_part_bytes += written as u64;
- Ok(written)
+ self.write_sql_unit(buffer)?;
+ Ok(buffer.len())
+ }
+
+ fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
+ let mut unit = String::new();
+ std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
+ self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
@@ -240,13 +257,9 @@ mod tests {
{
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
- // Each write() call is one full statement; the threshold check
- // happens before the call, so a small per-write payload combined
- // with a below-1MB threshold still needs enough total bytes to
- // exceed 1MB and trigger a second part. Use the internal minimum
- // by writing repeatedly.
for index in 0..6000 {
- writeln!(writer, "{long_statement} -- row {index}").unwrap();
+ let line = format!("{long_statement} -- row {index}\n");
+ writer.write_all(line.as_bytes()).unwrap();
}
writer.finish("mydb.sql").unwrap();
}
@@ -272,6 +285,109 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
+ #[test]
+ fn rotates_before_a_complete_statement_that_would_exceed_the_part_limit() {
+ let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let zip_path = dir.join("export.zip");
+ let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
+
+ {
+ let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
+ writer.write_all(statement.as_bytes()).unwrap();
+ writer.write_all(b"\n").unwrap();
+ writer.write_all(statement.as_bytes()).unwrap();
+ writer.finish("mydb.sql").unwrap();
+ }
+
+ let entries = read_zip_entries(&zip_path);
+ let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
+ assert_eq!(sql_entries.len(), 2);
+ assert!(sql_entries.iter().all(|(_, contents)| contents.len() <= 1024 * 1024));
+ assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), format!("{statement}\n"));
+ assert_eq!(String::from_utf8(sql_entries[1].1.clone()).unwrap(), statement);
+
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn writeln_with_multiple_format_arguments_stays_one_sql_unit() {
+ // std::fmt's default Write::write_fmt dispatches one write() call per
+ // format-string fragment, which would let a rotation slip in between
+ // "CREATE TABLE " and the column list even though the whole macro
+ // invocation is meant to emit a single DDL unit. Every real export
+ // call site formats a whole statement/DDL/routine body with `writeln!`
+ // (e.g. `writeln!(file, "{source}\n")`, `writeln!(file, "-- Date:
+ // {timestamp}")`), so the writer overrides `write_fmt` to buffer the
+ // formatted text and hand it to `write_sql_unit` in one call.
+ let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let zip_path = dir.join("export.zip");
+ let table_name = "orders";
+ let insert = "INSERT INTO orders VALUES (1);";
+ // part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Size the column
+ // list so the CREATE TABLE unit alone fits (leaving less headroom
+ // than the INSERT needs), forcing the rotation to land *between* the
+ // two units instead of never triggering or landing mid-DDL.
+ let overhead = "CREATE TABLE ".len() + table_name.len() + " (".len() + ");".len() + "\n".len();
+ let columns = "x".repeat(1024 * 1024 - overhead - 10);
+ let create_table = format!("CREATE TABLE {table_name} ({columns});");
+
+ {
+ let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
+ writeln!(writer, "CREATE TABLE {table_name} ({columns});").unwrap();
+ writeln!(writer, "{insert}").unwrap();
+ writer.finish("mydb.sql").unwrap();
+ }
+
+ let entries = read_zip_entries(&zip_path);
+ let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
+ sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
+
+ assert_eq!(sql_entries.len(), 2, "the oversized CREATE TABLE must rotate before the INSERT, not mid-DDL");
+ let first = String::from_utf8(sql_entries[0].1.clone()).unwrap();
+ let second = String::from_utf8(sql_entries[1].1.clone()).unwrap();
+ assert_eq!(first, format!("{create_table}\n"), "the whole CREATE TABLE must land in one part intact");
+ assert_eq!(second, format!("{insert}\n"));
+
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn keeps_an_oversized_individual_statement_whole_as_the_only_limit_exception() {
+ let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let zip_path = dir.join("export.zip");
+
+ let mut seed = 0x1234_5678_u32;
+ let huge_payload = (0..3 * 1024 * 1024)
+ .map(|_| {
+ seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
+ char::from(b'a' + (seed >> 24) as u8 % 26)
+ })
+ .collect::<String>();
+ let statement = format!("INSERT INTO t VALUES ('{huge_payload}');\n");
+
+ {
+ let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
+ writer.write_all(statement.as_bytes()).unwrap();
+ writer.finish("mydb.sql").unwrap();
+ }
+
+ let entries = read_zip_entries(&zip_path);
+ let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
+ sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
+
+ assert_eq!(sql_entries.len(), 1, "a single write must stay in one SQL part");
+ assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), statement);
+
+ let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap();
+ let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap();
+ assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len());
+
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
#[test]
fn sanitizes_zip_entry_stem_from_untrusted_names() {
assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd");