Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(sql-file): flush import batches by buffered bytes for large dumps

zipg29cd60be4fac
diff --git a/crates/dbx-core/src/data/sql_file_import.rs b/crates/dbx-core/src/data/sql_file_import.rs
index 8df34c6297..6ce84870ec 100644
--- a/crates/dbx-core/src/data/sql_file_import.rs
+++ b/crates/dbx-core/src/data/sql_file_import.rs
@@ -88,6 +88,16 @@ struct ControlledSqlFileImportStatement {
pub(crate) const SQL_FILE_READ_CHUNK_BYTES: usize = 256 * 1024;
const SQL_FILE_STATEMENT_BATCH_SIZE: usize = 256;
+/// Upper bound on how much SQL text may be buffered before a batch is executed.
+///
+/// Dumps that use extended inserts (mysqldump default, ~1 MB per statement) put
+/// hundreds of megabytes into a 256-statement batch, so the statement-count bound
+/// alone both spikes memory and delays the first execution: parsing/planning the
+/// buffered text emits no progress at all, which the UI shows as a frozen import
+/// (dbx#10246). Flushing on either bound keeps the buffered batch small and keeps
+/// progress events flowing for large files, while tiny statements still batch up to
+/// the statement-count bound so round trips do not increase.
+const SQL_FILE_STATEMENT_BATCH_MAX_BYTES: usize = 8 * 1024 * 1024;
const SQL_FILE_PREVIEW_ENCODING_SAMPLE_BYTES: usize = 1024 * 1024;
const SQL_FILE_PROGRESS_EMIT_INTERVAL: Duration = Duration::from_millis(100);
@@ -948,7 +958,11 @@ async fn execute_sql_file_paths_inner(
}
}
pending_statements.extend(next_statements);
- if pending_statements.len() < SQL_FILE_STATEMENT_BATCH_SIZE {
+ // Recomputed from the batch itself (which is emptied by
+ // `execute_sql_file_statement_batch`) so no separate counter can
+ // drift out of sync across files or batches.
+ let buffered_bytes = buffered_statement_bytes(&pending_statements);
+ if !sql_file_statement_batch_is_full(pending_statements.len(), buffered_bytes) {
continue;
}
execute_sql_file_statement_batch(
@@ -1883,6 +1897,17 @@ fn split_sql_file_import_statements_with_control(
statements
}
+/// A batch is ready for execution as soon as either the statement-count bound or
+/// the buffered-bytes bound is reached. `statement_bytes` only needs to be a
+/// proxy for the memory the batch pins, so the SQL text length is enough.
+fn sql_file_statement_batch_is_full(statement_count: usize, statement_bytes: usize) -> bool {
+ statement_count >= SQL_FILE_STATEMENT_BATCH_SIZE || statement_bytes >= SQL_FILE_STATEMENT_BATCH_MAX_BYTES
+}
+
+fn buffered_statement_bytes(statements: &[SqlStatementWithControl]) -> usize {
+ statements.iter().map(|statement| statement.sql.len()).sum()
+}
+
fn plan_sql_file_statements(
statements: &[SqlStatementWithControl],
db_type: Option<DatabaseType>,
@@ -2745,6 +2770,69 @@ mod tests {
}
}
+ /// A batch must also be bounded by how many bytes are buffered, not only by how
+ /// many statements it holds: extended-insert dumps (mysqldump default, ~1 MB per
+ /// INSERT) otherwise buffer hundreds of megabytes before the first statement runs,
+ /// and planning that buffer emits no progress at all (dbx#10246).
+ #[test]
+ fn extended_insert_dumps_flush_batches_by_buffered_bytes() {
+ const ROWS_PER_STATEMENT: usize = 1_000;
+ const STATEMENTS: usize = 24;
+ let payload = "x".repeat(900);
+ let mut dump = String::new();
+ for statement_index in 0..STATEMENTS {
+ dump.push_str("INSERT INTO t (id, payload) VALUES ");
+ for row_index in 0..ROWS_PER_STATEMENT {
+ if row_index > 0 {
+ dump.push(',');
+ }
+ dump.push_str(&format!("({},'{payload}')", statement_index * ROWS_PER_STATEMENT + row_index));
+ }
+ dump.push_str(";\n");
+ }
+ assert!(dump.len() > SQL_FILE_STATEMENT_BATCH_MAX_BYTES * 2, "the dump must exceed the byte bound");
+
+ let mut splitter =
+ StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible());
+ let mut pending: Vec<SqlStatementWithControl> = Vec::new();
+ let mut batches: Vec<(usize, usize)> = Vec::new();
+ let mut largest_statement = 0usize;
+ for chunk in dump.as_bytes().chunks(SQL_FILE_READ_CHUNK_BYTES) {
+ let next = splitter.push_chunk(std::str::from_utf8(chunk).unwrap());
+ largest_statement = largest_statement.max(buffered_statement_bytes(&next));
+ pending.extend(next);
+ let bytes = buffered_statement_bytes(&pending);
+ if sql_file_statement_batch_is_full(pending.len(), bytes) {
+ batches.push((pending.len(), bytes));
+ pending.clear();
+ }
+ }
+ pending.extend(splitter.finish());
+ largest_statement = largest_statement.max(buffered_statement_bytes(&pending));
+ batches.push((pending.len(), buffered_statement_bytes(&pending)));
+
+ assert_eq!(
+ batches.iter().map(|(count, _)| count).sum::<usize>(),
+ STATEMENTS,
+ "every statement must still be executed exactly once"
+ );
+ assert!(batches.len() > 1, "an extended-insert dump must not be buffered as a single batch");
+ assert!(batches[0].0 < STATEMENTS, "the first batch must not be the whole file");
+ assert!(largest_statement > 0);
+ assert!(
+ batches.iter().all(|(_, bytes)| *bytes <= SQL_FILE_STATEMENT_BATCH_MAX_BYTES + largest_statement),
+ "a batch may only overshoot the bound by the one statement that crossed it: {batches:?}"
+ );
+ }
+
+ #[test]
+ fn small_statement_dumps_keep_batching_up_to_the_statement_count_bound() {
+ assert!(!sql_file_statement_batch_is_full(SQL_FILE_STATEMENT_BATCH_SIZE - 1, 4 * 1024));
+ assert!(sql_file_statement_batch_is_full(SQL_FILE_STATEMENT_BATCH_SIZE, 4 * 1024));
+ assert!(!sql_file_statement_batch_is_full(1, SQL_FILE_STATEMENT_BATCH_MAX_BYTES - 1));
+ assert!(sql_file_statement_batch_is_full(1, SQL_FILE_STATEMENT_BATCH_MAX_BYTES));
+ }
+
#[test]
fn table_restore_stream_filters_each_statement_once_across_chunks() {
let selected = [