Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

perf(data-compare): page SQL Server and PostgreSQL reads by key

Brydon209172db0707
diff --git a/crates/dbx-core/src/data/data_compare.rs b/crates/dbx-core/src/data/data_compare.rs
index 85bc153995..3c4ce7f7ae 100644
--- a/crates/dbx-core/src/data/data_compare.rs
+++ b/crates/dbx-core/src/data/data_compare.rs
@@ -344,7 +344,7 @@ pub async fn prepare_data_compare_from_tables(
let target_database_type = connection_database_type(state, &options.target_connection_id).await?;
ensure_data_compare_supported(source_database_type, "source")?;
ensure_data_compare_supported(target_database_type, "target")?;
- let fetch_batch_size = options.fetch_batch_size.unwrap_or(1000).max(1);
+ let fetch_batch_size = options.fetch_batch_size.unwrap_or(DEFAULT_FETCH_BATCH_SIZE).max(1);
let source_count_sql =
build_count_table_sql(Some(source_database_type), Some(&options.source_schema), &options.source_table);
@@ -403,6 +403,7 @@ pub async fn prepare_data_compare_from_tables(
source_database_type,
fetch_batch_size,
row_budget,
+ source_row_count,
),
fetch_compare_rows(
state,
@@ -415,6 +416,7 @@ pub async fn prepare_data_compare_from_tables(
target_database_type,
fetch_batch_size,
row_budget,
+ target_row_count,
)
)?;
let source_truncated = source_row_count > row_budget as u64;
@@ -532,7 +534,7 @@ pub async fn prepare_data_compare_missing_target(
let target_database_type = connection_database_type(state, &options.target_connection_id).await?;
ensure_data_compare_supported(source_database_type, "source")?;
ensure_data_compare_supported(target_database_type, "target")?;
- let fetch_batch_size = options.fetch_batch_size.unwrap_or(1000).max(1);
+ let fetch_batch_size = options.fetch_batch_size.unwrap_or(DEFAULT_FETCH_BATCH_SIZE).max(1);
let source_columns = get_columns_core(
state,
&options.source_connection_id,
@@ -572,6 +574,7 @@ pub async fn prepare_data_compare_missing_target(
source_database_type,
fetch_batch_size,
row_budget,
+ source_row_count,
)
.await?;
let source_truncated = source_row_count > row_budget as u64;
@@ -776,6 +779,10 @@ pub fn compare_data_rows(options: CompareDataRowsOptions) -> Result<DataCompareR
Ok(DataCompareResult { added, removed, modified })
}
+/// Rows per page when a compare reads a table. Keyset paging keeps each page an index seek,
+/// so larger pages mostly just save round trips.
+const DEFAULT_FETCH_BATCH_SIZE: usize = 10_000;
+
static NULL_VALUE: Value = Value::Null;
type CompareRowValues = Vec<Value>;
@@ -1327,6 +1334,12 @@ fn build_rownum_data_compare_select_sql(
/// fetch turns a large table into an out-of-memory freeze on the client. Callers derive
/// `source_truncated` / `target_truncated` from the table's row count versus `row_budget`
/// so a partial diff is never mistaken for a complete one.
+///
+/// SQL Server and PostgreSQL page by key (seek) instead of by offset: an offset page
+/// re-reads every earlier row, which makes a full read quadratic in the table size.
+/// `expected_rows` is the table's counted size; when a keyset read ends with a different
+/// number of rows (NULL or non-unique keys, a key literal that does not round-trip), the
+/// table is re-read with offset paging so the result is never silently incomplete.
async fn fetch_compare_rows(
state: &AppState,
connection_id: &str,
@@ -1338,10 +1351,76 @@ async fn fetch_compare_rows(
database_type: DatabaseType,
fetch_batch_size: usize,
row_budget: usize,
+ expected_rows: u64,
) -> Result<Vec<Vec<Value>>, String> {
- let mut rows = Vec::new();
- let mut offset = 0usize;
+ if let Some(key_indexes) = keyset_key_indexes(database_type, columns, key_columns) {
+ // A keyset page error (e.g. a key column whose rendered text literal does
+ // not implicitly cast in a `>` predicate, such as PostgreSQL money/bytea)
+ // falls back to offset paging instead of failing a compare that the
+ // offset path can still serve.
+ let rows = match fetch_compare_rows_by_key(
+ state,
+ connection_id,
+ database,
+ schema,
+ table_name,
+ columns,
+ key_columns,
+ &key_indexes,
+ database_type,
+ fetch_batch_size,
+ row_budget,
+ )
+ .await
+ {
+ Ok(rows) => rows,
+ Err(error) => {
+ log::warn!(
+ "[data-compare:keyset-fallback] database_type={database_type:?} table={table_name} error={error}"
+ );
+ None
+ }
+ };
+ if let Some(rows) = rows {
+ let expected = expected_rows.min(row_budget as u64);
+ if rows.len() as u64 == expected {
+ return Ok(rows);
+ }
+ }
+ }
+
+ fetch_compare_rows_by_offset(
+ state,
+ connection_id,
+ database,
+ schema,
+ table_name,
+ columns,
+ key_columns,
+ database_type,
+ fetch_batch_size,
+ row_budget,
+ 0,
+ Vec::new(),
+ )
+ .await
+}
+#[allow(clippy::too_many_arguments)]
+async fn fetch_compare_rows_by_offset(
+ state: &AppState,
+ connection_id: &str,
+ database: &str,
+ schema: &str,
+ table_name: &str,
+ columns: &[String],
+ key_columns: &[String],
+ database_type: DatabaseType,
+ fetch_batch_size: usize,
+ row_budget: usize,
+ mut offset: usize,
+ mut rows: Vec<Vec<Value>>,
+) -> Result<Vec<Vec<Value>>, String> {
while rows.len() < row_budget {
let batch_size = fetch_batch_size.min(row_budget - rows.len());
let sql =
@@ -1370,6 +1449,131 @@ async fn fetch_compare_rows(
Ok(rows)
}
+/// Reads pages by key. Returns `None` when a page boundary cannot be expressed as a key
+/// predicate (a NULL or structured key value), so the caller falls back to offset paging.
+#[allow(clippy::too_many_arguments)]
+async fn fetch_compare_rows_by_key(
+ state: &AppState,
+ connection_id: &str,
+ database: &str,
+ schema: &str,
+ table_name: &str,
+ columns: &[String],
+ key_columns: &[String],
+ key_indexes: &[usize],
+ database_type: DatabaseType,
+ fetch_batch_size: usize,
+ row_budget: usize,
+) -> Result<Option<Vec<Vec<Value>>>, String> {
+ let mut rows: Vec<Vec<Value>> = Vec::new();
+ let mut after: Option<Vec<String>> = None;
+
+ while rows.len() < row_budget {
+ let batch_size = fetch_batch_size.min(row_budget - rows.len());
+ let sql = build_keyset_data_compare_select_sql(
+ database_type,
+ schema,
+ table_name,
+ columns,
+ key_columns,
+ batch_size,
+ after.as_deref(),
+ );
+ let result = execute_sql_statement_with_options(
+ state,
+ connection_id,
+ database,
+ &sql,
+ Some(schema),
+ None,
+ QueryExecutionOptions { max_rows: Some(batch_size), ..Default::default() },
+ )
+ .await?;
+ let fetched = result.rows.len();
+ if fetched == 0 {
+ break;
+ }
+ rows.extend(result.rows);
+ if fetched < batch_size {
+ break;
+ }
+ let Some(last_row) = rows.last() else { break };
+ let Some(boundary) = keyset_boundary_literals(database_type, last_row, key_indexes) else {
+ return Ok(None);
+ };
+ after = Some(boundary);
+ }
+
+ Ok(Some(rows))
+}
+
+/// Positions of the key columns in `columns`, when this database pages compares by key.
+fn keyset_key_indexes(database_type: DatabaseType, columns: &[String], key_columns: &[String]) -> Option<Vec<usize>> {
+ if !matches!(database_type, DatabaseType::SqlServer | DatabaseType::Postgres) || key_columns.is_empty() {
+ return None;
+ }
+ key_columns.iter().map(|key| columns.iter().position(|column| column == key)).collect()
+}
+
+/// SQL literals for the last row's key values; `None` when one cannot bound a key range.
+fn keyset_boundary_literals(database_type: DatabaseType, row: &[Value], key_indexes: &[usize]) -> Option<Vec<String>> {
+ key_indexes
+ .iter()
+ .map(|&index| {
+ let value = row.get(index)?;
+ matches!(value, Value::String(_) | Value::Number(_) | Value::Bool(_))
+ .then(|| format_grid_sql_literal(value, Some(database_type), None))
+ })
+ .collect()
+}
+
+/// Builds one keyset page: rows ordered by key, strictly after the `after` key literals.
+fn build_keyset_data_compare_select_sql(
+ database_type: DatabaseType,
+ schema: &str,
+ table_name: &str,
+ columns: &[String],
+ key_columns: &[String],
+ row_limit: usize,
+ after: Option<&[String]>,
+) -> String {
+ let table = qualified_table_name(Some(database_type), Some(schema), table_name);
+ let quote_column = |column: &str| quote_table_identifier(Some(database_type), column);
+ let select_columns = columns.iter().map(|column| quote_column(column)).collect::<Vec<_>>().join(", ");
+ let quoted_keys = key_columns.iter().map(|column| quote_column(column)).collect::<Vec<_>>();
+ let order_by = quoted_keys.iter().map(|column| format!("{column} ASC")).collect::<Vec<_>>().join(", ");
+ let where_sql = after
+ .map(|literals| format!(" WHERE {}", keyset_after_predicate(database_type, &quoted_keys, literals)))
+ .unwrap_or_default();
+
+ if database_type == DatabaseType::SqlServer {
+ format!("SELECT TOP ({row_limit}) {select_columns} FROM {table}{where_sql} ORDER BY {order_by}")
+ } else {
+ format!("SELECT {select_columns} FROM {table}{where_sql} ORDER BY {order_by} LIMIT {row_limit};")
+ }
+}
+
+/// `(k1, k2) > (v1, v2)` - PostgreSQL compares row values natively (and seeks a composite
+/// index with it); SQL Server needs the expanded `k1 > v1 OR (k1 = v1 AND k2 > v2)` form.
+fn keyset_after_predicate(database_type: DatabaseType, quoted_keys: &[String], literals: &[String]) -> String {
+ if quoted_keys.len() == 1 {
+ return format!("{} > {}", quoted_keys[0], literals[0]);
+ }
+ if database_type == DatabaseType::Postgres {
+ return format!("({}) > ({})", quoted_keys.join(", "), literals.join(", "));
+ }
+ (0..quoted_keys.len())
+ .map(|position| {
+ let mut terms = (0..position)
+ .map(|earlier| format!("{} = {}", quoted_keys[earlier], literals[earlier]))
+ .collect::<Vec<_>>();
+ terms.push(format!("{} > {}", quoted_keys[position], literals[position]));
+ format!("({})", terms.join(" AND "))
+ })
+ .collect::<Vec<_>>()
+ .join(" OR ")
+}
+
fn where_by_key(
key_values: &HashMap<String, Value>,
key_columns: &[String],
@@ -3101,6 +3305,64 @@ mod tests {
assert!(err.contains("Duplicate source key for column(s) [id]: \"-9222269855922960743\""), "{err}");
}
+ #[test]
+ fn builds_keyset_page_sql_for_sqlserver() {
+ let columns = vec!["id".to_string(), "name".to_string()];
+ let keys = vec!["id".to_string()];
+ assert_eq!(
+ build_keyset_data_compare_select_sql(DatabaseType::SqlServer, "dbo", "users", &columns, &keys, 500, None),
+ "SELECT TOP (500) [id], [name] FROM [dbo].[users] ORDER BY [id] ASC"
+ );
+ let boundary = keyset_boundary_literals(DatabaseType::SqlServer, &[json!(42), json!("x")], &[0]).unwrap();
+ assert_eq!(
+ build_keyset_data_compare_select_sql(
+ DatabaseType::SqlServer,
+ "dbo",
+ "users",
+ &columns,
+ &keys,
+ 500,
+ Some(&boundary)
+ ),
+ "SELECT TOP (500) [id], [name] FROM [dbo].[users] WHERE [id] > 42 ORDER BY [id] ASC"
+ );
+ }
+
+ #[test]
+ fn builds_keyset_page_sql_for_composite_keys() {
+ let columns = vec!["a".to_string(), "b".to_string(), "v".to_string()];
+ let keys = vec!["a".to_string(), "b".to_string()];
+ let row = [json!(1), json!("o'k"), json!("v")];
+ let mssql = keyset_boundary_literals(DatabaseType::SqlServer, &row, &[0, 1]).unwrap();
+ assert_eq!(
+ build_keyset_data_compare_select_sql(DatabaseType::SqlServer, "dbo", "t", &columns, &keys, 10, Some(&mssql)),
+ "SELECT TOP (10) [a], [b], [v] FROM [dbo].[t] WHERE ([a] > 1) OR ([a] = 1 AND [b] > N'o''k') ORDER BY [a] ASC, [b] ASC"
+ );
+ let pg = keyset_boundary_literals(DatabaseType::Postgres, &row, &[0, 1]).unwrap();
+ assert_eq!(
+ build_keyset_data_compare_select_sql(DatabaseType::Postgres, "public", "t", &columns, &keys, 10, Some(&pg)),
+ "SELECT \"a\", \"b\", \"v\" FROM \"public\".\"t\" WHERE (\"a\", \"b\") > (1, 'o''k') ORDER BY \"a\" ASC, \"b\" ASC LIMIT 10;"
+ );
+ }
+
+ #[test]
+ fn keyset_paging_only_applies_to_supported_databases_with_selected_keys() {
+ let columns = vec!["id".to_string(), "name".to_string()];
+ let keys = vec!["id".to_string()];
+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &columns, &keys), Some(vec![0]));
+ assert_eq!(keyset_key_indexes(DatabaseType::SqlServer, &columns, &keys), Some(vec![0]));
+ assert_eq!(keyset_key_indexes(DatabaseType::Mysql, &columns, &keys), None);
+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &columns, &[]), None);
+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &[], &keys), None);
+ }
+
+ #[test]
+ fn keyset_boundary_rejects_null_and_structured_keys() {
+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[Value::Null], &[0]), None);
+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[json!({"a": 1})], &[0]), None);
+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[json!(1)], &[3]), None);
+ }
+
#[test]
fn builds_backend_table_select_sql_with_explicit_columns_and_key_order() {
assert_eq!(