25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
perf(data-compare): page SQL Server and PostgreSQL reads by key
Brydon209172db0707
diff --git a/crates/dbx-core/src/data/data_compare.rs b/crates/dbx-core/src/data/data_compare.rsindex 85bc153995..3c4ce7f7ae 100644--- a/crates/dbx-core/src/data/data_compare.rs+++ b/crates/dbx-core/src/data/data_compare.rs@@ -344,7 +344,7 @@ pub async fn prepare_data_compare_from_tables(let target_database_type = connection_database_type(state, &options.target_connection_id).await?;ensure_data_compare_supported(source_database_type, "source")?;ensure_data_compare_supported(target_database_type, "target")?;- let fetch_batch_size = options.fetch_batch_size.unwrap_or(1000).max(1);+ let fetch_batch_size = options.fetch_batch_size.unwrap_or(DEFAULT_FETCH_BATCH_SIZE).max(1);let source_count_sql =build_count_table_sql(Some(source_database_type), Some(&options.source_schema), &options.source_table);@@ -403,6 +403,7 @@ pub async fn prepare_data_compare_from_tables(source_database_type,fetch_batch_size,row_budget,+ source_row_count,),fetch_compare_rows(state,@@ -415,6 +416,7 @@ pub async fn prepare_data_compare_from_tables(target_database_type,fetch_batch_size,row_budget,+ target_row_count,))?;let source_truncated = source_row_count > row_budget as u64;@@ -532,7 +534,7 @@ pub async fn prepare_data_compare_missing_target(let target_database_type = connection_database_type(state, &options.target_connection_id).await?;ensure_data_compare_supported(source_database_type, "source")?;ensure_data_compare_supported(target_database_type, "target")?;- let fetch_batch_size = options.fetch_batch_size.unwrap_or(1000).max(1);+ let fetch_batch_size = options.fetch_batch_size.unwrap_or(DEFAULT_FETCH_BATCH_SIZE).max(1);let source_columns = get_columns_core(state,&options.source_connection_id,@@ -572,6 +574,7 @@ pub async fn prepare_data_compare_missing_target(source_database_type,fetch_batch_size,row_budget,+ source_row_count,).await?;let source_truncated = source_row_count > row_budget as u64;@@ -776,6 +779,10 @@ pub fn compare_data_rows(options: CompareDataRowsOptions) -> Result<DataCompareROk(DataCompareResult { added, removed, modified })}+/// Rows per page when a compare reads a table. Keyset paging keeps each page an index seek,+/// so larger pages mostly just save round trips.+const DEFAULT_FETCH_BATCH_SIZE: usize = 10_000;+static NULL_VALUE: Value = Value::Null;type CompareRowValues = Vec<Value>;@@ -1327,6 +1334,12 @@ fn build_rownum_data_compare_select_sql(/// fetch turns a large table into an out-of-memory freeze on the client. Callers derive/// `source_truncated` / `target_truncated` from the table's row count versus `row_budget`/// so a partial diff is never mistaken for a complete one.+///+/// SQL Server and PostgreSQL page by key (seek) instead of by offset: an offset page+/// re-reads every earlier row, which makes a full read quadratic in the table size.+/// `expected_rows` is the table's counted size; when a keyset read ends with a different+/// number of rows (NULL or non-unique keys, a key literal that does not round-trip), the+/// table is re-read with offset paging so the result is never silently incomplete.async fn fetch_compare_rows(state: &AppState,connection_id: &str,@@ -1338,10 +1351,76 @@ async fn fetch_compare_rows(database_type: DatabaseType,fetch_batch_size: usize,row_budget: usize,+ expected_rows: u64,) -> Result<Vec<Vec<Value>>, String> {- let mut rows = Vec::new();- let mut offset = 0usize;+ if let Some(key_indexes) = keyset_key_indexes(database_type, columns, key_columns) {+ // A keyset page error (e.g. a key column whose rendered text literal does+ // not implicitly cast in a `>` predicate, such as PostgreSQL money/bytea)+ // falls back to offset paging instead of failing a compare that the+ // offset path can still serve.+ let rows = match fetch_compare_rows_by_key(+ state,+ connection_id,+ database,+ schema,+ table_name,+ columns,+ key_columns,+ &key_indexes,+ database_type,+ fetch_batch_size,+ row_budget,+ )+ .await+ {+ Ok(rows) => rows,+ Err(error) => {+ log::warn!(+ "[data-compare:keyset-fallback] database_type={database_type:?} table={table_name} error={error}"+ );+ None+ }+ };+ if let Some(rows) = rows {+ let expected = expected_rows.min(row_budget as u64);+ if rows.len() as u64 == expected {+ return Ok(rows);+ }+ }+ }++ fetch_compare_rows_by_offset(+ state,+ connection_id,+ database,+ schema,+ table_name,+ columns,+ key_columns,+ database_type,+ fetch_batch_size,+ row_budget,+ 0,+ Vec::new(),+ )+ .await+}+#[allow(clippy::too_many_arguments)]+async fn fetch_compare_rows_by_offset(+ state: &AppState,+ connection_id: &str,+ database: &str,+ schema: &str,+ table_name: &str,+ columns: &[String],+ key_columns: &[String],+ database_type: DatabaseType,+ fetch_batch_size: usize,+ row_budget: usize,+ mut offset: usize,+ mut rows: Vec<Vec<Value>>,+) -> Result<Vec<Vec<Value>>, String> {while rows.len() < row_budget {let batch_size = fetch_batch_size.min(row_budget - rows.len());let sql =@@ -1370,6 +1449,131 @@ async fn fetch_compare_rows(Ok(rows)}+/// Reads pages by key. Returns `None` when a page boundary cannot be expressed as a key+/// predicate (a NULL or structured key value), so the caller falls back to offset paging.+#[allow(clippy::too_many_arguments)]+async fn fetch_compare_rows_by_key(+ state: &AppState,+ connection_id: &str,+ database: &str,+ schema: &str,+ table_name: &str,+ columns: &[String],+ key_columns: &[String],+ key_indexes: &[usize],+ database_type: DatabaseType,+ fetch_batch_size: usize,+ row_budget: usize,+) -> Result<Option<Vec<Vec<Value>>>, String> {+ let mut rows: Vec<Vec<Value>> = Vec::new();+ let mut after: Option<Vec<String>> = None;++ while rows.len() < row_budget {+ let batch_size = fetch_batch_size.min(row_budget - rows.len());+ let sql = build_keyset_data_compare_select_sql(+ database_type,+ schema,+ table_name,+ columns,+ key_columns,+ batch_size,+ after.as_deref(),+ );+ let result = execute_sql_statement_with_options(+ state,+ connection_id,+ database,+ &sql,+ Some(schema),+ None,+ QueryExecutionOptions { max_rows: Some(batch_size), ..Default::default() },+ )+ .await?;+ let fetched = result.rows.len();+ if fetched == 0 {+ break;+ }+ rows.extend(result.rows);+ if fetched < batch_size {+ break;+ }+ let Some(last_row) = rows.last() else { break };+ let Some(boundary) = keyset_boundary_literals(database_type, last_row, key_indexes) else {+ return Ok(None);+ };+ after = Some(boundary);+ }++ Ok(Some(rows))+}++/// Positions of the key columns in `columns`, when this database pages compares by key.+fn keyset_key_indexes(database_type: DatabaseType, columns: &[String], key_columns: &[String]) -> Option<Vec<usize>> {+ if !matches!(database_type, DatabaseType::SqlServer | DatabaseType::Postgres) || key_columns.is_empty() {+ return None;+ }+ key_columns.iter().map(|key| columns.iter().position(|column| column == key)).collect()+}++/// SQL literals for the last row's key values; `None` when one cannot bound a key range.+fn keyset_boundary_literals(database_type: DatabaseType, row: &[Value], key_indexes: &[usize]) -> Option<Vec<String>> {+ key_indexes+ .iter()+ .map(|&index| {+ let value = row.get(index)?;+ matches!(value, Value::String(_) | Value::Number(_) | Value::Bool(_))+ .then(|| format_grid_sql_literal(value, Some(database_type), None))+ })+ .collect()+}++/// Builds one keyset page: rows ordered by key, strictly after the `after` key literals.+fn build_keyset_data_compare_select_sql(+ database_type: DatabaseType,+ schema: &str,+ table_name: &str,+ columns: &[String],+ key_columns: &[String],+ row_limit: usize,+ after: Option<&[String]>,+) -> String {+ let table = qualified_table_name(Some(database_type), Some(schema), table_name);+ let quote_column = |column: &str| quote_table_identifier(Some(database_type), column);+ let select_columns = columns.iter().map(|column| quote_column(column)).collect::<Vec<_>>().join(", ");+ let quoted_keys = key_columns.iter().map(|column| quote_column(column)).collect::<Vec<_>>();+ let order_by = quoted_keys.iter().map(|column| format!("{column} ASC")).collect::<Vec<_>>().join(", ");+ let where_sql = after+ .map(|literals| format!(" WHERE {}", keyset_after_predicate(database_type, "ed_keys, literals)))+ .unwrap_or_default();++ if database_type == DatabaseType::SqlServer {+ format!("SELECT TOP ({row_limit}) {select_columns} FROM {table}{where_sql} ORDER BY {order_by}")+ } else {+ format!("SELECT {select_columns} FROM {table}{where_sql} ORDER BY {order_by} LIMIT {row_limit};")+ }+}++/// `(k1, k2) > (v1, v2)` - PostgreSQL compares row values natively (and seeks a composite+/// index with it); SQL Server needs the expanded `k1 > v1 OR (k1 = v1 AND k2 > v2)` form.+fn keyset_after_predicate(database_type: DatabaseType, quoted_keys: &[String], literals: &[String]) -> String {+ if quoted_keys.len() == 1 {+ return format!("{} > {}", quoted_keys[0], literals[0]);+ }+ if database_type == DatabaseType::Postgres {+ return format!("({}) > ({})", quoted_keys.join(", "), literals.join(", "));+ }+ (0..quoted_keys.len())+ .map(|position| {+ let mut terms = (0..position)+ .map(|earlier| format!("{} = {}", quoted_keys[earlier], literals[earlier]))+ .collect::<Vec<_>>();+ terms.push(format!("{} > {}", quoted_keys[position], literals[position]));+ format!("({})", terms.join(" AND "))+ })+ .collect::<Vec<_>>()+ .join(" OR ")+}+fn where_by_key(key_values: &HashMap<String, Value>,key_columns: &[String],@@ -3101,6 +3305,64 @@ mod tests {assert!(err.contains("Duplicate source key for column(s) [id]: \"-9222269855922960743\""), "{err}");}+ #[test]+ fn builds_keyset_page_sql_for_sqlserver() {+ let columns = vec!["id".to_string(), "name".to_string()];+ let keys = vec!["id".to_string()];+ assert_eq!(+ build_keyset_data_compare_select_sql(DatabaseType::SqlServer, "dbo", "users", &columns, &keys, 500, None),+ "SELECT TOP (500) [id], [name] FROM [dbo].[users] ORDER BY [id] ASC"+ );+ let boundary = keyset_boundary_literals(DatabaseType::SqlServer, &[json!(42), json!("x")], &[0]).unwrap();+ assert_eq!(+ build_keyset_data_compare_select_sql(+ DatabaseType::SqlServer,+ "dbo",+ "users",+ &columns,+ &keys,+ 500,+ Some(&boundary)+ ),+ "SELECT TOP (500) [id], [name] FROM [dbo].[users] WHERE [id] > 42 ORDER BY [id] ASC"+ );+ }++ #[test]+ fn builds_keyset_page_sql_for_composite_keys() {+ let columns = vec!["a".to_string(), "b".to_string(), "v".to_string()];+ let keys = vec!["a".to_string(), "b".to_string()];+ let row = [json!(1), json!("o'k"), json!("v")];+ let mssql = keyset_boundary_literals(DatabaseType::SqlServer, &row, &[0, 1]).unwrap();+ assert_eq!(+ build_keyset_data_compare_select_sql(DatabaseType::SqlServer, "dbo", "t", &columns, &keys, 10, Some(&mssql)),+ "SELECT TOP (10) [a], [b], [v] FROM [dbo].[t] WHERE ([a] > 1) OR ([a] = 1 AND [b] > N'o''k') ORDER BY [a] ASC, [b] ASC"+ );+ let pg = keyset_boundary_literals(DatabaseType::Postgres, &row, &[0, 1]).unwrap();+ assert_eq!(+ build_keyset_data_compare_select_sql(DatabaseType::Postgres, "public", "t", &columns, &keys, 10, Some(&pg)),+ "SELECT \"a\", \"b\", \"v\" FROM \"public\".\"t\" WHERE (\"a\", \"b\") > (1, 'o''k') ORDER BY \"a\" ASC, \"b\" ASC LIMIT 10;"+ );+ }++ #[test]+ fn keyset_paging_only_applies_to_supported_databases_with_selected_keys() {+ let columns = vec!["id".to_string(), "name".to_string()];+ let keys = vec!["id".to_string()];+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &columns, &keys), Some(vec![0]));+ assert_eq!(keyset_key_indexes(DatabaseType::SqlServer, &columns, &keys), Some(vec![0]));+ assert_eq!(keyset_key_indexes(DatabaseType::Mysql, &columns, &keys), None);+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &columns, &[]), None);+ assert_eq!(keyset_key_indexes(DatabaseType::Postgres, &[], &keys), None);+ }++ #[test]+ fn keyset_boundary_rejects_null_and_structured_keys() {+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[Value::Null], &[0]), None);+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[json!({"a": 1})], &[0]), None);+ assert_eq!(keyset_boundary_literals(DatabaseType::Postgres, &[json!(1)], &[3]), None);+ }+#[test]fn builds_backend_table_select_sql_with_explicit_columns_and_key_order() {assert_eq!(