Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(transfer): page keyless PostgreSQL transfers by ctid block windows

zipg18bd51bd74cc
diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rs
index 27de55a237..cd246050af 100644
--- a/crates/dbx-core/src/data/transfer.rs
+++ b/crates/dbx-core/src/data/transfer.rs
@@ -5787,6 +5787,231 @@ fn advance_keyset_cursor(
}
}
+/// Column alias of the trailing `ctid` the windowed PostgreSQL read appends to
+/// the selected columns. It must not be spelled `ctid`: an output column with
+/// that name shadows the system column inside `ORDER BY`, which would sort each
+/// page as text instead of by physical position.
+const POSTGRES_CTID_ALIAS: &str = "dbx_row_ctid";
+
+/// Upper bound on the block window the PostgreSQL `ctid` fallback pages over.
+/// A larger window is estimated as a bigger slice of the table, and the planner
+/// stops choosing a Tid Range Scan for it.
+const POSTGRES_CTID_WINDOW_MAX_BLOCKS: u64 = 65_536;
+
+/// Pages a table must hold before a key-less PostgreSQL read switches from
+/// OFFSET paging to `ctid` windows. Below it the quadratic OFFSET scan is cheap
+/// enough that the extra catalog probe is not worth the behaviour change.
+const POSTGRES_CTID_PAGING_MIN_PAGES: u64 = 20;
+
+/// Physical layout of a source heap, used to size the first `ctid` window.
+struct PostgresCtidLayout {
+ total_blocks: u64,
+ rows_per_block: f64,
+}
+
+/// Paging state for the key-less PostgreSQL `ctid` fallback.
+///
+/// A `WHERE ctid > <last row>` cursor alone does not avoid the quadratic scan:
+/// PostgreSQL only turns that predicate into a Tid Range Scan when it expects a
+/// small range, and a bound anywhere inside a heap is estimated as a large part
+/// of it, so every page falls back to a sequential scan plus sort. Bounding the
+/// page with an upper `ctid` as well — a *window* of blocks — keeps the estimate
+/// proportional to the window, so each page reads only the blocks it needs.
+///
+/// Only plain heap tables qualify (see [`postgres_ctid_pager`]): a partitioned
+/// parent fans out to children whose `ctid`s overlap, so paging it by physical
+/// position could silently skip rows. Those keep OFFSET paging.
+struct PostgresCtidPager {
+ /// Cursor of the next page: the previous page's last `ctid`, or `None`
+ /// right after a window was consumed, when the next page restarts at the
+ /// current window's first block.
+ lower_bound: Option<String>,
+ /// First block covered by the window the next page reads.
+ window_start_block: u64,
+ /// Blocks each window covers, re-sized from the density actually observed.
+ window_blocks: u64,
+ /// Live block count of the relation when the transfer started.
+ total_blocks: u64,
+ /// Rows the pages of the window being read have returned so far.
+ rows_in_window: u64,
+}
+
+impl PostgresCtidPager {
+ fn new(layout: PostgresCtidLayout, page_rows: usize) -> Self {
+ Self {
+ lower_bound: None,
+ window_start_block: 0,
+ window_blocks: postgres_ctid_window_blocks(page_rows, layout.rows_per_block),
+ total_blocks: layout.total_blocks,
+ rows_in_window: 0,
+ }
+ }
+
+ fn window_end_block(&self) -> u64 {
+ self.window_start_block.saturating_add(self.window_blocks)
+ }
+
+ /// SQL for the next page: the selected columns plus the trailing `ctid` the
+ /// caller turns into the next cursor.
+ fn page_sql(&self, columns: &[String], table: &str, schema: &str, limit: usize) -> String {
+ let full_table = qualified_table(table, schema, &DatabaseType::Postgres, None);
+ let mut col_list = columns
+ .iter()
+ .map(|column| quote_identifier(column, &DatabaseType::Postgres))
+ .collect::<Vec<_>>()
+ .join(", ");
+ if !col_list.is_empty() {
+ col_list.push_str(", ");
+ }
+ let lower_bound = match self.lower_bound.as_deref() {
+ Some(bound) => format!("ctid > '{bound}'::tid"),
+ None => format!("ctid >= '({},0)'::tid", self.window_start_block),
+ };
+ format!(
+ "SELECT {col_list}ctid::text AS {POSTGRES_CTID_ALIAS} FROM {full_table} \
+ WHERE {lower_bound} AND ctid < '({},0)'::tid ORDER BY ctid LIMIT {limit}",
+ self.window_end_block()
+ )
+ }
+
+ /// Folds the page just read into the pager — dropping the trailing `ctid`
+ /// column from every row — and reports whether the heap was scanned to its
+ /// end. A full page stays inside the current window; a short one consumes
+ /// it and re-sizes the next window from the row density actually observed.
+ fn advance(&mut self, rows: &mut [Vec<serde_json::Value>], limit: usize) -> Result<bool, String> {
+ let page_rows = rows.len();
+ let mut last_ctid: Option<String> = None;
+ for row in rows.iter_mut() {
+ last_ctid = match row.pop() {
+ Some(serde_json::Value::String(cursor)) => Some(cursor),
+ _ => None,
+ };
+ }
+ self.rows_in_window = self.rows_in_window.saturating_add(page_rows as u64);
+
+ if limit > 0 && page_rows >= limit {
+ let Some(next) = last_ctid else {
+ return Err("ctid paging lost the row cursor".to_string());
+ };
+ if self.lower_bound.as_deref() == Some(next.as_str()) {
+ return Err("ctid paging did not advance past the last row".to_string());
+ }
+ self.lower_bound = Some(next);
+ return Ok(false);
+ }
+
+ self.window_start_block = self.window_end_block();
+ self.lower_bound = None;
+ if self.rows_in_window > 0 {
+ let rows_per_block = self.rows_in_window as f64 / self.window_blocks.max(1) as f64;
+ self.window_blocks = postgres_ctid_window_blocks(limit, rows_per_block);
+ }
+ self.rows_in_window = 0;
+ Ok(self.window_start_block >= self.total_blocks)
+ }
+}
+
+/// Blocks that hold roughly one page of rows at the given density. A window
+/// that runs short is re-sized from the density the transfer observes, so a
+/// stale `reltuples` only costs the first window.
+fn postgres_ctid_window_blocks(page_rows: usize, rows_per_block: f64) -> u64 {
+ let rows_per_block = if rows_per_block > 0.0 && rows_per_block.is_finite() { rows_per_block } else { 1.0 };
+ let blocks = (page_rows.max(1) as f64 / rows_per_block).ceil();
+ if !blocks.is_finite() || blocks < 1.0 {
+ return 1;
+ }
+ (blocks as u64).clamp(1, POSTGRES_CTID_WINDOW_MAX_BLOCKS)
+}
+
+/// Builds the `ctid` pager for a key-less PostgreSQL source table, or `None`
+/// when the relation cannot be paged by physical position.
+///
+/// Only plain heap tables qualify. A partitioned parent spreads `ctid`s over
+/// children that number their own blocks from zero, and a view or foreign table
+/// has no tuple identifiers at all; paging either one would silently drop rows,
+/// so both keep OFFSET paging. A traditional-inheritance parent has the same
+/// hazard: queries against it also return descendant rows, and every child
+/// numbers its `ctid`s from its own block zero. PostgreSQL rejects a user
+/// column named `ctid`, so the system column is always the one the pager reads.
+async fn postgres_ctid_pager(
+ state: &Arc<AppState>,
+ pool_key: &str,
+ schema: &str,
+ table: &str,
+ page_rows: usize,
+ row_count: u64,
+) -> Result<Option<PostgresCtidPager>, String> {
+ let sql = postgres_ctid_eligibility_sql(schema, table);
+ let result = execute_on_pool_with_max_rows(state, pool_key, &sql, Some(1)).await?;
+ let Some(row) = result.rows.first() else {
+ return Ok(None);
+ };
+ if row.get(3).and_then(json_value_bool) != Some(true) {
+ return Ok(None);
+ }
+ let total_blocks = row.first().and_then(json_value_u64).unwrap_or(0);
+ if total_blocks == 0 {
+ return Ok(None);
+ }
+ let reltuples = row.get(1).and_then(json_value_f64).unwrap_or(0.0);
+ let relpages = row.get(2).and_then(json_value_f64).unwrap_or(0.0);
+ // The exact row count sizes the first window even when the relation was
+ // never analyzed; `reltuples` only covers a count that failed.
+ let rows_per_block = if row_count > 0 {
+ row_count as f64 / total_blocks as f64
+ } else if reltuples > 0.0 && relpages > 0.0 {
+ reltuples / relpages
+ } else {
+ 1.0
+ };
+ Ok(Some(PostgresCtidPager::new(PostgresCtidLayout { total_blocks, rows_per_block }, page_rows)))
+}
+
+/// Catalog probe for `postgres_ctid_pager`: heap size in 8K blocks, planner
+/// row/page estimates, and whether the relation is a plain heap table that is
+/// neither an inheritance child nor a traditional-inheritance parent (a parent
+/// also returns descendant rows whose `ctid`s restart from each child's block
+/// zero, so window bounds anchored on the parent's size would drop rows;
+/// partitioned parents are already excluded by `relkind = 'r'`).
+fn postgres_ctid_eligibility_sql(schema: &str, table: &str) -> String {
+ let scope = if schema.is_empty() {
+ format!("c.relname = {} AND pg_catalog.pg_table_is_visible(c.oid)", quote_string_literal(table))
+ } else {
+ format!("n.nspname = {} AND c.relname = {}", quote_string_literal(schema), quote_string_literal(table))
+ };
+ format!(
+ "SELECT pg_relation_size(c.oid) / 8192, c.reltuples, c.relpages, \
+ (c.relkind = 'r' \
+ AND NOT EXISTS (SELECT 1 FROM pg_inherits i WHERE i.inhrelid = c.oid) \
+ AND NOT EXISTS (SELECT 1 FROM pg_inherits p WHERE p.inhparent = c.oid)) \
+ FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE {scope}"
+ )
+}
+
+fn json_value_bool(value: &serde_json::Value) -> Option<bool> {
+ match value {
+ serde_json::Value::Bool(value) => Some(*value),
+ serde_json::Value::String(value) => value.parse().ok(),
+ _ => None,
+ }
+}
+
+fn json_value_u64(value: &serde_json::Value) -> Option<u64> {
+ match value {
+ serde_json::Value::Number(number) => number.as_u64(),
+ serde_json::Value::String(value) => value.parse().ok(),
+ _ => None,
+ }
+}
+
+fn json_value_f64(value: &serde_json::Value) -> Option<f64> {
+ match value {
+ serde_json::Value::Number(number) => number.as_f64(),
+ serde_json::Value::String(value) => value.parse().ok(),
+ _ => None,
+ }
+}
+
/// Whether a table copy may stream through the PostgreSQL COPY protocol
/// (`COPY ... TO STDOUT` on the source piped into `COPY ... FROM STDIN` on the
/// target) instead of the paged SELECT + multi-row INSERT loop.
@@ -10156,6 +10381,36 @@ where
)
});
let mut hive_server_cursor = HiveServerTransferCursor::default();
+ // Key-less PostgreSQL heaps page by `ctid` windows instead of OFFSET: an
+ // OFFSET page re-reads every row before it, so a ten-million-row table gets
+ // slower as it runs and never finishes. Tables with a usable key keep the
+ // keyset cursor above, and the COPY fast path already covers the
+ // PostgreSQL-to-PostgreSQL case.
+ let mut ctid_pager = if copy_rows.is_none()
+ && keyset_indexes.is_none()
+ && !default_rows_only
+ && *source_db_type == DatabaseType::Postgres
+ && total_rows.is_some_and(|total| total > batch_size as u64 * POSTGRES_CTID_PAGING_MIN_PAGES)
+ {
+ match postgres_ctid_pager(
+ state,
+ source_pool_key,
+ &request.source_schema,
+ table,
+ batch_size,
+ total_rows.unwrap_or(0),
+ )
+ .await
+ {
+ Ok(pager) => pager,
+ Err(error) => {
+ log::warn!("[transfer] {table}: ctid paging unavailable ({error}); using OFFSET paging");
+ None
+ }
+ }
+ } else {
+ None
+ };
let transfer_result: Result<(), String> = async {
if copy_rows.is_some() {
@@ -10180,6 +10435,9 @@ where
.await?,
false,
)
+ } else if let Some(pager) = ctid_pager.as_ref() {
+ let sql = pager.page_sql(&col_names, table, &request.source_schema, batch_size);
+ (execute_on_pool_with_max_rows(state, source_pool_key, &sql, Some(batch_size)).await?, false)
} else {
let sql = if default_rows_only {
// Preserve row multiplicity without reading generated values
@@ -10226,9 +10484,28 @@ where
let row_count = result.rows.len();
if row_count == 0 {
+ // A `ctid` window can be empty — its rows were deleted, or the
+ // previous page ended exactly on the window boundary — while
+ // the windows after it still hold rows, so only stop once the
+ // heap has been walked to its end.
+ let continue_windows = match ctid_pager.as_mut() {
+ Some(pager) => !pager.advance(&mut result.rows, batch_size)?,
+ None => false,
+ };
+ if continue_windows {
+ continue;
+ }
break;
}
+ // Drops the trailing `ctid` cursor column from every row before the
+ // rows reach the INSERT builder, and reports whether this page
+ // consumed the last window of the heap.
+ let ctid_scan_finished = match ctid_pager.as_mut() {
+ Some(pager) => Some(pager.advance(&mut result.rows, batch_size)?),
+ None => None,
+ };
+
if let Some(indexes) = keyset_indexes.as_deref() {
match advance_keyset_cursor(&mut keyset_cursor, &result.rows, indexes, table)? {
KeysetAdvance::Advanced => {}
@@ -10308,7 +10585,12 @@ where
terminal: false,
});
- if (use_hive_server_cursor && !has_more) || (!use_hive_server_cursor && row_count < batch_size) {
+ if ctid_pager.is_some() {
+ // A short `ctid` page only means the current window ran out.
+ if ctid_scan_finished == Some(true) {
+ break;
+ }
+ } else if (use_hive_server_cursor && !has_more) || (!use_hive_server_cursor && row_count < batch_size) {
break;
}
}
@@ -15705,6 +15987,114 @@ PARTITION p_old VALUES LESS THAN (TO_DAYS('2026-01-01')))";
assert!(matches!(advance_keyset_cursor(&mut cursor, &Vec::new(), &[0], "t"), Ok(KeysetAdvance::Advanced)));
}
+ #[test]
+ fn postgres_ctid_eligibility_excludes_inheritance_children_and_parents() {
+ for sql in [postgres_ctid_eligibility_sql("", "events"), postgres_ctid_eligibility_sql("public", "events")] {
+ assert!(sql.contains("c.relkind = 'r'"), "only plain heap tables may page by ctid: {sql}");
+ assert!(sql.contains("i.inhrelid = c.oid"), "inheritance children restart ctid numbering per child: {sql}");
+ assert!(
+ sql.contains("p.inhparent = c.oid"),
+ "inheritance parents also return descendant rows and must keep OFFSET paging: {sql}"
+ );
+ }
+ }
+
+ #[test]
+ fn postgres_ctid_page_sql_bounds_each_page_to_a_block_window() {
+ let cols = vec!["id".to_string(), "userName".to_string()];
+ let pager = PostgresCtidPager::new(PostgresCtidLayout { total_blocks: 5_000, rows_per_block: 100.0 }, 1000);
+
+ let first = pager.page_sql(&cols, "events", "public", 1000);
+ assert_eq!(
+ first,
+ "SELECT \"id\", \"userName\", ctid::text AS dbx_row_ctid FROM \"public\".\"events\" \
+ WHERE ctid >= '(0,0)'::tid AND ctid < '(10,0)'::tid ORDER BY ctid LIMIT 1000"
+ );
+ // The cursor column is aliased, so `ORDER BY ctid` still resolves to the
+ // system column instead of the text projection.
+ assert!(!first.contains("AS ctid"));
+
+ let mut pager = pager;
+ let mut rows = vec![vec![json!("(7,3)"), json!("(9,5)")]; 1000];
+ assert_eq!(pager.advance(&mut rows, 1000), Ok(false));
+ let second = pager.page_sql(&cols, "events", "public", 1000);
+ assert!(second.contains("WHERE ctid > '(9,5)'::tid AND ctid < '(10,0)'::tid"), "{second}");
+ }
+
+ #[test]
+ fn postgres_ctid_page_sql_handles_row_count_only_transfers() {
+ let pager = PostgresCtidPager::new(PostgresCtidLayout { total_blocks: 10, rows_per_block: 1.0 }, 1000);
+ let sql = pager.page_sql(&[], "events", "public", 1000);
+ assert!(sql.starts_with("SELECT ctid::text AS dbx_row_ctid FROM"), "{sql}");
+ }
+
+ #[test]
+ fn postgres_ctid_window_sizing_follows_row_density() {
+ // 1000 rows per page at 100 rows per block needs ten blocks.
+ assert_eq!(postgres_ctid_window_blocks(1000, 100.0), 10);
+ // A single row per block needs one block per row.
+ assert_eq!(postgres_ctid_window_blocks(1000, 1.0), 1000);
+ // Dense heaps never shrink below one block, and unknown density assumes
+ // the sparsest layout so the first window still fits one page.
+ assert_eq!(postgres_ctid_window_blocks(1000, 1_000_000.0), 1);
+ assert_eq!(postgres_ctid_window_blocks(1000, 0.0), 1000);
+ assert_eq!(postgres_ctid_window_blocks(1000, f64::NAN), 1000);
+ assert_eq!(postgres_ctid_window_blocks(0, 1.0), 1);
+ }
+
+ #[test]
+ fn postgres_ctid_pager_walks_windows_and_resizes_from_density() {
+ let mut pager = PostgresCtidPager::new(PostgresCtidLayout { total_blocks: 40, rows_per_block: 100.0 }, 1000);
+ assert_eq!(pager.window_end_block(), 10);
+
+ // A full page stays inside the window; the cursor column is dropped from
+ // every row before the INSERT builder sees it.
+ let mut rows = vec![vec![json!("(9,5)"), json!("(9,5)")]; 1000];
+ assert_eq!(pager.advance(&mut rows, 1000), Ok(false));
+ assert_eq!(pager.lower_bound.as_deref(), Some("(9,5)"));
+ assert!(rows.iter().all(|row| row.len() == 1));
+
+ // A short page consumes the window: the next page restarts at the next
+ // block, and the window is re-sized from the observed 150 rows/block.
+ let mut rows = vec![vec![json!("(9,9)")]; 500];
+ assert_eq!(pager.advance(&mut rows, 1000), Ok(false));
+ assert_eq!(pager.window_start_block, 10);
+ assert_eq!(pager.window_end_block(), 17);
+ assert_eq!(pager.lower_bound, None);
+
+ // Empty windows advance the scan instead of ending it.
+ let mut rows: Vec<Vec<serde_json::Value>> = Vec::new();
+ assert_eq!(pager.advance(&mut rows, 1000), Ok(false));
+ assert_eq!(pager.window_start_block, 17);
+
+ // The heap is finished only once a window past its last block was read.
+ let mut window_start = 17;
+ while window_start < 40 {
+ let mut rows = vec![vec![json!("(1,1)")]; 1];
+ let finished = pager.advance(&mut rows, 1000).expect("window advance");
+ window_start = pager.window_start_block;
+ if finished {
+ break;
+ }
+ assert!(window_start < 1_000_000, "ctid paging did not terminate");
+ }
+ assert!(pager.window_start_block >= 40);
+ assert_eq!(pager.advance(&mut Vec::new(), 1000), Ok(true));
+ }
+
+ #[test]
+ fn postgres_ctid_pager_rejects_a_stalled_cursor() {
+ let mut pager = PostgresCtidPager::new(PostgresCtidLayout { total_blocks: 100, rows_per_block: 1.0 }, 1000);
+ let mut rows = vec![vec![json!("(1,1)")]; 1000];
+ assert_eq!(pager.advance(&mut rows, 1000), Ok(false));
+ // Re-reading the same last row would loop forever.
+ let mut rows = vec![vec![json!("(1,1)")]; 1000];
+ assert!(pager.advance(&mut rows, 1000).is_err());
+ // A page that lost its cursor column cannot be resumed either.
+ let mut rows = vec![vec![serde_json::Value::Null]; 1000];
+ assert!(pager.advance(&mut rows, 1000).is_err());
+ }
+
#[test]
fn copy_fast_path_requires_pg_compat_append_or_overwrite() {
for mode in [TransferMode::Append, TransferMode::Overwrite] {