Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(redis): bound collection detail page loading

t8y2210512e842f5
diff --git a/crates/dbx-core/src/db/redis_driver.rs b/crates/dbx-core/src/db/redis_driver.rs
index dac71a5325..1983cb9632 100644
--- a/crates/dbx-core/src/db/redis_driver.rs
+++ b/crates/dbx-core/src/db/redis_driver.rs
@@ -9,7 +9,7 @@ use redis::{
TlsMode, Value as RedisRawValue,
};
use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, future::Future, time::Duration, time::Instant};
+use std::{collections::HashMap, future::Future, sync::OnceLock, time::Duration, time::Instant};
use tokio::sync::{Mutex, MutexGuard};
use super::json_value_for_js;
@@ -26,6 +26,9 @@ const CLUSTER_CURSOR_NODE_MASK: u64 = (1 << CLUSTER_CURSOR_NODE_BITS) - 1;
const CLUSTER_CURSOR_SCAN_MASK: u64 = (1 << (64 - CLUSTER_CURSOR_NODE_BITS)) - 1;
const CLUSTER_SCAN_SESSION_LIMIT: usize = 128;
const CLUSTER_SCAN_SESSION_TTL: Duration = Duration::from_secs(5 * 60);
+const COLLECTION_OVERFLOW_SESSION_LIMIT: usize = 128;
+const COLLECTION_OVERFLOW_SESSION_TTL: Duration = Duration::from_secs(5 * 60);
+const COLLECTION_OVERFLOW_CURSOR_START: u64 = 1 << 52;
const INVALID_CLUSTER_SCAN_CURSOR_ERROR: &str = "Redis cluster scan cursor is invalid or expired";
const MAX_SAFE_INTEGER_CURSOR: u64 = (1 << 53) - 1;
@@ -421,6 +424,102 @@ impl RedisClusterScanSessions {
}
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RedisCollectionOverflowKind {
+ Hash,
+ Set,
+}
+
+#[derive(Debug, Clone)]
+enum RedisCollectionOverflowItems {
+ Hash(Vec<RedisHashItem>),
+ Set(Vec<RedisSetItem>),
+}
+
+#[derive(Debug, Clone)]
+struct RedisCollectionOverflowSession {
+ key: Vec<u8>,
+ kind: RedisCollectionOverflowKind,
+ filter: Option<String>,
+ sort_direction: Option<String>,
+ followup_cursor: u64,
+ items: RedisCollectionOverflowItems,
+ last_used: Instant,
+}
+
+#[derive(Debug)]
+struct RedisCollectionOverflowSessions {
+ next_cursor: u64,
+ entries: HashMap<u64, RedisCollectionOverflowSession>,
+}
+
+impl Default for RedisCollectionOverflowSessions {
+ fn default() -> Self {
+ Self { next_cursor: COLLECTION_OVERFLOW_CURSOR_START, entries: HashMap::new() }
+ }
+}
+
+impl RedisCollectionOverflowSessions {
+ fn take_matching(
+ &mut self,
+ cursor: u64,
+ key: &[u8],
+ kind: RedisCollectionOverflowKind,
+ filter: Option<&str>,
+ sort_direction: Option<&str>,
+ ) -> Option<RedisCollectionOverflowSession> {
+ self.remove_expired();
+ let matches = self.entries.get(&cursor).is_some_and(|session| {
+ session.key == key
+ && session.kind == kind
+ && session.filter.as_deref() == filter
+ && session.sort_direction.as_deref() == sort_direction
+ });
+ matches.then(|| self.entries.remove(&cursor)).flatten()
+ }
+
+ fn insert(&mut self, mut session: RedisCollectionOverflowSession) -> u64 {
+ self.remove_expired();
+ while self.entries.len() >= COLLECTION_OVERFLOW_SESSION_LIMIT {
+ let Some(oldest) = self.entries.iter().min_by_key(|(_, entry)| entry.last_used).map(|(id, _)| *id) else {
+ break;
+ };
+ self.entries.remove(&oldest);
+ }
+
+ let cursor = self.next_available_cursor();
+ session.last_used = Instant::now();
+ self.entries.insert(cursor, session);
+ cursor
+ }
+
+ fn reinsert(&mut self, cursor: u64, mut session: RedisCollectionOverflowSession) {
+ self.remove_expired();
+ session.last_used = Instant::now();
+ self.entries.insert(cursor, session);
+ }
+
+ fn remove_expired(&mut self) {
+ self.entries.retain(|_, entry| entry.last_used.elapsed() < COLLECTION_OVERFLOW_SESSION_TTL);
+ }
+
+ fn next_available_cursor(&mut self) -> u64 {
+ loop {
+ let cursor = self.next_cursor;
+ self.next_cursor =
+ if cursor >= MAX_SAFE_INTEGER_CURSOR { COLLECTION_OVERFLOW_CURSOR_START } else { cursor + 1 };
+ if cursor != 0 && !self.entries.contains_key(&cursor) {
+ return cursor;
+ }
+ }
+ }
+}
+
+fn collection_overflow_sessions() -> &'static Mutex<RedisCollectionOverflowSessions> {
+ static SESSIONS: OnceLock<Mutex<RedisCollectionOverflowSessions>> = OnceLock::new();
+ SESSIONS.get_or_init(|| Mutex::new(RedisCollectionOverflowSessions::default()))
+}
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RedisClusterAuth {
pub username: String,
@@ -2484,7 +2583,7 @@ where
}
"set" => {
let len: u64 = redis::cmd("SCARD").arg(key).query_async(con).await.unwrap_or(0);
- let (cursor, items) = sscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE).await?;
+ let (cursor, items) = sscan_bounded_page(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
RedisValueData::Set { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) }
}
"zset" => {
@@ -2496,7 +2595,7 @@ where
}
"hash" => {
let len: u64 = redis::cmd("HLEN").arg(key).query_async(con).await.unwrap_or(0);
- let (cursor, mut items) = hscan_page_raw(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
+ let (cursor, mut items) = hscan_bounded_page(con, key, 0, COLLECTION_PAGE_SIZE, None).await?;
attach_hash_field_ttls(con, key, &mut items).await?;
RedisValueData::Hash { items, total: len, scan_cursor: (cursor > 0).then_some(cursor) }
}
@@ -3556,9 +3655,9 @@ where
}
"set" => {
let (next_cursor, items) = if let Some(query) = filter_query {
- sscan_filtered_page_raw(con, key, cursor, count, query).await?
+ sscan_bounded_page(con, key, cursor, count, Some(query)).await?
} else {
- sscan_page_raw(con, key, cursor, count).await?
+ sscan_bounded_page(con, key, cursor, count, None).await?
};
Ok(RedisCollectionPage::Set { items, scan_cursor: (next_cursor > 0).then_some(next_cursor) })
}
@@ -3583,9 +3682,9 @@ where
}
"hash" => {
let (next_cursor, mut items) = if let Some(query) = filter_query {
- hscan_filtered_page_raw(con, key, cursor, count, query).await?
+ hscan_bounded_page(con, key, cursor, count, Some(query)).await?
} else {
- hscan_page_raw(con, key, cursor, count, None).await?
+ hscan_bounded_page(con, key, cursor, count, None).await?
};
attach_hash_field_ttls(con, key, &mut items).await?;
Ok(RedisCollectionPage::Hash { items, scan_cursor: (next_cursor > 0).then_some(next_cursor) })
@@ -3754,6 +3853,162 @@ where
Ok((items, cur))
}
+async fn hscan_bounded_page<C>(
+ con: &mut C,
+ key: &[u8],
+ cursor: u64,
+ count: usize,
+ filter_query: Option<&str>,
+) -> Result<(u64, Vec<RedisHashItem>), String>
+where
+ C: ConnectionLike + Send + Sync + Unpin,
+{
+ let count = count.max(1);
+ if let Some(page) = take_hash_overflow_page(key, cursor, count, filter_query).await {
+ return Ok(page);
+ }
+
+ let (next_cursor, mut items) = if let Some(query) = filter_query {
+ hscan_filtered_page_raw(con, key, cursor, count, query).await?
+ } else {
+ hscan_page_raw(con, key, cursor, count, None).await?
+ };
+ let next_cursor = store_hash_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
+ Ok((next_cursor, items))
+}
+
+async fn sscan_bounded_page<C>(
+ con: &mut C,
+ key: &[u8],
+ cursor: u64,
+ count: usize,
+ filter_query: Option<&str>,
+) -> Result<(u64, Vec<RedisSetItem>), String>
+where
+ C: ConnectionLike + Send + Sync + Unpin,
+{
+ let count = count.max(1);
+ if let Some(page) = take_set_overflow_page(key, cursor, count, filter_query).await {
+ return Ok(page);
+ }
+
+ let (next_cursor, mut items) = if let Some(query) = filter_query {
+ sscan_filtered_page_raw(con, key, cursor, count, query).await?
+ } else {
+ sscan_page_raw(con, key, cursor, count).await?
+ };
+ let next_cursor = store_set_overflow_page(key, &mut items, count, next_cursor, filter_query).await;
+ Ok((next_cursor, items))
+}
+
+async fn take_hash_overflow_page(
+ key: &[u8],
+ cursor: u64,
+ count: usize,
+ filter_query: Option<&str>,
+) -> Option<(u64, Vec<RedisHashItem>)> {
+ if cursor == 0 {
+ return None;
+ }
+
+ let mut sessions = collection_overflow_sessions().lock().await;
+ let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Hash, filter_query, None)?;
+ let page = match &mut session.items {
+ RedisCollectionOverflowItems::Hash(items) => take_vec_page(items, count),
+ RedisCollectionOverflowItems::Set(_) => return None,
+ };
+ let has_buffered_items = match &session.items {
+ RedisCollectionOverflowItems::Hash(items) => !items.is_empty(),
+ RedisCollectionOverflowItems::Set(_) => false,
+ };
+ let next_cursor = if has_buffered_items { cursor } else { session.followup_cursor };
+ if has_buffered_items {
+ sessions.reinsert(cursor, session);
+ }
+ Some((next_cursor, page))
+}
+
+async fn take_set_overflow_page(
+ key: &[u8],
+ cursor: u64,
+ count: usize,
+ filter_query: Option<&str>,
+) -> Option<(u64, Vec<RedisSetItem>)> {
+ if cursor == 0 {
+ return None;
+ }
+
+ let mut sessions = collection_overflow_sessions().lock().await;
+ let mut session = sessions.take_matching(cursor, key, RedisCollectionOverflowKind::Set, filter_query, None)?;
+ let page = match &mut session.items {
+ RedisCollectionOverflowItems::Set(items) => take_vec_page(items, count),
+ RedisCollectionOverflowItems::Hash(_) => return None,
+ };
+ let has_buffered_items = match &session.items {
+ RedisCollectionOverflowItems::Set(items) => !items.is_empty(),
+ RedisCollectionOverflowItems::Hash(_) => false,
+ };
+ let next_cursor = if has_buffered_items { cursor } else { session.followup_cursor };
+ if has_buffered_items {
+ sessions.reinsert(cursor, session);
+ }
+ Some((next_cursor, page))
+}
+
+async fn store_hash_overflow_page(
+ key: &[u8],
+ items: &mut Vec<RedisHashItem>,
+ count: usize,
+ followup_cursor: u64,
+ filter_query: Option<&str>,
+) -> u64 {
+ if items.len() <= count {
+ return followup_cursor;
+ }
+
+ let overflow = items.split_off(count);
+ let session = RedisCollectionOverflowSession {
+ key: key.to_vec(),
+ kind: RedisCollectionOverflowKind::Hash,
+ filter: filter_query.map(str::to_string),
+ sort_direction: None,
+ followup_cursor,
+ items: RedisCollectionOverflowItems::Hash(overflow),
+ last_used: Instant::now(),
+ };
+ collection_overflow_sessions().lock().await.insert(session)
+}
+
+async fn store_set_overflow_page(
+ key: &[u8],
+ items: &mut Vec<RedisSetItem>,
+ count: usize,
+ followup_cursor: u64,
+ filter_query: Option<&str>,
+) -> u64 {
+ if items.len() <= count {
+ return followup_cursor;
+ }
+
+ let overflow = items.split_off(count);
+ let session = RedisCollectionOverflowSession {
+ key: key.to_vec(),
+ kind: RedisCollectionOverflowKind::Set,
+ filter: filter_query.map(str::to_string),
+ sort_direction: None,
+ followup_cursor,
+ items: RedisCollectionOverflowItems::Set(overflow),
+ last_used: Instant::now(),
+ };
+ collection_overflow_sessions().lock().await.insert(session)
+}
+
+fn take_vec_page<T>(items: &mut Vec<T>, count: usize) -> Vec<T> {
+ let page_size = items.len().min(count.max(1));
+ let overflow = items.split_off(page_size);
+ std::mem::replace(items, overflow)
+}
+
async fn attach_hash_field_ttls<C>(con: &mut C, key: &[u8], items: &mut [RedisHashItem]) -> Result<(), String>
where
C: ConnectionLike + Send + Sync + Unpin,
@@ -4082,6 +4337,22 @@ mod tests {
RedisRawValue::Array(vec![bulk(cursor), RedisRawValue::Array(entries)])
}
+ fn hscan_response_owned(cursor: &str, pairs: Vec<(String, String)>) -> RedisRawValue {
+ let entries = pairs.into_iter().flat_map(|(field, value)| [bulk(&field), bulk(&value)]).collect();
+ RedisRawValue::Array(vec![bulk(cursor), RedisRawValue::Array(entries)])
+ }
+
+ fn int_scan_response_owned(cursor: &str, members: Vec<String>) -> RedisRawValue {
+ RedisRawValue::Array(vec![
+ bulk(cursor),
+ RedisRawValue::Array(members.into_iter().map(|member| bulk(&member)).collect()),
+ ])
+ }
+
+ fn httl_response(count: usize, ttl: i64) -> RedisRawValue {
+ RedisRawValue::Array((0..count).map(|_| RedisRawValue::Int(ttl)).collect())
+ }
+
fn zrange_response(pairs: Vec<(&str, &str)>) -> RedisRawValue {
RedisRawValue::Array(pairs.into_iter().flat_map(|(member, score)| [bulk(member), bulk(score)]).collect())
}
@@ -4457,6 +4728,136 @@ mod tests {
assert_eq!(con.command_count("STRLEN"), 1);
}
+ #[tokio::test]
+ async fn hash_value_pages_compact_hscan_before_field_ttls() {
+ let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();
+ let mut con = FakeRedisConnection::new(vec![
+ bulk("hash"),
+ RedisRawValue::Int(-1),
+ RedisRawValue::Int(512),
+ hscan_response_owned("0", pairs),
+ httl_response(super::COLLECTION_PAGE_SIZE, -1),
+ httl_response(super::COLLECTION_PAGE_SIZE, -1),
+ httl_response(112, -1),
+ ]);
+
+ let value = super::get_value(&mut con, b"large-hash").await.unwrap();
+
+ let RedisValueData::Hash { items, total, scan_cursor } = value.data else {
+ panic!("expected a Hash value");
+ };
+ let cursor = scan_cursor.expect("overflow cursor");
+ assert!(cursor >= super::COLLECTION_OVERFLOW_CURSOR_START);
+ assert_eq!(total, 512);
+ assert_eq!(items.len(), super::COLLECTION_PAGE_SIZE);
+ assert_eq!(redis_blob_display_text(&items[0].field), "field:0");
+ assert_eq!(redis_blob_display_text(&items[199].field), "field:199");
+ assert!(items.iter().all(|item| item.field_ttl == Some(-1)));
+ assert_eq!(con.command_count("HSCAN"), 1);
+ assert_eq!(con.command_count("HTTL"), 1);
+ let httl_command = con.commands.iter().find(|command| command.contains("\r\nHTTL\r\n")).unwrap();
+ assert!(httl_command.contains("\r\nFIELDS\r\n"));
+ assert!(httl_command.contains("\r\n200\r\n"));
+ assert!(!httl_command.contains("\r\nfield:200\r\n"));
+
+ let second =
+ super::load_more_collection(&mut con, b"large-hash", "hash", cursor, 200, None, None).await.unwrap();
+ let RedisCollectionPage::Hash { items, scan_cursor } = second else {
+ panic!("expected a Hash page");
+ };
+ assert_eq!(scan_cursor, Some(cursor));
+ assert_eq!(items.len(), 200);
+ assert_eq!(redis_blob_display_text(&items[0].field), "field:200");
+ assert_eq!(redis_blob_display_text(&items[199].field), "field:399");
+ assert_eq!(con.command_count("HSCAN"), 1);
+ assert_eq!(con.command_count("HTTL"), 2);
+
+ let third =
+ super::load_more_collection(&mut con, b"large-hash", "hash", cursor, 200, None, None).await.unwrap();
+ let RedisCollectionPage::Hash { items, scan_cursor } = third else {
+ panic!("expected a Hash page");
+ };
+ assert_eq!(scan_cursor, None);
+ assert_eq!(items.len(), 112);
+ assert_eq!(redis_blob_display_text(&items[0].field), "field:400");
+ assert_eq!(redis_blob_display_text(&items[111].field), "field:511");
+ assert_eq!(con.command_count("HSCAN"), 1);
+ assert_eq!(con.command_count("HTTL"), 3);
+ }
+
+ #[tokio::test]
+ async fn large_hash_value_stays_bounded_when_httl_is_unsupported() {
+ let pairs = (0..512).map(|index| (format!("field:{index}"), format!("value:{index}"))).collect();
+ let unsupported = redis::RedisError::from((
+ redis::ErrorKind::ResponseError,
+ "An error was signalled by the server",
+ "unknown command 'HTTL'".to_string(),
+ ));
+ let mut con = FakeRedisConnection::with_results(vec![
+ Ok(bulk("hash")),
+ Ok(RedisRawValue::Int(-1)),
+ Ok(RedisRawValue::Int(512)),
+ Ok(hscan_response_owned("0", pairs)),
+ Err(unsupported),
+ ]);
+
+ let value = super::get_value(&mut con, b"large-hash-no-httl").await.unwrap();
+
+ let RedisValueData::Hash { items, total, scan_cursor } = value.data else {
+ panic!("expected a Hash value");
+ };
+ assert_eq!(total, 512);
+ assert_eq!(items.len(), super::COLLECTION_PAGE_SIZE);
+ assert!(scan_cursor.is_some());
+ assert!(items.iter().all(|item| item.field_ttl.is_none()));
+ assert_eq!(con.command_count("HSCAN"), 1);
+ assert_eq!(con.command_count("HTTL"), 1);
+ }
+
+ #[tokio::test]
+ async fn set_value_pages_compact_sscan_response() {
+ let members = (0..512).map(|index| format!("member:{index}")).collect();
+ let mut con = FakeRedisConnection::new(vec![
+ bulk("set"),
+ RedisRawValue::Int(-1),
+ RedisRawValue::Int(512),
+ int_scan_response_owned("0", members),
+ ]);
+
+ let value = super::get_value(&mut con, b"large-intset").await.unwrap();
+
+ let RedisValueData::Set { items, total, scan_cursor } = value.data else {
+ panic!("expected a Set value");
+ };
+ let cursor = scan_cursor.expect("overflow cursor");
+ assert!(cursor >= super::COLLECTION_OVERFLOW_CURSOR_START);
+ assert_eq!(total, 512);
+ assert_eq!(items.len(), super::COLLECTION_PAGE_SIZE);
+ assert_eq!(redis_blob_display_text(&items[0].member), "member:0");
+ assert_eq!(redis_blob_display_text(&items[199].member), "member:199");
+
+ let second =
+ super::load_more_collection(&mut con, b"large-intset", "set", cursor, 200, None, None).await.unwrap();
+ let RedisCollectionPage::Set { items, scan_cursor } = second else {
+ panic!("expected a Set page");
+ };
+ assert_eq!(scan_cursor, Some(cursor));
+ assert_eq!(items.len(), 200);
+ assert_eq!(redis_blob_display_text(&items[0].member), "member:200");
+ assert_eq!(redis_blob_display_text(&items[199].member), "member:399");
+
+ let third =
+ super::load_more_collection(&mut con, b"large-intset", "set", cursor, 200, None, None).await.unwrap();
+ let RedisCollectionPage::Set { items, scan_cursor } = third else {
+ panic!("expected a Set page");
+ };
+ assert_eq!(scan_cursor, None);
+ assert_eq!(items.len(), 112);
+ assert_eq!(redis_blob_display_text(&items[0].member), "member:400");
+ assert_eq!(redis_blob_display_text(&items[111].member), "member:511");
+ assert_eq!(con.command_count("SSCAN"), 1);
+ }
+
#[tokio::test]
async fn stream_value_uses_xlen_for_its_total_size() {
let entries = RedisRawValue::Array(vec![RedisRawValue::Array(vec![