Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(redis): fetch full page on first load more when overflow buffer holds partial items (#13102)

lxk9551e834665ffa1
diff --git a/apps/desktop/src/components/redis/RedisValueViewer.collectionSearch.spec.ts b/apps/desktop/src/components/redis/RedisValueViewer.collectionSearch.spec.ts
index 2fa5a494eb..8b88d1c781 100644
--- a/apps/desktop/src/components/redis/RedisValueViewer.collectionSearch.spec.ts
+++ b/apps/desktop/src/components/redis/RedisValueViewer.collectionSearch.spec.ts
@@ -160,6 +160,7 @@ const testI18nMessages = {
items: "{count} items",
loadedMembers: "{loaded} / {total} members loaded",
loadedItems: "{loaded} / {total} items loaded",
+ loadedFields: "{loaded} / {total} fields loaded",
loadMoreKeys: "Load more",
fields: "{count} fields",
ttlSecond: "{count}s",
@@ -704,4 +705,42 @@ describe("Redis collection member search", () => {
await settle();
expect(mocks.toast).toHaveBeenCalledWith("Collection cursor is invalid, expired, or belongs to a different connection/database", 3000);
});
+
+ it("loads 200 more items on the first load more click for a 200-item initial page (#12640 follow-up)", async () => {
+ const initialItems = Array.from({ length: 200 }, (_, i) => ({
+ field: blob(`field:${i}`),
+ value: blob(`val:${i}`),
+ }));
+ mocks.redisGetValue.mockResolvedValue({
+ key_display: "key",
+ key_raw: "key",
+ ttl: -1,
+ redis_type: "hash",
+ data: {
+ kind: "hash",
+ items: initialItems,
+ total: 500,
+ scan_cursor: 1234,
+ },
+ });
+
+ const moreItems = Array.from({ length: 200 }, (_, i) => ({
+ field: blob(`field:${200 + i}`),
+ value: blob(`val:${200 + i}`),
+ }));
+ mocks.redisLoadMore.mockResolvedValueOnce({
+ kind: "hash",
+ items: moreItems,
+ scan_cursor: 5678,
+ });
+
+ mountViewer();
+ await settle();
+ expect(document.body.textContent).toContain("200 / 500 fields loaded");
+
+ loadMoreButton().click();
+ await settle();
+ expect(mocks.redisLoadMore).toHaveBeenCalledWith("connection", 0, "key", "hash", 1234, 200, undefined, undefined);
+ expect(document.body.textContent).toContain("400 / 500 fields loaded");
+ });
});
diff --git a/crates/dbx-driver-redis/src/lib.rs b/crates/dbx-driver-redis/src/lib.rs
index 8a78737c8b..834f16d9cb 100644
--- a/crates/dbx-driver-redis/src/lib.rs
+++ b/crates/dbx-driver-redis/src/lib.rs
@@ -8,7 +8,7 @@ use redis::{
};
use serde::{Deserialize, Serialize};
use std::{
- collections::HashMap,
+ collections::{HashMap, HashSet},
fs,
future::Future,
sync::{Arc, OnceLock},
@@ -5319,25 +5319,61 @@ where
C: ConnectionLike + Send + Sync + Unpin,
{
let count = count.max(1);
- if let Some(page) = take_hash_overflow_page(scope, key, cursor, count, filter_query).await {
- return Ok(page);
+ let (buffered_cursor, mut items) = match take_hash_overflow_page(scope, key, cursor, count, filter_query).await {
+ Some((next, items)) => {
+ let has_more_buffered = {
+ let sessions = collection_overflow_sessions().lock().await;
+ match sessions.entries.get(&cursor).map(|s| &s.items) {
+ Some(RedisCollectionOverflowItems::Hash(items)) => !items.is_empty(),
+ _ => false,
+ }
+ };
+ let page_bytes: usize = items.iter().map(hash_item_page_bytes).sum();
+ if items.len() >= count || next == 0 || has_more_buffered || page_bytes >= COLLECTION_PAGE_MAX_BYTES {
+ return Ok((next, items));
+ }
+ (next, items)
+ }
+ None => (cursor, Vec::new()),
+ };
+
+ let (scan_cursor, previous_session) = if buffered_cursor == cursor {
+ take_collection_scan_cursor(scope, key, cursor, RedisCollectionOverflowKind::Hash, filter_query).await?
+ } else {
+ (buffered_cursor, None)
+ };
+
+ if cursor != 0 && scan_cursor == 0 {
+ return Ok((0, items));
}
- let (scan_cursor, previous_session) =
- take_collection_scan_cursor(scope, key, cursor, RedisCollectionOverflowKind::Hash, filter_query).await?;
+
+ let needed = count.saturating_sub(items.len()).max(1);
let scanned = if let Some(query) = filter_query {
- hscan_filtered_page_raw(con, key, scan_cursor, count, query).await
+ hscan_filtered_page_raw(con, key, scan_cursor, needed, query).await
} else {
- hscan_page_raw(con, key, scan_cursor, count, None).await
+ hscan_page_raw(con, key, scan_cursor, needed, None).await
};
- let (next_cursor, mut items) = match scanned {
+
+ let (next_cursor, scanned_items) = match scanned {
Ok(page) => page,
Err(error) => {
- if let Some(session) = previous_session {
+ if let Some(mut session) = previous_session {
+ if !items.is_empty() {
+ session.items = RedisCollectionOverflowItems::Hash(items);
+ }
collection_overflow_sessions().lock().await.reinsert(cursor, session);
}
return Err(error);
}
};
+
+ let mut seen: HashSet<String> = items.iter().map(|item| item.field.raw_base64.clone()).collect();
+ for item in scanned_items {
+ if seen.insert(item.field.raw_base64.clone()) {
+ items.push(item);
+ }
+ }
+
let page_len = budgeted_page_len(&items, count, hash_item_page_bytes);
let next_cursor = store_hash_overflow_page(scope, key, &mut items, page_len, next_cursor, filter_query).await;
Ok((next_cursor, items))
@@ -5355,25 +5391,61 @@ where
C: ConnectionLike + Send + Sync + Unpin,
{
let count = count.max(1);
- if let Some(page) = take_set_overflow_page(scope, key, cursor, count, filter_query).await {
- return Ok(page);
+ let (buffered_cursor, mut items) = match take_set_overflow_page(scope, key, cursor, count, filter_query).await {
+ Some((next, items)) => {
+ let has_more_buffered = {
+ let sessions = collection_overflow_sessions().lock().await;
+ match sessions.entries.get(&cursor).map(|s| &s.items) {
+ Some(RedisCollectionOverflowItems::Set(items)) => !items.is_empty(),
+ _ => false,
+ }
+ };
+ let page_bytes: usize = items.iter().map(set_item_page_bytes).sum();
+ if items.len() >= count || next == 0 || has_more_buffered || page_bytes >= COLLECTION_PAGE_MAX_BYTES {
+ return Ok((next, items));
+ }
+ (next, items)
+ }
+ None => (cursor, Vec::new()),
+ };
+
+ let (scan_cursor, previous_session) = if buffered_cursor == cursor {
+ take_collection_scan_cursor(scope, key, cursor, RedisCollectionOverflowKind::Set, filter_query).await?
+ } else {
+ (buffered_cursor, None)
+ };
+
+ if cursor != 0 && scan_cursor == 0 {
+ return Ok((0, items));
}
- let (scan_cursor, previous_session) =
- take_collection_scan_cursor(scope, key, cursor, RedisCollectionOverflowKind::Set, filter_query).await?;
+
+ let needed = count.saturating_sub(items.len()).max(1);
let scanned = if let Some(query) = filter_query {
- sscan_filtered_page_raw(con, key, scan_cursor, count, query).await
+ sscan_filtered_page_raw(con, key, scan_cursor, needed, query).await
} else {
- sscan_page_raw(con, key, scan_cursor, count).await
+ sscan_page_raw(con, key, scan_cursor, needed).await
};
- let (next_cursor, mut items) = match scanned {
+
+ let (next_cursor, scanned_items) = match scanned {
Ok(page) => page,
Err(error) => {
- if let Some(session) = previous_session {
+ if let Some(mut session) = previous_session {
+ if !items.is_empty() {
+ session.items = RedisCollectionOverflowItems::Set(items);
+ }
collection_overflow_sessions().lock().await.reinsert(cursor, session);
}
return Err(error);
}
};
+
+ let mut seen: HashSet<String> = items.iter().map(|item| item.member.raw_base64.clone()).collect();
+ for item in scanned_items {
+ if seen.insert(item.member.raw_base64.clone()) {
+ items.push(item);
+ }
+ }
+
let page_len = budgeted_page_len(&items, count, set_item_page_bytes);
let next_cursor = store_set_overflow_page(scope, key, &mut items, page_len, next_cursor, filter_query).await;
Ok((next_cursor, items))
@@ -5407,8 +5479,7 @@ async fn take_hash_overflow_page(
RedisCollectionOverflowItems::Hash(items) => !items.is_empty(),
RedisCollectionOverflowItems::Set(_) => false,
};
- let retain_cursor =
- has_buffered_items || (session.followup_cursor != 0 && scope != &RedisOperationScope::default());
+ let retain_cursor = has_buffered_items || session.followup_cursor != 0;
let next_cursor = if retain_cursor { cursor } else { session.followup_cursor };
if retain_cursor {
sessions.reinsert(cursor, session);
@@ -5444,8 +5515,7 @@ async fn take_set_overflow_page(
RedisCollectionOverflowItems::Set(items) => !items.is_empty(),
RedisCollectionOverflowItems::Hash(_) => false,
};
- let retain_cursor =
- has_buffered_items || (session.followup_cursor != 0 && scope != &RedisOperationScope::default());
+ let retain_cursor = has_buffered_items || session.followup_cursor != 0;
let next_cursor = if retain_cursor { cursor } else { session.followup_cursor };
if retain_cursor {
sessions.reinsert(cursor, session);
@@ -6407,6 +6477,163 @@ mod tests {
assert_eq!(con.command_count("HSCAN"), pages);
}
+ /// Regression test for #12640 follow-up:
+ /// In a Redis hash where initial HSCAN returns 201 items (COUNT is a hint), the
+ /// initial page takes 200 items and stores 1 item in the overflow session with followup_cursor != 0.
+ /// When the user clicks "Load more" (count = 200) for the first time, the driver must NOT return
+ /// just the 1 buffered item (200 -> 201). Instead, it must drain the 1 buffered item, continue
+ /// scanning the server for the remaining needed items, and return a full page of 200 items (200 -> 400).
+ #[tokio::test]
+ async fn scoped_hash_pagination_first_load_more_with_overflow_reaches_full_page() {
+ const TOTAL_FIELDS: usize = 500;
+ const PAGE: usize = 200;
+ let scope = super::RedisOperationScope { connection_id: "issue-12640-first-load-more-hash".to_string(), db: 0 };
+
+ let initial_pairs: Vec<(String, String)> =
+ (0..201).map(|i| (format!("field:{i}"), format!("value:{i}"))).collect();
+ let second_pairs: Vec<(String, String)> =
+ (201..401).map(|i| (format!("field:{i}"), format!("value:{i}"))).collect();
+ let third_pairs: Vec<(String, String)> =
+ (401..500).map(|i| (format!("field:{i}"), format!("value:{i}"))).collect();
+
+ let responses = vec![
+ bulk("hash"),
+ RedisRawValue::Int(-1),
+ RedisRawValue::Int(TOTAL_FIELDS as i64),
+ hscan_response_owned("100", initial_pairs),
+ httl_response(PAGE, -1),
+ hscan_response_owned("200", second_pairs),
+ httl_response(PAGE, -1),
+ hscan_response_owned("0", third_pairs),
+ httl_response(100, -1),
+ ];
+ let mut con = FakeRedisConnection::new(responses);
+
+ // Initial read: 200 items
+ let value = super::get_value_scoped(&mut con, b"hash-key", &scope).await.unwrap();
+ let RedisValueData::Hash { items: first_page, scan_cursor, total } = value.data else {
+ panic!("expected hash");
+ };
+ assert_eq!(total, TOTAL_FIELDS as u64);
+ assert_eq!(first_page.len(), PAGE);
+ assert_eq!(redis_blob_display_text(&first_page[0].field), "field:0");
+ assert_eq!(redis_blob_display_text(&first_page[199].field), "field:199");
+ let cursor = scan_cursor.expect("must offer continuation");
+ assert!(cursor >= super::COLLECTION_OVERFLOW_CURSOR_START);
+
+ let mut seen = first_page;
+
+ // First Load More: MUST return a full page of 200 items (NOT 1 item)!
+ let second_page =
+ super::load_more_collection_scoped(&mut con, b"hash-key", "hash", cursor, PAGE, None, None, &scope)
+ .await
+ .unwrap();
+ let RedisCollectionPage::Hash { items: p2_items, scan_cursor: c2 } = second_page else {
+ panic!("expected hash page");
+ };
+ assert_eq!(p2_items.len(), PAGE, "first load_more must return full page of 200 items, not +1");
+ assert_eq!(redis_blob_display_text(&p2_items[0].field), "field:200");
+ assert_eq!(redis_blob_display_text(&p2_items[199].field), "field:399");
+ seen.extend(p2_items);
+ assert_eq!(seen.len(), 400, "total loaded after first load_more must be 400 (200 -> 400)");
+
+ let cursor2 = c2.expect("must offer second continuation");
+ assert!(cursor2 >= super::COLLECTION_OVERFLOW_CURSOR_START);
+
+ // Second Load More: returns remaining 100 items (400..499)
+ let third_page =
+ super::load_more_collection_scoped(&mut con, b"hash-key", "hash", cursor2, PAGE, None, None, &scope)
+ .await
+ .unwrap();
+ let RedisCollectionPage::Hash { items: p3_items, scan_cursor: c3 } = third_page else {
+ panic!("expected hash page");
+ };
+ assert_eq!(p3_items.len(), 100);
+ assert_eq!(redis_blob_display_text(&p3_items[0].field), "field:400");
+ assert_eq!(redis_blob_display_text(&p3_items[99].field), "field:499");
+ seen.extend(p3_items);
+ assert_eq!(seen.len(), TOTAL_FIELDS);
+ assert_eq!(c3, None, "walk is complete");
+
+ let unique: std::collections::HashSet<_> = seen.iter().map(|item| item.field.raw_base64.clone()).collect();
+ assert_eq!(unique.len(), TOTAL_FIELDS, "every field arrives exactly once");
+ assert_eq!(con.command_count("HSCAN"), 3);
+ }
+
+ /// Regression test for #12640 follow-up on Redis sets:
+ /// SSCAN returning 201 items on initial load results in 1 item buffered in overflow session.
+ /// First load_more must return a full page of 200 items (200 -> 400).
+ #[tokio::test]
+ async fn scoped_set_pagination_first_load_more_with_overflow_reaches_full_page() {
+ const TOTAL_MEMBERS: usize = 500;
+ const PAGE: usize = 200;
+ let scope = super::RedisOperationScope { connection_id: "issue-12640-first-load-more-set".to_string(), db: 0 };
+
+ let initial_members: Vec<String> = (0..201).map(|i| format!("member:{i}")).collect();
+ let second_members: Vec<String> = (201..401).map(|i| format!("member:{i}")).collect();
+ let third_members: Vec<String> = (401..500).map(|i| format!("member:{i}")).collect();
+
+ let responses = vec![
+ bulk("set"),
+ RedisRawValue::Int(-1),
+ RedisRawValue::Int(TOTAL_MEMBERS as i64),
+ int_scan_response_owned("100", initial_members),
+ int_scan_response_owned("200", second_members),
+ int_scan_response_owned("0", third_members),
+ ];
+ let mut con = FakeRedisConnection::new(responses);
+
+ // Initial read: 200 items
+ let value = super::get_value_scoped(&mut con, b"set-key", &scope).await.unwrap();
+ let RedisValueData::Set { items: first_page, scan_cursor, total } = value.data else {
+ panic!("expected set");
+ };
+ assert_eq!(total, TOTAL_MEMBERS as u64);
+ assert_eq!(first_page.len(), PAGE);
+ assert_eq!(redis_blob_display_text(&first_page[0].member), "member:0");
+ assert_eq!(redis_blob_display_text(&first_page[199].member), "member:199");
+ let cursor = scan_cursor.expect("must offer continuation");
+ assert!(cursor >= super::COLLECTION_OVERFLOW_CURSOR_START);
+
+ let mut seen = first_page;
+
+ // First Load More: MUST return a full page of 200 items (NOT 1 item)!
+ let second_page =
+ super::load_more_collection_scoped(&mut con, b"set-key", "set", cursor, PAGE, None, None, &scope)
+ .await
+ .unwrap();
+ let RedisCollectionPage::Set { items: p2_items, scan_cursor: c2 } = second_page else {
+ panic!("expected set page");
+ };
+ assert_eq!(p2_items.len(), PAGE, "first load_more must return full page of 200 items, not +1");
+ assert_eq!(redis_blob_display_text(&p2_items[0].member), "member:200");
+ assert_eq!(redis_blob_display_text(&p2_items[199].member), "member:399");
+ seen.extend(p2_items);
+ assert_eq!(seen.len(), 400, "total loaded after first load_more must be 400 (200 -> 400)");
+
+ let cursor2 = c2.expect("must offer second continuation");
+ assert!(cursor2 >= super::COLLECTION_OVERFLOW_CURSOR_START);
+
+ // Second Load More: returns remaining 100 items (400..499)
+ let third_page =
+ super::load_more_collection_scoped(&mut con, b"set-key", "set", cursor2, PAGE, None, None, &scope)
+ .await
+ .unwrap();
+ let RedisCollectionPage::Set { items: p3_items, scan_cursor: c3 } = third_page else {
+ panic!("expected set page");
+ };
+ assert_eq!(p3_items.len(), 100);
+ assert_eq!(redis_blob_display_text(&p3_items[0].member), "member:400");
+ assert_eq!(redis_blob_display_text(&p3_items[99].member), "member:499");
+ seen.extend(p3_items);
+ assert_eq!(seen.len(), TOTAL_MEMBERS);
+ assert_eq!(c3, None, "walk is complete");
+
+ let unique: std::collections::HashSet<_> = seen.iter().map(|item| item.member.raw_base64.clone()).collect();
+ assert_eq!(unique.len(), TOTAL_MEMBERS, "every member arrives exactly once");
+ assert_eq!(con.command_count("SSCAN"), 3);
+ }
+
#[tokio::test]
async fn set_overflow_rejects_unknown_and_foreign_tokens_without_server_scan() {
let scope = super::RedisOperationScope { connection_id: "set-scope-test".to_string(), db: 1 };
@@ -6522,6 +6749,82 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn partial_overflow_fill_restores_buffer_after_scan_failure() {
+ for key_type in ["hash", "set"] {
+ for connection_id in ["", "retry-partial-overflow"] {
+ let scope = super::RedisOperationScope { connection_id: connection_id.into(), db: 0 };
+ let response = if key_type == "hash" {
+ hscan_response("128", vec![("one", "value"), ("two", "value")])
+ } else {
+ scan_response("128", vec!["one", "two"])
+ };
+ let mut first = FakeRedisConnection::new(vec![response, httl_response(1, -1)]);
+ let page =
+ super::load_more_collection_scoped(&mut first, b"retry-key", key_type, 0, 1, None, None, &scope)
+ .await
+ .unwrap();
+ let cursor = match page {
+ RedisCollectionPage::Hash { scan_cursor, .. } | RedisCollectionPage::Set { scan_cursor, .. } => {
+ scan_cursor.unwrap()
+ }
+ _ => panic!("expected collection"),
+ };
+ let mut failed = FakeRedisConnection::with_results(vec![Err(redis::RedisError::from((
+ redis::ErrorKind::Io,
+ "temporary scan failure",
+ )))]);
+ assert!(super::load_more_collection_scoped(
+ &mut failed,
+ b"retry-key",
+ key_type,
+ cursor,
+ 2,
+ None,
+ None,
+ &scope
+ )
+ .await
+ .is_err());
+ assert!(
+ super::collection_overflow_sessions().lock().await.entries.contains_key(&cursor),
+ "lost {key_type} buffer in scope {connection_id:?}"
+ );
+ let response = if key_type == "hash" {
+ hscan_response("0", vec![("three", "value")])
+ } else {
+ scan_response("0", vec!["three"])
+ };
+ let mut retry = FakeRedisConnection::new(vec![response, httl_response(2, -1)]);
+ let page = super::load_more_collection_scoped(
+ &mut retry,
+ b"retry-key",
+ key_type,
+ cursor,
+ 2,
+ None,
+ None,
+ &scope,
+ )
+ .await
+ .unwrap();
+ let (names, next) = match page {
+ RedisCollectionPage::Hash { items, scan_cursor } => {
+ (items.iter().map(|item| redis_blob_display_text(&item.field)).collect::<Vec<_>>(), scan_cursor)
+ }
+ RedisCollectionPage::Set { items, scan_cursor } => (
+ items.iter().map(|item| redis_blob_display_text(&item.member)).collect::<Vec<_>>(),
+ scan_cursor,
+ ),
+ _ => panic!("expected collection"),
+ };
+ assert_eq!(names, vec!["two", "three"]);
+ assert_eq!(next, None);
+ assert!(retry.commands[0].contains("\r\n128\r\n"));
+ }
+ }
+ }
+
#[tokio::test]
async fn scoped_collection_overflow_keeps_followup_scoped_and_retries_failed_scan() {
for key_type in ["hash", "set"] {