Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(rocketmq): paginate topic consumer groups past 200

jearhe35946427a689
diff --git a/crates/dbx-core/src/mq/adapters/rocketmq.rs b/crates/dbx-core/src/mq/adapters/rocketmq.rs
index 9fadb22348..d5cabd8018 100644
--- a/crates/dbx-core/src/mq/adapters/rocketmq.rs
+++ b/crates/dbx-core/src/mq/adapters/rocketmq.rs
@@ -144,6 +144,13 @@ impl RocketMqAdmin {
})
.await
}
+
+ async fn list_all_topic_consumer_groups(&self, topic: &str, enrich: bool) -> Result<Vec<SubscriptionInfo>, String> {
+ collect_paginated_consumer_group_pages(|offset| {
+ self.call("mq_list_consumer_groups", rocketmq_topic_consumer_group_page_params(topic, offset, enrich))
+ })
+ .await
+ }
}
#[async_trait]
@@ -380,20 +387,7 @@ impl MessageQueueAdmin for RocketMqAdmin {
}
// Topic path: skip list enrich; batch lag in one agent RPC (no N+1).
- let result: serde_json::Value = self
- .call(
- "mq_list_consumer_groups",
- serde_json::json!({
- "topic": topic.topic,
- "limit": 200,
- "offset": 0,
- "enrich": false,
- "includeLag": true,
- }),
- )
- .await?;
- let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
- Ok(groups.iter().map(rocketmq_subscription_from_group).collect())
+ self.list_all_topic_consumer_groups(&topic.topic, false).await
}
async fn enrich_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> {
@@ -403,19 +397,7 @@ impl MessageQueueAdmin for RocketMqAdmin {
}
// Topic path: skip list enrich; batch lag in one agent RPC (no N+1).
- let mut params = serde_json::json!({
- "limit": 500,
- "offset": 0,
- "enrich": true,
- });
- if !topic.topic.is_empty() {
- params["topic"] = serde_json::json!(topic.topic);
- params["limit"] = serde_json::json!(200);
- params["includeLag"] = serde_json::json!(true);
- }
- let result: serde_json::Value = self.call("mq_list_consumer_groups", params).await?;
- let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
- Ok(groups.iter().map(rocketmq_subscription_from_group).collect())
+ self.list_all_topic_consumer_groups(&topic.topic, true).await
}
async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> {
@@ -793,6 +775,16 @@ fn rocketmq_cluster_consumer_group_page_params(offset: usize, enrich: bool) -> s
})
}
+fn rocketmq_topic_consumer_group_page_params(topic: &str, offset: usize, enrich: bool) -> serde_json::Value {
+ serde_json::json!({
+ "topic": topic,
+ "limit": CONSUMER_GROUP_LIST_PAGE_SIZE,
+ "offset": offset,
+ "enrich": enrich,
+ "includeLag": true,
+ })
+}
+
/// Collect all pages from the agent while preserving its sorted order.
///
/// The offset advances by the number of rows actually returned, rather than the
@@ -1465,6 +1457,37 @@ mod tests {
assert_eq!(enriched.get("enrich").and_then(|value| value.as_bool()), Some(true));
}
+ #[tokio::test]
+ async fn topic_consumer_group_pagination_loads_list_and_enrich_pages() {
+ let groups: Vec<serde_json::Value> = (1..=201)
+ .map(|index| serde_json::json!({ "groupId": format!("group-{index:04}"), "groupType": "NORMAL" }))
+ .collect();
+ let total = groups.len();
+
+ for enrich in [false, true] {
+ let mut offsets = Vec::new();
+ let subscriptions = collect_paginated_consumer_group_pages(|offset| {
+ offsets.push(offset);
+ let params = rocketmq_topic_consumer_group_page_params("orders", offset, enrich);
+ assert_eq!(params.get("topic").and_then(|value| value.as_str()), Some("orders"));
+ assert_eq!(params.get("limit").and_then(|value| value.as_i64()), Some(500));
+ assert_eq!(params.get("offset").and_then(|value| value.as_u64()), Some(offset as u64));
+ assert_eq!(params.get("enrich").and_then(|value| value.as_bool()), Some(enrich));
+ assert_eq!(params.get("includeLag").and_then(|value| value.as_bool()), Some(true));
+
+ let end = offset.saturating_add(200).min(total);
+ let page = groups[offset..end].to_vec();
+ async move { Ok(serde_json::json!({ "groups": page, "total": total })) }
+ })
+ .await
+ .expect("topic consumer group pages should be collected");
+
+ assert_eq!(offsets, vec![0, 200]);
+ assert_eq!(subscriptions.len(), 201);
+ assert_eq!(subscriptions[200].name, "group-0201");
+ }
+ }
+
#[tokio::test]
async fn consumer_group_pagination_loads_groups_after_first_page() {
let groups: Vec<serde_json::Value> = (1..=1000)