Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(kafka): preserve consumer groups on metadata timeouts

Zzz4cb47ec9d069
diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
index 4e64879019..b3aaa844bd 100644
--- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
+++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
@@ -160,8 +160,8 @@ public final class KafkaAgent {
case "mq_get_topic_config" -> getTopicConfig(params);
case "mq_alter_topic_config" -> alterTopicConfig(params);
// Consumer groups
- case "mq_list_consumer_groups" -> listConsumerGroups(params);
- case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params);
+ case "mq_list_consumer_groups" -> listConsumerGroups(params, requireAdmin());
+ case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params, requireAdmin());
case "mq_describe_consumer_group" -> describeConsumerGroup(params);
case "mq_delete_consumer_group" -> deleteConsumerGroup(params);
case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params);
@@ -1035,8 +1035,7 @@ public final class KafkaAgent {
return descriptions;
}
- private static Object listConsumerGroups(JsonObject params) throws Exception {
- AdminClient admin = requireAdmin();
+ static Object listConsumerGroups(JsonObject params, Admin admin) throws Exception {
int timeout = requestTimeout(params);
long deadline = System.currentTimeMillis() + timeout;
String filterTopic = stringOrEmpty(params, "topic");
@@ -1053,15 +1052,8 @@ public final class KafkaAgent {
return Collections.singletonMap("groups", Collections.emptyList());
}
- // Batch-describe every group in one Kafka Admin request. Per-group
- // failures degrade to an empty member list instead of failing the batch.
- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
- groupIds,
- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
- );
- Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));
-
- // Batch-resolve committed offsets for every group in one Admin request.
+ // Start both independent lookups before waiting. A slow description must
+ // not consume the deadline before committed offsets can even be requested.
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
@@ -1071,6 +1063,10 @@ public final class KafkaAgent {
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))
);
+ DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
+ groupIds,
+ new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
+ );
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
@@ -1082,6 +1078,7 @@ public final class KafkaAgent {
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
+ Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));
// End offsets: one batched Admin request limited to the requested
// topic's partitions (deduplicated across groups). Without a topic
@@ -1137,6 +1134,7 @@ public final class KafkaAgent {
g.put("groupId", groupId);
g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
+ g.put("membersAvailable", description != null);
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
g.put("committedOffsets", offsetRows(committed, filterTopic));
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
@@ -1207,8 +1205,7 @@ public final class KafkaAgent {
return rows;
}
- private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
- AdminClient admin = requireAdmin();
+ static Object getConsumerGroupSnapshot(JsonObject params, Admin admin) throws Exception {
int timeout = requestTimeout(params);
long deadline = System.currentTimeMillis() + timeout;
Collection<ConsumerGroupListing> listings = admin.listConsumerGroups(
@@ -1229,12 +1226,8 @@ public final class KafkaAgent {
Map<String, Boolean> committedOffsetsAvailable = new HashMap<>();
Map<String, LinkedHashSet<String>> errors = new HashMap<>();
- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
- groupIds,
- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
- );
- descriptions.putAll(awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));
-
+ // Offset and description requests share one deadline, but neither needs
+ // the other's result. Launch both before awaiting per-group responses.
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
@@ -1243,6 +1236,10 @@ public final class KafkaAgent {
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))
);
+ DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
+ groupIds,
+ new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
+ );
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> groupOffsets = offsetsResult
@@ -1256,6 +1253,7 @@ public final class KafkaAgent {
snapshotErrors(errors, groupId).add("Committed offsets unavailable: " + normalizeErrorMessage(error));
}
}
+ descriptions.putAll(awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));
Map<String, Set<TopicPartition>> assignedPartitions = new HashMap<>();
Set<TopicPartition> allPartitions = new LinkedHashSet<>();
diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
index 9bec43c3c7..0507f4b628 100644
--- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
+++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
@@ -9,6 +9,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.gson.JsonParser;
+import com.google.gson.Gson;
+import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -25,6 +27,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Optional;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.CountDownLatch;
@@ -36,6 +41,13 @@ import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.admin.AlterConfigOp;
+import org.apache.kafka.clients.admin.Admin;
+import org.apache.kafka.clients.admin.ConsumerGroupListing;
+import org.apache.kafka.clients.admin.ListConsumerGroupsResult;
+import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsOptions;
+import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult;
+import org.apache.kafka.clients.admin.ListOffsetsResult;
+import org.apache.kafka.clients.admin.internals.CoordinatorKey;
import org.apache.kafka.clients.admin.ConsumerGroupDescription;
import org.apache.kafka.clients.admin.DescribeConsumerGroupsResult;
import org.apache.kafka.clients.admin.Config;
@@ -44,6 +56,7 @@ import org.apache.kafka.clients.admin.RaftVoterEndpoint;
import org.apache.kafka.clients.admin.TopicDescription;
import org.apache.kafka.clients.admin.TopicListing;
import org.apache.kafka.common.KafkaFuture;
+import org.apache.kafka.common.ConsumerGroupState;
import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
@@ -59,6 +72,8 @@ import org.apache.zookeeper.client.ZKClientConfig;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
import org.apache.zookeeper.server.ZooKeeperServer;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.junit.jupiter.api.io.TempDir;
class KafkaAgentTest {
@@ -168,6 +183,85 @@ class KafkaAgentTest {
assertTrue(elapsed < 1000, "waited " + elapsed + "ms; five groups must share one 300ms deadline");
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void consumerGroupQueriesKeepCommittedTopicsWhenDescriptionsTimeOut(boolean snapshot) throws Exception {
+ TopicPartition partition = new TopicPartition("orders", 0);
+ ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
+ try {
+ Admin admin = (Admin) Proxy.newProxyInstance(Admin.class.getClassLoader(), new Class<?>[] {Admin.class},
+ (proxy, method, args) -> {
+ return switch (method.getName()) {
+ case "listConsumerGroups" -> {
+ var constructor = ListConsumerGroupsResult.class.getDeclaredConstructor(KafkaFuture.class);
+ constructor.setAccessible(true);
+ yield constructor.newInstance(KafkaFuture.completedFuture(List.of(
+ new ConsumerGroupListing("orders-service", false),
+ new ConsumerGroupListing("slow-group", false)
+ )));
+ }
+ case "describeConsumerGroups" -> new DescribeConsumerGroupsResult(Map.of(
+ "orders-service", KafkaFuture.completedFuture(new ConsumerGroupDescription(
+ "orders-service", false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()
+ )),
+ "slow-group", new KafkaFutureImpl<>()
+ ));
+ case "listConsumerGroupOffsets" -> {
+ // Model an offsets response that needs a small amount of network time.
+ // If descriptions consumed the deadline before this request was sent,
+ // the broker cannot return even these independently available offsets.
+ int timeout = ((ListConsumerGroupOffsetsOptions) args[1]).timeoutMs();
+ KafkaFutureImpl<Map<TopicPartition, OffsetAndMetadata>> offsets = new KafkaFutureImpl<>();
+ executor.schedule(() -> {
+ if (timeout < 50) {
+ offsets.completeExceptionally(new org.apache.kafka.common.errors.TimeoutException("offsets deadline expired"));
+ } else {
+ offsets.complete(Map.of(partition, new OffsetAndMetadata(3)));
+ }
+ }, Math.min(50, timeout), TimeUnit.MILLISECONDS);
+ var constructor = ListConsumerGroupOffsetsResult.class.getDeclaredConstructor(Map.class);
+ constructor.setAccessible(true);
+ yield constructor.newInstance(Map.of(
+ CoordinatorKey.byGroupId("orders-service"), offsets,
+ CoordinatorKey.byGroupId("slow-group"), offsets
+ ));
+ }
+ case "listOffsets" -> new ListOffsetsResult(Map.of(
+ partition, KafkaFuture.completedFuture(new ListOffsetsResult.ListOffsetsResultInfo(5, -1, Optional.empty()))
+ ));
+ default -> throw new UnsupportedOperationException(method.getName());
+ };
+ });
+ JsonObject params = new JsonObject();
+ params.addProperty("timeout_ms", 300);
+ params.addProperty("topic", "orders");
+ Object result = snapshot
+ ? KafkaAgent.getConsumerGroupSnapshot(params, admin)
+ : KafkaAgent.listConsumerGroups(params, admin);
+ var groups = new Gson().toJsonTree(result).getAsJsonObject().getAsJsonArray("groups");
+ assertEquals(2, groups.size());
+ for (var row : groups) {
+ JsonObject group = row.getAsJsonObject();
+ if (snapshot) {
+ assertEquals(1, group.getAsJsonArray("topics").size(), "committed topic survives description timeout");
+ assertEquals("orders", group.getAsJsonArray("topics").get(0).getAsString());
+ assertEquals(3, group.getAsJsonArray("partitions").get(0).getAsJsonObject().get("currentOffset").getAsInt());
+ if (group.get("groupId").getAsString().equals("slow-group")) {
+ assertTrue(group.get("error").getAsString().contains("Consumer group details unavailable"));
+ assertFalse(group.has("memberCount"));
+ }
+ } else {
+ assertEquals(!group.get("groupId").getAsString().equals("slow-group"), group.get("membersAvailable").getAsBoolean());
+ assertEquals(1, group.getAsJsonArray("committedOffsets").size(), "committed offsets survive description timeout");
+ assertEquals("orders", group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("topic").getAsString());
+ assertEquals(3, group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("offset").getAsInt());
+ }
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
@Test
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
diff --git a/apps/desktop/src/components/mq/SubscriptionsPanel.vue b/apps/desktop/src/components/mq/SubscriptionsPanel.vue
index 728d40650b..b3761078a9 100644
--- a/apps/desktop/src/components/mq/SubscriptionsPanel.vue
+++ b/apps/desktop/src/components/mq/SubscriptionsPanel.vue
@@ -609,6 +609,7 @@ onBeforeUnmount(() => {
<span v-if="sub.onlineMembers == null" class="text-muted">-</span>
<span v-else>{{ sub.onlineMembers }}</span>
</template>
+ <span v-else-if="mqSystemKind === 'kafka' && sub.onlineMembers == null" class="text-muted">-</span>
<template v-else>{{ t("mqSubscriptions.consumerCount", { count: sub.consumers.length }) }}</template>
</td>
<td class="actions">
diff --git a/apps/desktop/src/components/mq/__tests__/SubscriptionsPanel.spec.ts b/apps/desktop/src/components/mq/__tests__/SubscriptionsPanel.spec.ts
index f793e3e252..088f9c80a0 100644
--- a/apps/desktop/src/components/mq/__tests__/SubscriptionsPanel.spec.ts
+++ b/apps/desktop/src/components/mq/__tests__/SubscriptionsPanel.spec.ts
@@ -17,7 +17,7 @@ const backend = vi.hoisted(() => ({
}));
vi.mock("vue-i18n", () => ({
- useI18n: () => ({ t: (key: string) => key }),
+ useI18n: () => ({ t: (key: string, params?: { count?: number }) => (key === "mqSubscriptions.consumerCount" ? `${params?.count}` : key) }),
}));
vi.mock("@/lib/backend/api", () => backend);
@@ -139,6 +139,24 @@ afterEach(() => {
root = null;
});
+describe("SubscriptionsPanel Kafka partial results", () => {
+ it.each([undefined, 0, 1])("distinguishes unavailable members from a known count of %s", async (onlineMembers) => {
+ backend.mqListSubscriptions.mockResolvedValue([
+ {
+ ...subscription("orders-service", "consumer-group"),
+ onlineMembers,
+ backlogUnavailable: onlineMembers == null,
+ consumers: Array.from({ length: onlineMembers ?? 0 }, () => ({ consumerName: "consumer-orders" })),
+ },
+ ]);
+ const panel = await mountPanel();
+ const row = panel.querySelector(".subscriptions-table tbody tr")!;
+
+ expect(row.querySelector('[data-testid="online-members"]')?.textContent?.trim()).toBe(onlineMembers == null ? "-" : `${onlineMembers}`);
+ expect(row.querySelectorAll("td")[2].textContent?.trim()).toBe(onlineMembers == null ? "-" : "0");
+ });
+});
+
describe("SubscriptionsPanel message peek", () => {
it("warns when a Kafka peek is incomplete and clears the warning after a complete refresh", async () => {
backend.mqPeekMessages
diff --git a/apps/desktop/src/types/mq.ts b/apps/desktop/src/types/mq.ts
index c5c49bffb8..818d635836 100644
--- a/apps/desktop/src/types/mq.ts
+++ b/apps/desktop/src/types/mq.ts
@@ -238,7 +238,7 @@ export interface SubscriptionInfo {
consumers: ConsumerInfo[];
/** RocketMQ subscribed topics (cluster-wide listing). */
topics?: string[];
- /** RocketMQ online consumer client count. */
+ /** Online clients for RocketMQ groups or Kafka topic subscriptions; absent when unavailable. */
onlineMembers?: number;
/** RocketMQ consumer group delivery type: NORMAL / FIFO / SYSTEM. */
consumerGroupType?: string;
diff --git a/crates/dbx-core/src/admin/mq/adapters/kafka.rs b/crates/dbx-core/src/admin/mq/adapters/kafka.rs
index c09ed0bc00..e7998cf47b 100644
--- a/crates/dbx-core/src/admin/mq/adapters/kafka.rs
+++ b/crates/dbx-core/src/admin/mq/adapters/kafka.rs
@@ -280,8 +280,12 @@ impl MessageQueueAdmin for KafkaAdmin {
// handful of Kafka Admin calls. Subscriptions are filtered in memory below.
// (The previous 1 + 2N serial RPC pattern — describe + lag per group — took
// tens of seconds on remote clusters with many groups, see #7163.)
- let result: serde_json::Value =
- self.call("mq_list_consumer_groups", serde_json::json!({ "topic": topic.topic })).await?;
+ let result: serde_json::Value = self
+ .call_with_agent_timeout(
+ "mq_list_consumer_groups",
+ serde_json::json!({ "topic": topic.topic, "timeout_ms": self.config.request_timeout_ms() }),
+ )
+ .await?;
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
// For each group, check both active assignments and committed offsets.
@@ -296,7 +300,8 @@ impl MessageQueueAdmin for KafkaAdmin {
}
async fn get_kafka_consumer_group_snapshot(&self) -> Result<KafkaConsumerGroupSnapshot, String> {
- self.call("mq_get_consumer_group_snapshot", consumer_group_snapshot_params(&self.config)).await
+ self.call_with_agent_timeout("mq_get_consumer_group_snapshot", consumer_group_snapshot_params(&self.config))
+ .await
}
async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> {
@@ -806,10 +811,8 @@ fn kafka_subscription_from_group_row(group_id: &str, topic: &str, row: &serde_js
})
.unwrap_or_default();
- // Sum lag over the topic's committed partitions. A partition without a
- // resolved end offset contributes nothing (matches the agent's lag probe:
- // unknown end offsets are not reported as zero lag).
- let msg_backlog = committed
+ // Keep the known lag, but do not present an incomplete probe as healthy zero.
+ let partition_lags = committed
.iter()
.filter_map(|partition| {
let partition_id = partition.get("partition").and_then(|v| v.as_i64())?;
@@ -817,12 +820,17 @@ fn kafka_subscription_from_group_row(group_id: &str, topic: &str, row: &serde_js
let end_offset = end_offsets.get(&partition_id).copied()?;
Some((end_offset - committed_offset).max(0))
})
- .sum();
+ .collect::<Vec<_>>();
+ let backlog_unavailable = committed.is_empty() || partition_lags.len() != committed.len();
+ let online_members =
+ (row.get("membersAvailable").and_then(|v| v.as_bool()) != Some(false)).then_some(consumers.len() as u32);
Some(SubscriptionInfo {
name: group_id.to_string(),
sub_type: "consumer-group".to_string(),
- msg_backlog,
+ msg_backlog: partition_lags.iter().sum(),
+ backlog_unavailable: Some(backlog_unavailable),
+ online_members,
msg_rate_out: 0.0,
msg_throughput_out: 0.0,
consumers,
@@ -1216,6 +1224,8 @@ mod tests {
assert_eq!(sub.name, "orders-service");
assert_eq!(sub.sub_type, "consumer-group");
assert_eq!(sub.msg_backlog, 11); // (10-3) + (12-8)
+ assert_eq!(sub.backlog_unavailable, Some(false));
+ assert_eq!(sub.online_members, Some(0));
assert!(sub.consumers.is_empty());
}
@@ -1251,6 +1261,8 @@ mod tests {
assert_eq!(sub.name, "live-service");
assert_eq!(sub.msg_backlog, 0);
assert_eq!(sub.consumers.len(), 1);
+ assert_eq!(sub.online_members, Some(1));
+ assert_eq!(sub.backlog_unavailable, Some(true));
assert_eq!(sub.consumers[0].consumer_name, "consumer-events");
assert_eq!(sub.consumers[0].address, "/10.0.0.10");
}
@@ -1298,6 +1310,23 @@ mod tests {
.expect("the group still has visible committed offsets");
assert_eq!(sub.msg_backlog, 7); // only partition 0 contributes
+ assert_eq!(sub.backlog_unavailable, Some(true));
+ }
+
+ #[test]
+ fn kafka_subscription_from_group_row_preserves_unknown_members_and_backlog() {
+ let row = serde_json::json!({
+ "groupId": "orders-service",
+ "membersAvailable": false,
+ "members": [],
+ "committedOffsets": [{ "topic": "orders", "partition": 0, "offset": 3 }],
+ "endOffsets": []
+ });
+
+ let sub = kafka_subscription_from_group_row("orders-service", "orders", &row).unwrap();
+
+ assert_eq!(sub.online_members, None);
+ assert_eq!(sub.backlog_unavailable, Some(true));
}
#[test]
@@ -1341,6 +1370,97 @@ mod tests {
assert!(!partitions.map(|p| p > 0).unwrap_or(false));
}
+ async fn delayed_consumer_group_admin() -> (KafkaAdmin, tempfile::TempPath) {
+ let script_path = tempfile::NamedTempFile::new().expect("create test agent script").into_temp_path();
+ std::fs::write(
+ &script_path,
+ r#"import json
+import sys
+import time
+
+print(json.dumps({"ready": True}), flush=True)
+for line in sys.stdin:
+ request = json.loads(line)
+ method = request["method"]
+ if method in ("mq_get_consumer_group_snapshot", "mq_list_consumer_groups"):
+ assert request["params"]["timeout_ms"] == 1000, "forward the configured agent deadline"
+ time.sleep(1.2)
+ if method == "mq_list_consumer_groups" and request["params"].get("topic") == "denied":
+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"],
+ "error": {"code": -1, "message": "Group authorization failed"}}), flush=True)
+ continue
+ if method == "mq_get_consumer_group_snapshot":
+ result = {"groups": [
+ {"groupId": "orders-service", "state": "EMPTY", "topics": ["orders"],
+ "totalLag": 2, "lagAvailable": True},
+ {"groupId": "unreachable", "state": "UNKNOWN", "lagAvailable": False,
+ "error": "Consumer group details unavailable: timed out"},
+ ]}
+ elif method == "mq_list_consumer_groups":
+ result = {"groups": [
+ {"groupId": "orders-service", "members": [],
+ "committedOffsets": [{"topic": "orders", "partition": 0, "offset": 3}],
+ "endOffsets": [{"topic": "orders", "partition": 0, "offset": 5}]},
+ {"groupId": "unreachable", "members": [], "committedOffsets": []},
+ ]}
+ elif method == "mq_list_topics":
+ result = {"topics": [{"name": "orders", "internal": False}]}
+ else:
+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"],
+ "error": {"code": -1, "message": "Group authorization failed"}}), flush=True)
+ continue
+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True)
+"#,
+ )
+ .expect("write test agent script");
+ let python = if cfg!(windows) { "python" } else { "python3" };
+ let client = AgentDriverClient::spawn(
+ AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),
+ )
+ .await
+ .expect("spawn test agent");
+ let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
+ config.query_timeout_secs = 1;
+ (KafkaAdmin { client: Arc::new(Mutex::new(client)), config }, script_path)
+ }
+
+ #[tokio::test]
+ async fn consumer_group_snapshot_does_not_preempt_agent_partial_results() {
+ let (admin, _script) = delayed_consumer_group_admin().await;
+ let snapshot =
+ admin.get_kafka_consumer_group_snapshot().await.expect("receive the bounded agent's partial results");
+ assert_eq!(snapshot.groups.len(), 2);
+ assert_eq!(snapshot.groups[0].group_id, "orders-service");
+ assert_eq!(snapshot.groups[0].total_lag, Some(2));
+ assert!(!snapshot.groups[1].lag_available);
+ assert_eq!(snapshot.groups[1].error.as_deref(), Some("Consumer group details unavailable: timed out"));
+
+ // The deadline must not kill the shared agent and break the next tab's request.
+ let topics = admin
+ .list_topics(
+ &NamespaceRef { tenant: "_kafka".to_string(), namespace: "_kafka".to_string() },
+ ListTopicsOpts::default(),
+ )
+ .await
+ .expect("agent remains usable after a partial snapshot");
+ assert_eq!(topics[0].name, "orders");
+ }
+
+ #[tokio::test]
+ async fn topic_subscriptions_do_not_preempt_agent_partial_results() {
+ let (admin, _script) = delayed_consumer_group_admin().await;
+ let subscriptions = admin
+ .list_subscriptions(&TopicRef { topic: "orders".to_string(), ..topic_ref() })
+ .await
+ .expect("receive topic subscriptions");
+ assert_eq!(subscriptions.len(), 1);
+ assert_eq!(subscriptions[0].name, "orders-service");
+ assert_eq!(subscriptions[0].msg_backlog, 2);
+ let error =
+ admin.list_subscriptions(&TopicRef { topic: "denied".to_string(), ..topic_ref() }).await.unwrap_err();
+ assert!(error.contains("Group authorization failed"), "agent errors must remain visible: {error}");
+ }
+
#[tokio::test]
async fn topic_listing_does_not_preempt_agent_timeout_fallback() {
let script_path = std::env::temp_dir().join(format!("dbx-kafka-topic-timeout-{}.py", uuid::Uuid::new_v4()));
diff --git a/crates/dbx-core/src/admin/mq/types.rs b/crates/dbx-core/src/admin/mq/types.rs
index a2c43d2246..d291044824 100644
--- a/crates/dbx-core/src/admin/mq/types.rs
+++ b/crates/dbx-core/src/admin/mq/types.rs
@@ -580,7 +580,8 @@ pub struct SubscriptionInfo {
/// RocketMQ: subscribed topic names for this consumer group (cluster-wide listing).
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub topics: Vec<String>,
- /// RocketMQ: online consumer client count (cluster-wide listing).
+ /// Online consumer client count for RocketMQ groups or Kafka topic subscriptions.
+ /// Absent when member information is unavailable.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub online_members: Option<u32>,
/// RocketMQ: NORMAL / FIFO / SYSTEM (aligned with Dashboard).