25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。
TypeScript aicliclickhousedatabasedatabase-clientdatabase-managementdockerguimcpmongodbmysqlpostgresqlredisrustsql-serversqlitetaurivue
fix(kafka): parallelize legacy group queries and recover failed agents
lcq928403e4fe8fe4b
diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.javaindex b389d052dd..647503f825 100644--- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java+++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java@@ -41,6 +41,8 @@ public final class KafkaAgent {private static final PrintStream JSON_RPC_OUT = System.out;private static final Gson GSON = new GsonBuilder().serializeNulls().create();private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;+ private static final int MAX_GROUP_QUERY_ADMINS = 12;+ private static final int GROUPS_PER_QUERY_ADMIN = 16;private static final int MAX_PEEK_MESSAGE_COUNT = 100;private static final int MAX_PEEK_SCAN_RECORDS = 1_000;private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;@@ -66,6 +68,7 @@ public final class KafkaAgent {));private static AdminClient adminClient;+ private static final List<AdminClient> groupQueryAdmins = new ArrayList<>();private static KafkaProducer<String, byte[]> producer;private static JsonObject activeConnection;private static volatile boolean shutdownRequested;@@ -160,8 +163,8 @@ public final class KafkaAgent {case "mq_get_topic_config" -> getTopicConfig(params);case "mq_alter_topic_config" -> alterTopicConfig(params);// Consumer groups- case "mq_list_consumer_groups" -> listConsumerGroups(params, requireAdmin());- case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params, requireAdmin());+ case "mq_list_consumer_groups" -> listConsumerGroups(params, requireAdmin(), KafkaAgent::consumerGroupAdmins);+ case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params, requireAdmin(), KafkaAgent::consumerGroupAdmins);case "mq_describe_consumer_group" -> describeConsumerGroup(params);case "mq_delete_consumer_group" -> deleteConsumerGroup(params);case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params);@@ -283,6 +286,10 @@ public final class KafkaAgent {}private static void closeClients() {+ for (AdminClient groupAdmin : groupQueryAdmins) {+ groupAdmin.close(Duration.ZERO);+ }+ groupQueryAdmins.clear();if (adminClient != null) {adminClient.close(Duration.ofSeconds(5));adminClient = null;@@ -1035,7 +1042,56 @@ public final class KafkaAgent {return descriptions;}+ private static List<Admin> consumerGroupAdmins(int groupCount) {+ int count = Math.min(MAX_GROUP_QUERY_ADMINS, (groupCount + GROUPS_PER_QUERY_ADMIN - 1) / GROUPS_PER_QUERY_ADMIN);+ while (groupQueryAdmins.size() < count - 1) {+ groupQueryAdmins.add(buildAdminClient(activeConnection));+ }+ List<Admin> admins = new ArrayList<>();+ admins.add(adminClient);+ admins.addAll(groupQueryAdmins.subList(0, count - 1));+ return admins;+ }++ private record ConsumerGroupQueries(+ DescribeConsumerGroupsResult descriptions,+ Map<String, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets+ ) {}++ private static ConsumerGroupQueries startConsumerGroupQueries(List<String> groupIds, List<Admin> admins, long deadline) {+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new LinkedHashMap<>();+ Map<String, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets = new LinkedHashMap<>();+ // Older brokers cannot batch coordinator lookups or offset fetches, and+ // Kafka Admin allows one in-flight request per connection. Independent+ // clients bound that otherwise sequential work for large group lists.+ for (int index = 0; index < admins.size(); index++) {+ List<String> shard = new ArrayList<>();+ Map<String, ListConsumerGroupOffsetsSpec> specs = new LinkedHashMap<>();+ for (int groupIndex = index; groupIndex < groupIds.size(); groupIndex += admins.size()) {+ String groupId = groupIds.get(groupIndex);+ shard.add(groupId);+ specs.put(groupId, new ListConsumerGroupOffsetsSpec());+ }+ if (shard.isEmpty()) continue;+ Admin admin = admins.get(index);+ ListConsumerGroupOffsetsResult result = admin.listConsumerGroupOffsets(+ specs, new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))+ );+ descriptions.putAll(admin.describeConsumerGroups(+ shard, new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))+ ).describedGroups());+ for (String groupId : shard) {+ offsets.put(groupId, result.partitionsToOffsetAndMetadata(groupId));+ }+ }+ return new ConsumerGroupQueries(new DescribeConsumerGroupsResult(descriptions), offsets);+ }+static Object listConsumerGroups(JsonObject params, Admin admin) throws Exception {+ return listConsumerGroups(params, admin, ignored -> List.of(admin));+ }++ static Object listConsumerGroups(JsonObject params, Admin admin, java.util.function.IntFunction<List<Admin>> groupAdmins) throws Exception {int timeout = requestTimeout(params);long deadline = System.currentTimeMillis() + timeout;String filterTopic = stringOrEmpty(params, "topic");@@ -1052,25 +1108,11 @@ public final class KafkaAgent {return Collections.singletonMap("groups", Collections.emptyList());}- // Start both independent lookups before waiting. A slow description must- // not consume the deadline before committed offsets can even be requested.- Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();- for (String groupId : groupIds) {- offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());- }+ ConsumerGroupQueries queries = startConsumerGroupQueries(groupIds, groupAdmins.apply(groupIds.size()), deadline);Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();- ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(- offsetSpecs,- new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))- );- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(- groupIds,- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))- );for (String groupId : groupIds) {try {- Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult- .partitionsToOffsetAndMetadata(groupId)+ Map<TopicPartition, OffsetAndMetadata> rows = queries.offsets().get(groupId).get(remainingMillis(deadline), TimeUnit.MILLISECONDS);committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);} catch (Exception error) {@@ -1078,7 +1120,7 @@ public final class KafkaAgent {logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));}}- Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));+ Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(queries.descriptions(), groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));// End offsets: one batched Admin request limited to the requested// topic's partitions (deduplicated across groups). Without a topic@@ -1206,6 +1248,10 @@ public final class KafkaAgent {}static Object getConsumerGroupSnapshot(JsonObject params, Admin admin) throws Exception {+ return getConsumerGroupSnapshot(params, admin, ignored -> List.of(admin));+ }++ static Object getConsumerGroupSnapshot(JsonObject params, Admin admin, java.util.function.IntFunction<List<Admin>> groupAdmins) throws Exception {int timeout = requestTimeout(params);long deadline = System.currentTimeMillis() + timeout;Collection<ConsumerGroupListing> listings = admin.listConsumerGroups(@@ -1226,24 +1272,10 @@ public final class KafkaAgent {Map<String, Boolean> committedOffsetsAvailable = new HashMap<>();Map<String, LinkedHashSet<String>> errors = new HashMap<>();- // Offset and description requests share one deadline, but neither needs- // the other's result. Launch both before awaiting per-group responses.- Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();- for (String groupId : groupIds) {- offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());- }- ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(- offsetSpecs,- new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))- );- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(- groupIds,- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))- );+ ConsumerGroupQueries queries = startConsumerGroupQueries(groupIds, groupAdmins.apply(groupIds.size()), deadline);for (String groupId : groupIds) {try {- Map<TopicPartition, OffsetAndMetadata> groupOffsets = offsetsResult- .partitionsToOffsetAndMetadata(groupId)+ Map<TopicPartition, OffsetAndMetadata> groupOffsets = queries.offsets().get(groupId).get(remainingMillis(deadline), TimeUnit.MILLISECONDS);committedOffsets.put(groupId, groupOffsets == null ? Collections.emptyMap() : groupOffsets);committedOffsetsAvailable.put(groupId, true);@@ -1253,7 +1285,7 @@ public final class KafkaAgent {snapshotErrors(errors, groupId).add("Committed offsets unavailable: " + normalizeErrorMessage(error));}}- descriptions.putAll(awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));+ descriptions.putAll(awaitGroupDescriptions(queries.descriptions(), groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));Map<String, Set<TopicPartition>> assignedPartitions = new HashMap<>();Set<TopicPartition> allPartitions = new LinkedHashSet<>();diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.javaindex aba0408099..adee6ad36f 100644--- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java+++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java@@ -187,8 +187,21 @@ class KafkaAgentTest {@ParameterizedTest@ValueSource(booleans = {true, false})- void consumerGroupQueriesKeepCommittedTopicsWhenDescriptionsTimeOut(boolean snapshot) throws Exception {+ void consumerGroupQueriesKeepCommittedTopicsWhenManyDescriptionsTimeOut(boolean snapshot) throws Exception {TopicPartition partition = new TopicPartition("orders", 0);+ List<ConsumerGroupListing> listings = new ArrayList<>();+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new LinkedHashMap<>();+ for (int index = 0; index < 197; index++) {+ String groupId = index == 0 ? "orders-service" : String.format("slow-group-%03d", index);+ listings.add(new ConsumerGroupListing(groupId, false));+ descriptions.put(groupId, index == 0+ ? KafkaFuture.completedFuture(new ConsumerGroupDescription(+ groupId, false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()+ ))+ : new KafkaFutureImpl<>());+ }+ AtomicInteger descriptionRequests = new AtomicInteger();+ AtomicInteger offsetRequests = new AtomicInteger();ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();try {Admin admin = (Admin) Proxy.newProxyInstance(Admin.class.getClassLoader(), new Class<?>[] {Admin.class},@@ -197,18 +210,16 @@ class KafkaAgentTest {case "listConsumerGroups" -> {var constructor = ListConsumerGroupsResult.class.getDeclaredConstructor(KafkaFuture.class);constructor.setAccessible(true);- yield constructor.newInstance(KafkaFuture.completedFuture(List.of(- new ConsumerGroupListing("orders-service", false),- new ConsumerGroupListing("slow-group", false)- )));+ yield constructor.newInstance(KafkaFuture.completedFuture(listings));+ }+ case "describeConsumerGroups" -> {+ descriptionRequests.incrementAndGet();+ assertEquals(197, ((java.util.Collection<?>) args[0]).size());+ yield new DescribeConsumerGroupsResult(descriptions);}- case "describeConsumerGroups" -> new DescribeConsumerGroupsResult(Map.of(- "orders-service", KafkaFuture.completedFuture(new ConsumerGroupDescription(- "orders-service", false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()- )),- "slow-group", new KafkaFutureImpl<>()- ));case "listConsumerGroupOffsets" -> {+ offsetRequests.incrementAndGet();+ assertEquals(197, ((Map<?, ?>) args[0]).size());// Model an offsets response that needs a small amount of network time.// If descriptions consumed the deadline before this request was sent,// the broker cannot return even these independently available offsets.@@ -223,10 +234,11 @@ class KafkaAgentTest {}, Math.min(50, timeout), TimeUnit.MILLISECONDS);var constructor = ListConsumerGroupOffsetsResult.class.getDeclaredConstructor(Map.class);constructor.setAccessible(true);- yield constructor.newInstance(Map.of(- CoordinatorKey.byGroupId("orders-service"), offsets,- CoordinatorKey.byGroupId("slow-group"), offsets- ));+ Map<CoordinatorKey, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> groupOffsets = new LinkedHashMap<>();+ for (ConsumerGroupListing listing : listings) {+ groupOffsets.put(CoordinatorKey.byGroupId(listing.groupId()), offsets);+ }+ yield constructor.newInstance(groupOffsets);}case "listOffsets" -> new ListOffsetsResult(Map.of(partition, KafkaFuture.completedFuture(new ListOffsetsResult.ListOffsetsResultInfo(5, -1, Optional.empty()))@@ -237,23 +249,28 @@ class KafkaAgentTest {JsonObject params = new JsonObject();params.addProperty("timeout_ms", 300);params.addProperty("topic", "orders");+ long start = System.nanoTime();Object result = snapshot? KafkaAgent.getConsumerGroupSnapshot(params, admin): KafkaAgent.listConsumerGroups(params, admin);+ long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);+ assertTrue(elapsed < 1_500, "waited " + elapsed + "ms; 197 groups must share one 300ms deadline");+ assertEquals(1, descriptionRequests.get());+ assertEquals(1, offsetRequests.get());var groups = new Gson().toJsonTree(result).getAsJsonObject().getAsJsonArray("groups");- assertEquals(2, groups.size());+ assertEquals(197, groups.size());for (var row : groups) {JsonObject group = row.getAsJsonObject();if (snapshot) {assertEquals(1, group.getAsJsonArray("topics").size(), "committed topic survives description timeout");assertEquals("orders", group.getAsJsonArray("topics").get(0).getAsString());assertEquals(3, group.getAsJsonArray("partitions").get(0).getAsJsonObject().get("currentOffset").getAsInt());- if (group.get("groupId").getAsString().equals("slow-group")) {+ if (!group.get("groupId").getAsString().equals("orders-service")) {assertTrue(group.get("error").getAsString().contains("Consumer group details unavailable"));assertFalse(group.has("memberCount"));}} else {- assertEquals(!group.get("groupId").getAsString().equals("slow-group"), group.get("membersAvailable").getAsBoolean());+ assertEquals(group.get("groupId").getAsString().equals("orders-service"), group.get("membersAvailable").getAsBoolean());assertEquals(1, group.getAsJsonArray("committedOffsets").size(), "committed offsets survive description timeout");assertEquals("orders", group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("topic").getAsString());assertEquals(3, group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("offset").getAsInt());@@ -264,6 +281,96 @@ class KafkaAgentTest {}}+ @ParameterizedTest+ @ValueSource(booleans = {true, false})+ void consumerGroupQueriesShardBothLookupsAcrossIndependentAdmins(boolean snapshot) throws Exception {+ TopicPartition partition = new TopicPartition("orders", 0);+ List<ConsumerGroupListing> listings = new ArrayList<>();+ for (int index = 0; index < 197; index++) {+ listings.add(new ConsumerGroupListing(String.format("group-%03d", index), false));+ }+ Map<String, Integer> offsetOwners = new HashMap<>();+ Map<String, Integer> descriptionOwners = new HashMap<>();+ Map<String, KafkaFutureImpl<Map<TopicPartition, OffsetAndMetadata>>> pendingOffsets = new HashMap<>();+ List<Admin> admins = new ArrayList<>();+ for (int index = 0; index < 12; index++) {+ int owner = index;+ admins.add((Admin) Proxy.newProxyInstance(Admin.class.getClassLoader(), new Class<?>[] {Admin.class},+ (proxy, method, args) -> switch (method.getName()) {+ case "listConsumerGroups" -> {+ assertEquals(0, owner);+ var constructor = ListConsumerGroupsResult.class.getDeclaredConstructor(KafkaFuture.class);+ constructor.setAccessible(true);+ yield constructor.newInstance(KafkaFuture.completedFuture(listings));+ }+ case "listConsumerGroupOffsets" -> {+ Map<?, ?> specs = (Map<?, ?>) args[0];+ assertTrue(specs.size() <= 17, "197 groups must be split across 12 clients");+ Map<CoordinatorKey, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets = new HashMap<>();+ for (Object key : specs.keySet()) {+ String groupId = (String) key;+ assertNull(offsetOwners.put(groupId, owner), "offsets requested twice for " + groupId);+ KafkaFutureImpl<Map<TopicPartition, OffsetAndMetadata>> future = new KafkaFutureImpl<>();+ pendingOffsets.put(groupId, future);+ offsets.put(CoordinatorKey.byGroupId(groupId), future);+ }+ var constructor = ListConsumerGroupOffsetsResult.class.getDeclaredConstructor(Map.class);+ constructor.setAccessible(true);+ yield constructor.newInstance(offsets);+ }+ case "describeConsumerGroups" -> {+ java.util.Collection<?> ids = (java.util.Collection<?>) args[0];+ assertTrue(ids.size() <= 17, "descriptions must use the same shards as offsets");+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new HashMap<>();+ for (Object key : ids) {+ String groupId = (String) key;+ assertNull(descriptionOwners.put(groupId, owner), "description requested twice for " + groupId);+ descriptions.put(groupId, KafkaFuture.completedFuture(new ConsumerGroupDescription(+ groupId, false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()+ )));+ }+ // None of the offset futures completes until every shard was sent.+ // Waiting on one client before dispatching the others loses its rows.+ if (offsetOwners.size() == 197 && descriptionOwners.size() == 197) {+ pendingOffsets.values().forEach(future -> future.complete(Map.of(partition, new OffsetAndMetadata(3))));+ }+ yield new DescribeConsumerGroupsResult(descriptions);+ }+ case "listOffsets" -> new ListOffsetsResult(Map.of(+ partition, KafkaFuture.completedFuture(new ListOffsetsResult.ListOffsetsResultInfo(5, -1, Optional.empty()))+ ));+ default -> throw new UnsupportedOperationException(method.getName());+ }));+ }+ JsonObject params = new JsonObject();+ params.addProperty("timeout_ms", 300);+ params.addProperty("topic", "orders");+ java.util.function.IntFunction<List<Admin>> provider = count -> {+ assertEquals(197, count);+ return admins;+ };+ Object result = snapshot+ ? KafkaAgent.getConsumerGroupSnapshot(params, admins.get(0), provider)+ : KafkaAgent.listConsumerGroups(params, admins.get(0), provider);+ assertEquals(197, offsetOwners.size());+ assertEquals(offsetOwners, descriptionOwners);+ assertEquals(12, new java.util.HashSet<>(offsetOwners.values()).size());+ var groups = new Gson().toJsonTree(result).getAsJsonObject().getAsJsonArray("groups");+ assertEquals(197, groups.size());+ for (var row : groups) {+ JsonObject group = row.getAsJsonObject();+ if (snapshot) {+ assertEquals(2, group.get("totalLag").getAsInt());+ assertTrue(group.get("lagAvailable").getAsBoolean());+ assertFalse(group.has("error"));+ } else {+ assertTrue(group.get("membersAvailable").getAsBoolean());+ assertEquals(1, group.getAsJsonArray("committedOffsets").size());+ assertEquals(1, group.getAsJsonArray("endOffsets").size());+ }+ }+ }+@Testvoid consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();diff --git a/crates/dbx-core/src/admin/mq/adapters/kafka.rs b/crates/dbx-core/src/admin/mq/adapters/kafka.rsindex c6536bf8e0..a4052808eb 100644--- a/crates/dbx-core/src/admin/mq/adapters/kafka.rs+++ b/crates/dbx-core/src/admin/mq/adapters/kafka.rs@@ -10,12 +10,13 @@use std::collections::HashMap;use std::sync::Arc;+use std::time::Duration;use async_trait::async_trait;use serde::de::DeserializeOwned;use tokio::sync::Mutex;-use crate::db::agent_driver::{AgentDriverClient, AgentLaunchSpec};+use crate::db::agent_driver::{AgentCallError, AgentDriverClient, AgentLaunchSpec};use crate::mq::auth::MqAuth;use crate::mq::config::MqAdminConfig;use crate::mq::port::MessageQueueAdmin;@@ -59,27 +60,26 @@ const KAFKA_CAPABILITIES: MqCapabilities = MqCapabilities {};pub struct KafkaAdmin {- client: Arc<Mutex<AgentDriverClient>>,+ client: Arc<Mutex<Option<AgentDriverClient>>>,config: MqAdminConfig,+ launch: AgentLaunchSpec,}impl KafkaAdmin {/// Spawn the Kafka Java agent, perform handshake, and connect.pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result<Self, String> {- let mut client = AgentDriverClient::spawn(launch).await?;+ let client = Self::spawn_connected_client(&cfg, launch.clone()).await?;+ log::info!("Kafka admin connected via agent (bootstrap servers: {})", bootstrap_servers(&cfg));+ Ok(Self { client: Arc::new(Mutex::new(Some(client))), config: cfg, launch })+ }- // Handshake+ async fn spawn_connected_client(cfg: &MqAdminConfig, launch: AgentLaunchSpec) -> Result<AgentDriverClient, String> {+ let mut client = AgentDriverClient::spawn(launch).await?;let _: serde_json::Value =client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?;-- // Build the connection params from MqAdminConfig- let conn_params = build_connection_params(&cfg);- let connect_params = serde_json::json!({ "connection": conn_params });+ let connect_params = serde_json::json!({ "connection": build_connection_params(cfg) });let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?;-- log::info!("Kafka admin connected via agent (bootstrap servers: {})", bootstrap_servers(&cfg));-- Ok(Self { client: Arc::new(Mutex::new(client)), config: cfg })+ Ok(client)}/// Send a JSON-RPC call to the Kafka agent and deserialize the result.@@ -88,19 +88,42 @@ impl KafkaAdmin {method: &str,params: serde_json::Value,) -> Result<T, String> {- let mut client = self.client.lock().await;- client.call_with_timeout(method, params, self.config.rpc_timeout()).await+ self.call_with_rpc_timeout(method, params, self.config.rpc_timeout()).await}- /// The Kafka agent bounds these operations with its configured request timeout.- /// Do not preempt that with the driver's generic RPC timeout.- async fn call_with_agent_timeout<T: DeserializeOwned + Send + 'static>(+ /// Consumer-group RPCs share one agent deadline. Allow time to serialize+ /// partial results, while still bounding an unresponsive agent process.+ async fn call_with_consumer_group_timeout<T: DeserializeOwned + Send + 'static>(+ &self,+ method: &str,+ mut params: serde_json::Value,+ ) -> Result<T, String> {+ let request_timeout_ms = self.config.request_timeout_ms();+ params["timeout_ms"] = serde_json::json!(request_timeout_ms);+ let timeout = Duration::from_millis(request_timeout_ms).saturating_add(Duration::from_secs(5));+ self.call_with_rpc_timeout(method, params, Some(timeout)).await+ }++ async fn call_with_rpc_timeout<T: DeserializeOwned + Send + 'static>(&self,method: &str,params: serde_json::Value,+ timeout: Option<Duration>,) -> Result<T, String> {let mut client = self.client.lock().await;- client.call_with_timeout(method, params, None).await+ if client.is_none() {+ *client = Some(Self::spawn_connected_client(&self.config, self.launch.clone()).await?);+ }+ let result = client.as_mut().unwrap().call_typed_with_timeout(method, params, timeout).await;+ if matches!(+ &result,+ Err(AgentCallError::Transport { .. } | AgentCallError::Timeout { .. } | AgentCallError::Canceled { .. })+ ) {+ // Reconnect before the next operation. Never replay a request whose+ // outcome is unknown, especially topic/offset mutations or sends.+ client.take();+ }+ result.map_err(AgentCallError::into_legacy_string)}/// Send a JSON-RPC call that returns `{ok: true}` on success.@@ -192,7 +215,8 @@ impl MessageQueueAdmin for KafkaAdmin {// ---- Topics ----async fn list_topics(&self, _ns: &NamespaceRef, _opts: ListTopicsOpts) -> Result<Vec<TopicInfo>, String> {- let result: serde_json::Value = self.call_with_agent_timeout("mq_list_topics", serde_json::json!({})).await?;+ let result: serde_json::Value =+ self.call_with_rpc_timeout("mq_list_topics", serde_json::json!({}), None).await?;let topics = result.get("topics").and_then(|v| v.as_array()).cloned().unwrap_or_default();Ok(topics@@ -281,7 +305,7 @@ impl MessageQueueAdmin for KafkaAdmin {// (The previous 1 + 2N serial RPC pattern — describe + lag per group — took// tens of seconds on remote clusters with many groups, see #7163.)let result: serde_json::Value = self- .call_with_agent_timeout(+ .call_with_consumer_group_timeout("mq_list_consumer_groups",serde_json::json!({ "topic": topic.topic, "timeout_ms": self.config.request_timeout_ms() }),)@@ -300,8 +324,11 @@ impl MessageQueueAdmin for KafkaAdmin {}async fn get_kafka_consumer_group_snapshot(&self) -> Result<KafkaConsumerGroupSnapshot, String> {- self.call_with_agent_timeout("mq_get_consumer_group_snapshot", consumer_group_snapshot_params(&self.config))- .await+ self.call_with_consumer_group_timeout(+ "mq_get_consumer_group_snapshot",+ consumer_group_snapshot_params(&self.config),+ )+ .await}async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> {@@ -342,7 +369,7 @@ impl MessageQueueAdmin for KafkaAdmin {options: PeekMessagesOptions,) -> Result<PeekMessagesResult, String> {let params = peek_messages_params(&self.config, topic, count, options);- let result: serde_json::Value = self.call_with_agent_timeout("mq_peek_messages", params).await?;+ let result: serde_json::Value = self.call_with_rpc_timeout("mq_peek_messages", params, None).await?;Ok(peek_messages_result_from_agent(&result))}@@ -1374,6 +1401,112 @@ mod tests {assert!(!partitions.map(|p| p > 0).unwrap_or(false));}+ async fn recoverable_test_admin() -> (KafkaAdmin, tempfile::TempDir) {+ let directory = tempfile::tempdir().expect("create fake Kafka agent directory");+ let script = directory.path().join("agent.py");+ let calls = directory.path().join("calls.txt");+ std::fs::write(+ &script,+ r#"import json+import sys+import time++with open(sys.argv[1], "a") as calls:+ calls.write("spawn\n")+print(json.dumps({"ready": True}), flush=True)+for line in sys.stdin:+ request = json.loads(line)+ method = request["method"]+ with open(sys.argv[1], "a") as calls:+ calls.write(method + "\n")+ if method in ("mq_create_topic", "mq_get_consumer_group_snapshot"):+ time.sleep(60)+ if method == "mq_delete_topic":+ sys.exit(1)+ if method == "mq_get_topic_config":+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"],+ "error": {"code": -1, "message": "Topic authorization failed"}}), flush=True)+ continue+ result = {"topics": [{"name": "orders", "internal": False}]} if method == "mq_list_topics" else {}+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True)+"#,+ )+ .expect("write fake Kafka agent");+ let python = if cfg!(windows) { "python" } else { "python3" };+ let launch = AgentLaunchSpec::new(python)+ .with_args([script.to_string_lossy().to_string(), calls.to_string_lossy().to_string()]);+ let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);+ config.query_timeout_secs = 1;+ (KafkaAdmin::new(config, launch).await.expect("connect fake Kafka agent"), directory)+ }++ async fn assert_recovered_topics(admin: &KafkaAdmin) {+ let topics = admin+ .list_topics(+ &NamespaceRef { tenant: "_kafka".to_string(), namespace: "_kafka".to_string() },+ ListTopicsOpts::default(),+ )+ .await+ .expect("the next request reconnects the Kafka agent");+ assert_eq!(topics[0].name, "orders");+ }++ #[tokio::test]+ async fn kafka_agent_recovers_after_timeout_without_replaying_mutation() {+ let (admin, directory) = recoverable_test_admin().await;+ let error = admin.create_topic(&topic_ref(), Some(1)).await.unwrap_err();+ assert!(error.contains("timed out"), "unexpected error: {error}");+ assert_recovered_topics(&admin).await;++ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);+ assert_eq!(calls.lines().filter(|line| *line == "mq_create_topic").count(), 1);+ }++ #[tokio::test]+ async fn kafka_agent_recovers_after_exit_without_replaying_mutation() {+ let (admin, directory) = recoverable_test_admin().await;+ admin.delete_topic(&topic_ref(), false).await.expect_err("the agent exited before replying");+ assert_recovered_topics(&admin).await;++ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);+ assert_eq!(calls.lines().filter(|line| *line == "mq_delete_topic").count(), 1);+ }++ #[tokio::test]+ async fn kafka_agent_hanging_snapshot_is_bounded_and_recovers() {+ let (admin, directory) = recoverable_test_admin().await;+ let result =+ tokio::time::timeout(std::time::Duration::from_secs(8), admin.get_kafka_consumer_group_snapshot()).await;+ if result.is_err() {+ // Dropping the old adapter also releases its blocked stdout worker.+ drop(admin);+ panic!("a stalled Kafka snapshot must have a finite outer RPC deadline");+ }+ let error = result.unwrap().unwrap_err();+ assert!(error.contains("timed out"), "unexpected error: {error}");+ assert_recovered_topics(&admin).await;++ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);+ }++ #[tokio::test]+ async fn kafka_agent_rpc_error_keeps_the_existing_process() {+ let (admin, directory) = recoverable_test_admin().await;+ let error = admin.get_topic_internal_stats(&topic_ref()).await.unwrap_err();+ assert!(error.contains("Topic authorization failed"), "unexpected error: {error}");+ assert_recovered_topics(&admin).await;++ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 1);+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 1);+ }+async fn delayed_consumer_group_admin() -> (KafkaAdmin, tempfile::TempPath) {let script_path = tempfile::NamedTempFile::new().expect("create test agent script").into_temp_path();std::fs::write(@@ -1418,14 +1551,11 @@ for line in sys.stdin:).expect("write test agent script");let python = if cfg!(windows) { "python" } else { "python3" };- let client = AgentDriverClient::spawn(- AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),- )- .await- .expect("spawn test agent");+ let launch = AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]);+ let client = AgentDriverClient::spawn(launch.clone()).await.expect("spawn test agent");let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);config.query_timeout_secs = 1;- (KafkaAdmin { client: Arc::new(Mutex::new(client)), config }, script_path)+ (KafkaAdmin { client: Arc::new(Mutex::new(Some(client))), config, launch }, script_path)}#[tokio::test]@@ -1489,15 +1619,12 @@ for line in sys.stdin:.expect("write test agent script");let python = if cfg!(windows) { "python" } else { "python3" };- let client = AgentDriverClient::spawn(- AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),- )- .await- .expect("spawn test agent");+ let launch = AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]);+ let client = AgentDriverClient::spawn(launch.clone()).await.expect("spawn test agent");// The delayed response models the Kafka agent returning its fallback after metadata timeout.let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);config.query_timeout_secs = 1;- let admin = KafkaAdmin { client: Arc::new(Mutex::new(client)), config };+ let admin = KafkaAdmin { client: Arc::new(Mutex::new(Some(client))), config, launch };let result = admin.list_topics(