Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

fix(kafka): parallelize legacy group queries and recover failed agents

lcq928403e4fe8fe4b
diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
index b389d052dd..647503f825 100644
--- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
+++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java
@@ -41,6 +41,8 @@ public final class KafkaAgent {
private static final PrintStream JSON_RPC_OUT = System.out;
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+ private static final int MAX_GROUP_QUERY_ADMINS = 12;
+ private static final int GROUPS_PER_QUERY_ADMIN = 16;
private static final int MAX_PEEK_MESSAGE_COUNT = 100;
private static final int MAX_PEEK_SCAN_RECORDS = 1_000;
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
@@ -66,6 +68,7 @@ public final class KafkaAgent {
));
private static AdminClient adminClient;
+ private static final List<AdminClient> groupQueryAdmins = new ArrayList<>();
private static KafkaProducer<String, byte[]> producer;
private static JsonObject activeConnection;
private static volatile boolean shutdownRequested;
@@ -160,8 +163,8 @@ public final class KafkaAgent {
case "mq_get_topic_config" -> getTopicConfig(params);
case "mq_alter_topic_config" -> alterTopicConfig(params);
// Consumer groups
- case "mq_list_consumer_groups" -> listConsumerGroups(params, requireAdmin());
- case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params, requireAdmin());
+ case "mq_list_consumer_groups" -> listConsumerGroups(params, requireAdmin(), KafkaAgent::consumerGroupAdmins);
+ case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params, requireAdmin(), KafkaAgent::consumerGroupAdmins);
case "mq_describe_consumer_group" -> describeConsumerGroup(params);
case "mq_delete_consumer_group" -> deleteConsumerGroup(params);
case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params);
@@ -283,6 +286,10 @@ public final class KafkaAgent {
}
private static void closeClients() {
+ for (AdminClient groupAdmin : groupQueryAdmins) {
+ groupAdmin.close(Duration.ZERO);
+ }
+ groupQueryAdmins.clear();
if (adminClient != null) {
adminClient.close(Duration.ofSeconds(5));
adminClient = null;
@@ -1035,7 +1042,56 @@ public final class KafkaAgent {
return descriptions;
}
+ private static List<Admin> consumerGroupAdmins(int groupCount) {
+ int count = Math.min(MAX_GROUP_QUERY_ADMINS, (groupCount + GROUPS_PER_QUERY_ADMIN - 1) / GROUPS_PER_QUERY_ADMIN);
+ while (groupQueryAdmins.size() < count - 1) {
+ groupQueryAdmins.add(buildAdminClient(activeConnection));
+ }
+ List<Admin> admins = new ArrayList<>();
+ admins.add(adminClient);
+ admins.addAll(groupQueryAdmins.subList(0, count - 1));
+ return admins;
+ }
+
+ private record ConsumerGroupQueries(
+ DescribeConsumerGroupsResult descriptions,
+ Map<String, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets
+ ) {}
+
+ private static ConsumerGroupQueries startConsumerGroupQueries(List<String> groupIds, List<Admin> admins, long deadline) {
+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new LinkedHashMap<>();
+ Map<String, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets = new LinkedHashMap<>();
+ // Older brokers cannot batch coordinator lookups or offset fetches, and
+ // Kafka Admin allows one in-flight request per connection. Independent
+ // clients bound that otherwise sequential work for large group lists.
+ for (int index = 0; index < admins.size(); index++) {
+ List<String> shard = new ArrayList<>();
+ Map<String, ListConsumerGroupOffsetsSpec> specs = new LinkedHashMap<>();
+ for (int groupIndex = index; groupIndex < groupIds.size(); groupIndex += admins.size()) {
+ String groupId = groupIds.get(groupIndex);
+ shard.add(groupId);
+ specs.put(groupId, new ListConsumerGroupOffsetsSpec());
+ }
+ if (shard.isEmpty()) continue;
+ Admin admin = admins.get(index);
+ ListConsumerGroupOffsetsResult result = admin.listConsumerGroupOffsets(
+ specs, new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))
+ );
+ descriptions.putAll(admin.describeConsumerGroups(
+ shard, new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
+ ).describedGroups());
+ for (String groupId : shard) {
+ offsets.put(groupId, result.partitionsToOffsetAndMetadata(groupId));
+ }
+ }
+ return new ConsumerGroupQueries(new DescribeConsumerGroupsResult(descriptions), offsets);
+ }
+
static Object listConsumerGroups(JsonObject params, Admin admin) throws Exception {
+ return listConsumerGroups(params, admin, ignored -> List.of(admin));
+ }
+
+ static Object listConsumerGroups(JsonObject params, Admin admin, java.util.function.IntFunction<List<Admin>> groupAdmins) throws Exception {
int timeout = requestTimeout(params);
long deadline = System.currentTimeMillis() + timeout;
String filterTopic = stringOrEmpty(params, "topic");
@@ -1052,25 +1108,11 @@ public final class KafkaAgent {
return Collections.singletonMap("groups", Collections.emptyList());
}
- // Start both independent lookups before waiting. A slow description must
- // not consume the deadline before committed offsets can even be requested.
- Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
- for (String groupId : groupIds) {
- offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
- }
+ ConsumerGroupQueries queries = startConsumerGroupQueries(groupIds, groupAdmins.apply(groupIds.size()), deadline);
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
- ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
- offsetSpecs,
- new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))
- );
- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
- groupIds,
- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
- );
for (String groupId : groupIds) {
try {
- Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
- .partitionsToOffsetAndMetadata(groupId)
+ Map<TopicPartition, OffsetAndMetadata> rows = queries.offsets().get(groupId)
.get(remainingMillis(deadline), TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
} catch (Exception error) {
@@ -1078,7 +1120,7 @@ public final class KafkaAgent {
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
- Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));
+ Map<String, ConsumerGroupDescription> descriptions = awaitGroupDescriptions(queries.descriptions(), groupIds, deadline, (groupId, message) -> logger().debug("Consumer group details unavailable for {}: {}", groupId, message));
// End offsets: one batched Admin request limited to the requested
// topic's partitions (deduplicated across groups). Without a topic
@@ -1206,6 +1248,10 @@ public final class KafkaAgent {
}
static Object getConsumerGroupSnapshot(JsonObject params, Admin admin) throws Exception {
+ return getConsumerGroupSnapshot(params, admin, ignored -> List.of(admin));
+ }
+
+ static Object getConsumerGroupSnapshot(JsonObject params, Admin admin, java.util.function.IntFunction<List<Admin>> groupAdmins) throws Exception {
int timeout = requestTimeout(params);
long deadline = System.currentTimeMillis() + timeout;
Collection<ConsumerGroupListing> listings = admin.listConsumerGroups(
@@ -1226,24 +1272,10 @@ public final class KafkaAgent {
Map<String, Boolean> committedOffsetsAvailable = new HashMap<>();
Map<String, LinkedHashSet<String>> errors = new HashMap<>();
- // Offset and description requests share one deadline, but neither needs
- // the other's result. Launch both before awaiting per-group responses.
- Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
- for (String groupId : groupIds) {
- offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
- }
- ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
- offsetSpecs,
- new ListConsumerGroupOffsetsOptions().timeoutMs((int) remainingMillis(deadline))
- );
- DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
- groupIds,
- new DescribeConsumerGroupsOptions().timeoutMs((int) remainingMillis(deadline))
- );
+ ConsumerGroupQueries queries = startConsumerGroupQueries(groupIds, groupAdmins.apply(groupIds.size()), deadline);
for (String groupId : groupIds) {
try {
- Map<TopicPartition, OffsetAndMetadata> groupOffsets = offsetsResult
- .partitionsToOffsetAndMetadata(groupId)
+ Map<TopicPartition, OffsetAndMetadata> groupOffsets = queries.offsets().get(groupId)
.get(remainingMillis(deadline), TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, groupOffsets == null ? Collections.emptyMap() : groupOffsets);
committedOffsetsAvailable.put(groupId, true);
@@ -1253,7 +1285,7 @@ public final class KafkaAgent {
snapshotErrors(errors, groupId).add("Committed offsets unavailable: " + normalizeErrorMessage(error));
}
}
- descriptions.putAll(awaitGroupDescriptions(described, groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));
+ descriptions.putAll(awaitGroupDescriptions(queries.descriptions(), groupIds, deadline, (groupId, message) -> snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + message)));
Map<String, Set<TopicPartition>> assignedPartitions = new HashMap<>();
Set<TopicPartition> allPartitions = new LinkedHashSet<>();
diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
index aba0408099..adee6ad36f 100644
--- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
+++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java
@@ -187,8 +187,21 @@ class KafkaAgentTest {
@ParameterizedTest
@ValueSource(booleans = {true, false})
- void consumerGroupQueriesKeepCommittedTopicsWhenDescriptionsTimeOut(boolean snapshot) throws Exception {
+ void consumerGroupQueriesKeepCommittedTopicsWhenManyDescriptionsTimeOut(boolean snapshot) throws Exception {
TopicPartition partition = new TopicPartition("orders", 0);
+ List<ConsumerGroupListing> listings = new ArrayList<>();
+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new LinkedHashMap<>();
+ for (int index = 0; index < 197; index++) {
+ String groupId = index == 0 ? "orders-service" : String.format("slow-group-%03d", index);
+ listings.add(new ConsumerGroupListing(groupId, false));
+ descriptions.put(groupId, index == 0
+ ? KafkaFuture.completedFuture(new ConsumerGroupDescription(
+ groupId, false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()
+ ))
+ : new KafkaFutureImpl<>());
+ }
+ AtomicInteger descriptionRequests = new AtomicInteger();
+ AtomicInteger offsetRequests = new AtomicInteger();
ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
try {
Admin admin = (Admin) Proxy.newProxyInstance(Admin.class.getClassLoader(), new Class<?>[] {Admin.class},
@@ -197,18 +210,16 @@ class KafkaAgentTest {
case "listConsumerGroups" -> {
var constructor = ListConsumerGroupsResult.class.getDeclaredConstructor(KafkaFuture.class);
constructor.setAccessible(true);
- yield constructor.newInstance(KafkaFuture.completedFuture(List.of(
- new ConsumerGroupListing("orders-service", false),
- new ConsumerGroupListing("slow-group", false)
- )));
+ yield constructor.newInstance(KafkaFuture.completedFuture(listings));
+ }
+ case "describeConsumerGroups" -> {
+ descriptionRequests.incrementAndGet();
+ assertEquals(197, ((java.util.Collection<?>) args[0]).size());
+ yield new DescribeConsumerGroupsResult(descriptions);
}
- case "describeConsumerGroups" -> new DescribeConsumerGroupsResult(Map.of(
- "orders-service", KafkaFuture.completedFuture(new ConsumerGroupDescription(
- "orders-service", false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()
- )),
- "slow-group", new KafkaFutureImpl<>()
- ));
case "listConsumerGroupOffsets" -> {
+ offsetRequests.incrementAndGet();
+ assertEquals(197, ((Map<?, ?>) args[0]).size());
// Model an offsets response that needs a small amount of network time.
// If descriptions consumed the deadline before this request was sent,
// the broker cannot return even these independently available offsets.
@@ -223,10 +234,11 @@ class KafkaAgentTest {
}, Math.min(50, timeout), TimeUnit.MILLISECONDS);
var constructor = ListConsumerGroupOffsetsResult.class.getDeclaredConstructor(Map.class);
constructor.setAccessible(true);
- yield constructor.newInstance(Map.of(
- CoordinatorKey.byGroupId("orders-service"), offsets,
- CoordinatorKey.byGroupId("slow-group"), offsets
- ));
+ Map<CoordinatorKey, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> groupOffsets = new LinkedHashMap<>();
+ for (ConsumerGroupListing listing : listings) {
+ groupOffsets.put(CoordinatorKey.byGroupId(listing.groupId()), offsets);
+ }
+ yield constructor.newInstance(groupOffsets);
}
case "listOffsets" -> new ListOffsetsResult(Map.of(
partition, KafkaFuture.completedFuture(new ListOffsetsResult.ListOffsetsResultInfo(5, -1, Optional.empty()))
@@ -237,23 +249,28 @@ class KafkaAgentTest {
JsonObject params = new JsonObject();
params.addProperty("timeout_ms", 300);
params.addProperty("topic", "orders");
+ long start = System.nanoTime();
Object result = snapshot
? KafkaAgent.getConsumerGroupSnapshot(params, admin)
: KafkaAgent.listConsumerGroups(params, admin);
+ long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
+ assertTrue(elapsed < 1_500, "waited " + elapsed + "ms; 197 groups must share one 300ms deadline");
+ assertEquals(1, descriptionRequests.get());
+ assertEquals(1, offsetRequests.get());
var groups = new Gson().toJsonTree(result).getAsJsonObject().getAsJsonArray("groups");
- assertEquals(2, groups.size());
+ assertEquals(197, groups.size());
for (var row : groups) {
JsonObject group = row.getAsJsonObject();
if (snapshot) {
assertEquals(1, group.getAsJsonArray("topics").size(), "committed topic survives description timeout");
assertEquals("orders", group.getAsJsonArray("topics").get(0).getAsString());
assertEquals(3, group.getAsJsonArray("partitions").get(0).getAsJsonObject().get("currentOffset").getAsInt());
- if (group.get("groupId").getAsString().equals("slow-group")) {
+ if (!group.get("groupId").getAsString().equals("orders-service")) {
assertTrue(group.get("error").getAsString().contains("Consumer group details unavailable"));
assertFalse(group.has("memberCount"));
}
} else {
- assertEquals(!group.get("groupId").getAsString().equals("slow-group"), group.get("membersAvailable").getAsBoolean());
+ assertEquals(group.get("groupId").getAsString().equals("orders-service"), group.get("membersAvailable").getAsBoolean());
assertEquals(1, group.getAsJsonArray("committedOffsets").size(), "committed offsets survive description timeout");
assertEquals("orders", group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("topic").getAsString());
assertEquals(3, group.getAsJsonArray("committedOffsets").get(0).getAsJsonObject().get("offset").getAsInt());
@@ -264,6 +281,96 @@ class KafkaAgentTest {
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void consumerGroupQueriesShardBothLookupsAcrossIndependentAdmins(boolean snapshot) throws Exception {
+ TopicPartition partition = new TopicPartition("orders", 0);
+ List<ConsumerGroupListing> listings = new ArrayList<>();
+ for (int index = 0; index < 197; index++) {
+ listings.add(new ConsumerGroupListing(String.format("group-%03d", index), false));
+ }
+ Map<String, Integer> offsetOwners = new HashMap<>();
+ Map<String, Integer> descriptionOwners = new HashMap<>();
+ Map<String, KafkaFutureImpl<Map<TopicPartition, OffsetAndMetadata>>> pendingOffsets = new HashMap<>();
+ List<Admin> admins = new ArrayList<>();
+ for (int index = 0; index < 12; index++) {
+ int owner = index;
+ admins.add((Admin) Proxy.newProxyInstance(Admin.class.getClassLoader(), new Class<?>[] {Admin.class},
+ (proxy, method, args) -> switch (method.getName()) {
+ case "listConsumerGroups" -> {
+ assertEquals(0, owner);
+ var constructor = ListConsumerGroupsResult.class.getDeclaredConstructor(KafkaFuture.class);
+ constructor.setAccessible(true);
+ yield constructor.newInstance(KafkaFuture.completedFuture(listings));
+ }
+ case "listConsumerGroupOffsets" -> {
+ Map<?, ?> specs = (Map<?, ?>) args[0];
+ assertTrue(specs.size() <= 17, "197 groups must be split across 12 clients");
+ Map<CoordinatorKey, KafkaFuture<Map<TopicPartition, OffsetAndMetadata>>> offsets = new HashMap<>();
+ for (Object key : specs.keySet()) {
+ String groupId = (String) key;
+ assertNull(offsetOwners.put(groupId, owner), "offsets requested twice for " + groupId);
+ KafkaFutureImpl<Map<TopicPartition, OffsetAndMetadata>> future = new KafkaFutureImpl<>();
+ pendingOffsets.put(groupId, future);
+ offsets.put(CoordinatorKey.byGroupId(groupId), future);
+ }
+ var constructor = ListConsumerGroupOffsetsResult.class.getDeclaredConstructor(Map.class);
+ constructor.setAccessible(true);
+ yield constructor.newInstance(offsets);
+ }
+ case "describeConsumerGroups" -> {
+ java.util.Collection<?> ids = (java.util.Collection<?>) args[0];
+ assertTrue(ids.size() <= 17, "descriptions must use the same shards as offsets");
+ Map<String, KafkaFuture<ConsumerGroupDescription>> descriptions = new HashMap<>();
+ for (Object key : ids) {
+ String groupId = (String) key;
+ assertNull(descriptionOwners.put(groupId, owner), "description requested twice for " + groupId);
+ descriptions.put(groupId, KafkaFuture.completedFuture(new ConsumerGroupDescription(
+ groupId, false, List.of(), "", ConsumerGroupState.EMPTY, Node.noNode()
+ )));
+ }
+ // None of the offset futures completes until every shard was sent.
+ // Waiting on one client before dispatching the others loses its rows.
+ if (offsetOwners.size() == 197 && descriptionOwners.size() == 197) {
+ pendingOffsets.values().forEach(future -> future.complete(Map.of(partition, new OffsetAndMetadata(3))));
+ }
+ yield new DescribeConsumerGroupsResult(descriptions);
+ }
+ case "listOffsets" -> new ListOffsetsResult(Map.of(
+ partition, KafkaFuture.completedFuture(new ListOffsetsResult.ListOffsetsResultInfo(5, -1, Optional.empty()))
+ ));
+ default -> throw new UnsupportedOperationException(method.getName());
+ }));
+ }
+ JsonObject params = new JsonObject();
+ params.addProperty("timeout_ms", 300);
+ params.addProperty("topic", "orders");
+ java.util.function.IntFunction<List<Admin>> provider = count -> {
+ assertEquals(197, count);
+ return admins;
+ };
+ Object result = snapshot
+ ? KafkaAgent.getConsumerGroupSnapshot(params, admins.get(0), provider)
+ : KafkaAgent.listConsumerGroups(params, admins.get(0), provider);
+ assertEquals(197, offsetOwners.size());
+ assertEquals(offsetOwners, descriptionOwners);
+ assertEquals(12, new java.util.HashSet<>(offsetOwners.values()).size());
+ var groups = new Gson().toJsonTree(result).getAsJsonObject().getAsJsonArray("groups");
+ assertEquals(197, groups.size());
+ for (var row : groups) {
+ JsonObject group = row.getAsJsonObject();
+ if (snapshot) {
+ assertEquals(2, group.get("totalLag").getAsInt());
+ assertTrue(group.get("lagAvailable").getAsBoolean());
+ assertFalse(group.has("error"));
+ } else {
+ assertTrue(group.get("membersAvailable").getAsBoolean());
+ assertEquals(1, group.getAsJsonArray("committedOffsets").size());
+ assertEquals(1, group.getAsJsonArray("endOffsets").size());
+ }
+ }
+ }
+
@Test
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
diff --git a/crates/dbx-core/src/admin/mq/adapters/kafka.rs b/crates/dbx-core/src/admin/mq/adapters/kafka.rs
index c6536bf8e0..a4052808eb 100644
--- a/crates/dbx-core/src/admin/mq/adapters/kafka.rs
+++ b/crates/dbx-core/src/admin/mq/adapters/kafka.rs
@@ -10,12 +10,13 @@
use std::collections::HashMap;
use std::sync::Arc;
+use std::time::Duration;
use async_trait::async_trait;
use serde::de::DeserializeOwned;
use tokio::sync::Mutex;
-use crate::db::agent_driver::{AgentDriverClient, AgentLaunchSpec};
+use crate::db::agent_driver::{AgentCallError, AgentDriverClient, AgentLaunchSpec};
use crate::mq::auth::MqAuth;
use crate::mq::config::MqAdminConfig;
use crate::mq::port::MessageQueueAdmin;
@@ -59,27 +60,26 @@ const KAFKA_CAPABILITIES: MqCapabilities = MqCapabilities {
};
pub struct KafkaAdmin {
- client: Arc<Mutex<AgentDriverClient>>,
+ client: Arc<Mutex<Option<AgentDriverClient>>>,
config: MqAdminConfig,
+ launch: AgentLaunchSpec,
}
impl KafkaAdmin {
/// Spawn the Kafka Java agent, perform handshake, and connect.
pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result<Self, String> {
- let mut client = AgentDriverClient::spawn(launch).await?;
+ let client = Self::spawn_connected_client(&cfg, launch.clone()).await?;
+ log::info!("Kafka admin connected via agent (bootstrap servers: {})", bootstrap_servers(&cfg));
+ Ok(Self { client: Arc::new(Mutex::new(Some(client))), config: cfg, launch })
+ }
- // Handshake
+ async fn spawn_connected_client(cfg: &MqAdminConfig, launch: AgentLaunchSpec) -> Result<AgentDriverClient, String> {
+ let mut client = AgentDriverClient::spawn(launch).await?;
let _: serde_json::Value =
client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?;
-
- // Build the connection params from MqAdminConfig
- let conn_params = build_connection_params(&cfg);
- let connect_params = serde_json::json!({ "connection": conn_params });
+ let connect_params = serde_json::json!({ "connection": build_connection_params(cfg) });
let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?;
-
- log::info!("Kafka admin connected via agent (bootstrap servers: {})", bootstrap_servers(&cfg));
-
- Ok(Self { client: Arc::new(Mutex::new(client)), config: cfg })
+ Ok(client)
}
/// Send a JSON-RPC call to the Kafka agent and deserialize the result.
@@ -88,19 +88,42 @@ impl KafkaAdmin {
method: &str,
params: serde_json::Value,
) -> Result<T, String> {
- let mut client = self.client.lock().await;
- client.call_with_timeout(method, params, self.config.rpc_timeout()).await
+ self.call_with_rpc_timeout(method, params, self.config.rpc_timeout()).await
}
- /// The Kafka agent bounds these operations with its configured request timeout.
- /// Do not preempt that with the driver's generic RPC timeout.
- async fn call_with_agent_timeout<T: DeserializeOwned + Send + 'static>(
+ /// Consumer-group RPCs share one agent deadline. Allow time to serialize
+ /// partial results, while still bounding an unresponsive agent process.
+ async fn call_with_consumer_group_timeout<T: DeserializeOwned + Send + 'static>(
+ &self,
+ method: &str,
+ mut params: serde_json::Value,
+ ) -> Result<T, String> {
+ let request_timeout_ms = self.config.request_timeout_ms();
+ params["timeout_ms"] = serde_json::json!(request_timeout_ms);
+ let timeout = Duration::from_millis(request_timeout_ms).saturating_add(Duration::from_secs(5));
+ self.call_with_rpc_timeout(method, params, Some(timeout)).await
+ }
+
+ async fn call_with_rpc_timeout<T: DeserializeOwned + Send + 'static>(
&self,
method: &str,
params: serde_json::Value,
+ timeout: Option<Duration>,
) -> Result<T, String> {
let mut client = self.client.lock().await;
- client.call_with_timeout(method, params, None).await
+ if client.is_none() {
+ *client = Some(Self::spawn_connected_client(&self.config, self.launch.clone()).await?);
+ }
+ let result = client.as_mut().unwrap().call_typed_with_timeout(method, params, timeout).await;
+ if matches!(
+ &result,
+ Err(AgentCallError::Transport { .. } | AgentCallError::Timeout { .. } | AgentCallError::Canceled { .. })
+ ) {
+ // Reconnect before the next operation. Never replay a request whose
+ // outcome is unknown, especially topic/offset mutations or sends.
+ client.take();
+ }
+ result.map_err(AgentCallError::into_legacy_string)
}
/// Send a JSON-RPC call that returns `{ok: true}` on success.
@@ -192,7 +215,8 @@ impl MessageQueueAdmin for KafkaAdmin {
// ---- Topics ----
async fn list_topics(&self, _ns: &NamespaceRef, _opts: ListTopicsOpts) -> Result<Vec<TopicInfo>, String> {
- let result: serde_json::Value = self.call_with_agent_timeout("mq_list_topics", serde_json::json!({})).await?;
+ let result: serde_json::Value =
+ self.call_with_rpc_timeout("mq_list_topics", serde_json::json!({}), None).await?;
let topics = result.get("topics").and_then(|v| v.as_array()).cloned().unwrap_or_default();
Ok(topics
@@ -281,7 +305,7 @@ impl MessageQueueAdmin for KafkaAdmin {
// (The previous 1 + 2N serial RPC pattern — describe + lag per group — took
// tens of seconds on remote clusters with many groups, see #7163.)
let result: serde_json::Value = self
- .call_with_agent_timeout(
+ .call_with_consumer_group_timeout(
"mq_list_consumer_groups",
serde_json::json!({ "topic": topic.topic, "timeout_ms": self.config.request_timeout_ms() }),
)
@@ -300,8 +324,11 @@ impl MessageQueueAdmin for KafkaAdmin {
}
async fn get_kafka_consumer_group_snapshot(&self) -> Result<KafkaConsumerGroupSnapshot, String> {
- self.call_with_agent_timeout("mq_get_consumer_group_snapshot", consumer_group_snapshot_params(&self.config))
- .await
+ self.call_with_consumer_group_timeout(
+ "mq_get_consumer_group_snapshot",
+ consumer_group_snapshot_params(&self.config),
+ )
+ .await
}
async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> {
@@ -342,7 +369,7 @@ impl MessageQueueAdmin for KafkaAdmin {
options: PeekMessagesOptions,
) -> Result<PeekMessagesResult, String> {
let params = peek_messages_params(&self.config, topic, count, options);
- let result: serde_json::Value = self.call_with_agent_timeout("mq_peek_messages", params).await?;
+ let result: serde_json::Value = self.call_with_rpc_timeout("mq_peek_messages", params, None).await?;
Ok(peek_messages_result_from_agent(&result))
}
@@ -1374,6 +1401,112 @@ mod tests {
assert!(!partitions.map(|p| p > 0).unwrap_or(false));
}
+ async fn recoverable_test_admin() -> (KafkaAdmin, tempfile::TempDir) {
+ let directory = tempfile::tempdir().expect("create fake Kafka agent directory");
+ let script = directory.path().join("agent.py");
+ let calls = directory.path().join("calls.txt");
+ std::fs::write(
+ &script,
+ r#"import json
+import sys
+import time
+
+with open(sys.argv[1], "a") as calls:
+ calls.write("spawn\n")
+print(json.dumps({"ready": True}), flush=True)
+for line in sys.stdin:
+ request = json.loads(line)
+ method = request["method"]
+ with open(sys.argv[1], "a") as calls:
+ calls.write(method + "\n")
+ if method in ("mq_create_topic", "mq_get_consumer_group_snapshot"):
+ time.sleep(60)
+ if method == "mq_delete_topic":
+ sys.exit(1)
+ if method == "mq_get_topic_config":
+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"],
+ "error": {"code": -1, "message": "Topic authorization failed"}}), flush=True)
+ continue
+ result = {"topics": [{"name": "orders", "internal": False}]} if method == "mq_list_topics" else {}
+ print(json.dumps({"jsonrpc": "2.0", "id": request["id"], "result": result}), flush=True)
+"#,
+ )
+ .expect("write fake Kafka agent");
+ let python = if cfg!(windows) { "python" } else { "python3" };
+ let launch = AgentLaunchSpec::new(python)
+ .with_args([script.to_string_lossy().to_string(), calls.to_string_lossy().to_string()]);
+ let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
+ config.query_timeout_secs = 1;
+ (KafkaAdmin::new(config, launch).await.expect("connect fake Kafka agent"), directory)
+ }
+
+ async fn assert_recovered_topics(admin: &KafkaAdmin) {
+ let topics = admin
+ .list_topics(
+ &NamespaceRef { tenant: "_kafka".to_string(), namespace: "_kafka".to_string() },
+ ListTopicsOpts::default(),
+ )
+ .await
+ .expect("the next request reconnects the Kafka agent");
+ assert_eq!(topics[0].name, "orders");
+ }
+
+ #[tokio::test]
+ async fn kafka_agent_recovers_after_timeout_without_replaying_mutation() {
+ let (admin, directory) = recoverable_test_admin().await;
+ let error = admin.create_topic(&topic_ref(), Some(1)).await.unwrap_err();
+ assert!(error.contains("timed out"), "unexpected error: {error}");
+ assert_recovered_topics(&admin).await;
+
+ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();
+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);
+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);
+ assert_eq!(calls.lines().filter(|line| *line == "mq_create_topic").count(), 1);
+ }
+
+ #[tokio::test]
+ async fn kafka_agent_recovers_after_exit_without_replaying_mutation() {
+ let (admin, directory) = recoverable_test_admin().await;
+ admin.delete_topic(&topic_ref(), false).await.expect_err("the agent exited before replying");
+ assert_recovered_topics(&admin).await;
+
+ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();
+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);
+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);
+ assert_eq!(calls.lines().filter(|line| *line == "mq_delete_topic").count(), 1);
+ }
+
+ #[tokio::test]
+ async fn kafka_agent_hanging_snapshot_is_bounded_and_recovers() {
+ let (admin, directory) = recoverable_test_admin().await;
+ let result =
+ tokio::time::timeout(std::time::Duration::from_secs(8), admin.get_kafka_consumer_group_snapshot()).await;
+ if result.is_err() {
+ // Dropping the old adapter also releases its blocked stdout worker.
+ drop(admin);
+ panic!("a stalled Kafka snapshot must have a finite outer RPC deadline");
+ }
+ let error = result.unwrap().unwrap_err();
+ assert!(error.contains("timed out"), "unexpected error: {error}");
+ assert_recovered_topics(&admin).await;
+
+ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();
+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 2);
+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 2);
+ }
+
+ #[tokio::test]
+ async fn kafka_agent_rpc_error_keeps_the_existing_process() {
+ let (admin, directory) = recoverable_test_admin().await;
+ let error = admin.get_topic_internal_stats(&topic_ref()).await.unwrap_err();
+ assert!(error.contains("Topic authorization failed"), "unexpected error: {error}");
+ assert_recovered_topics(&admin).await;
+
+ let calls = std::fs::read_to_string(directory.path().join("calls.txt")).unwrap();
+ assert_eq!(calls.lines().filter(|line| *line == "spawn").count(), 1);
+ assert_eq!(calls.lines().filter(|line| *line == "connect").count(), 1);
+ }
+
async fn delayed_consumer_group_admin() -> (KafkaAdmin, tempfile::TempPath) {
let script_path = tempfile::NamedTempFile::new().expect("create test agent script").into_temp_path();
std::fs::write(
@@ -1418,14 +1551,11 @@ for line in sys.stdin:
)
.expect("write test agent script");
let python = if cfg!(windows) { "python" } else { "python3" };
- let client = AgentDriverClient::spawn(
- AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),
- )
- .await
- .expect("spawn test agent");
+ let launch = AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]);
+ let client = AgentDriverClient::spawn(launch.clone()).await.expect("spawn test agent");
let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
config.query_timeout_secs = 1;
- (KafkaAdmin { client: Arc::new(Mutex::new(client)), config }, script_path)
+ (KafkaAdmin { client: Arc::new(Mutex::new(Some(client))), config, launch }, script_path)
}
#[tokio::test]
@@ -1489,15 +1619,12 @@ for line in sys.stdin:
.expect("write test agent script");
let python = if cfg!(windows) { "python" } else { "python3" };
- let client = AgentDriverClient::spawn(
- AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),
- )
- .await
- .expect("spawn test agent");
+ let launch = AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]);
+ let client = AgentDriverClient::spawn(launch.clone()).await.expect("spawn test agent");
// The delayed response models the Kafka agent returning its fallback after metadata timeout.
let mut config = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
config.query_timeout_secs = 1;
- let admin = KafkaAdmin { client: Arc::new(Mutex::new(client)), config };
+ let admin = KafkaAdmin { client: Arc::new(Mutex::new(Some(client))), config, launch };
let result = admin
.list_topics(