Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Fix IO client reconnect and broken integration tests (#3346)

Koen0764c5d077ee
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index 88df04254..f745867c7 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -25,7 +25,7 @@ jobs:
- name: Pull Docker images
run: docker compose -f profile/dev-testing.yml pull # Only Keycloak and PostgreSQL for running integration tests
- name: Run integration tests
- timeout-minutes: 20
+ timeout-minutes: 40
run: |
# Run Integration Tests
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
index 1856f42f9..d7d98351d 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
@@ -28,6 +28,7 @@ import io.netty.channel.*;
import io.netty.handler.codec.DecoderException;
import io.netty.handler.codec.EncoderException;
import jakarta.validation.constraints.NotNull;
+import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
@@ -298,14 +299,29 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {
if (ex != null) {
// Cleanup resources
disconnect();
- } else {
- synchronized (this) {
- if (connectionStatus == ConnectionStatus.CONNECTING) {
+ return;
+ }
+
+ boolean reconnect = false;
+
+ synchronized (this) {
+ if (connectionStatus == ConnectionStatus.CONNECTING) {
+ if (isChannelReady()) {
LOG.fine("Connection attempt success: " + getClientUri());
onConnectionStatusChanged(ConnectionStatus.CONNECTED);
+ } else {
+ // The channel closed between the attempt succeeding and this callback running, so
+ // reporting it as connected would leave the client stuck on a dead channel with no
+ // further close event to act on
+ LOG.fine("Connection closed before it could be reported: " + getClientUri());
+ reconnect = true;
}
}
}
+
+ if (reconnect) {
+ doReconnect();
+ }
},
executorService);
@@ -482,6 +498,7 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {
.addListener(
closedFuture -> {
boolean reconnect = false;
+ CompletableFuture<Void> failedAttempt = null;
if (!closedFuture.isSuccess() && closedFuture.cause() != null) {
LOG.info(
@@ -495,9 +512,20 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {
if (connectionStatus == ConnectionStatus.CONNECTED) {
onConnectionStatusChanged(ConnectionStatus.CONNECTING);
reconnect = true;
+ } else if (connectionStatus == ConnectionStatus.CONNECTING) {
+ // Closed before this attempt reported success, which happens when the peer
+ // rejects the connection as soon as it is opened. Fail the attempt so the retry
+ // policy handles it, otherwise the attempt goes on to report itself connected on
+ // a channel that has already gone and nothing ever reconnects.
+ failedAttempt = connectFuture;
}
}
+ if (failedAttempt != null) {
+ failedAttempt.completeExceptionally(
+ new IOException("Connection closed while connecting: " + getClientUri()));
+ }
+
if (reconnect) {
doReconnect();
}
diff --git a/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovy b/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovy
new file mode 100644
index 000000000..991733871
--- /dev/null
+++ b/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovy
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2026, OpenRemote Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+package org.openremote.agent.protocol.io
+
+import io.netty.channel.Channel
+import io.netty.channel.EventLoopGroup
+import io.netty.channel.embedded.EmbeddedChannel
+import org.openremote.model.asset.agent.ConnectionStatus
+import spock.lang.Specification
+
+import java.util.concurrent.CompletableFuture
+
+class AbstractNettyIOClientTest extends Specification {
+
+ /** Enough of a client to own a channel; nothing here connects to anything. */
+ private static class StubIOClient extends AbstractNettyIOClient<String> {
+
+ @Override
+ protected Class<? extends Channel> getChannelClass() {
+ return EmbeddedChannel
+ }
+
+ @Override
+ protected EventLoopGroup getWorkerGroup() {
+ return null
+ }
+
+ @Override
+ protected CompletableFuture<Void> startChannel() {
+ return new CompletableFuture<Void>()
+ }
+
+ @Override
+ String getClientUri() {
+ return "stub://test"
+ }
+ }
+
+ def "a peer that closes the channel before the attempt reports success fails that attempt"() {
+
+ given: "a client part way through a connection attempt"
+ def client = new StubIOClient()
+ client.connectionStatus = ConnectionStatus.CONNECTING
+ def connectAttempt = new CompletableFuture<Void>()
+ client.connectFuture = connectAttempt
+
+ and: "a channel carrying the client's close handling"
+ def channel = new EmbeddedChannel()
+ client.initChannel(channel)
+
+ when: "the peer closes it before the attempt has reported success"
+ channel.close().sync()
+
+ then: "the attempt fails, so the retry policy reconnects instead of the close being dropped"
+ // A close seen while the status is still CONNECTING used to fall through silently, and the
+ // attempt then reported itself connected on a channel that had already gone. Nothing closes a
+ // second time, so the client stayed CONNECTED forever and never reconnected.
+ connectAttempt.isCompletedExceptionally()
+
+ and: "the client is not left claiming to be connected"
+ client.connectionStatus != ConnectionStatus.CONNECTED
+ }
+
+ def "a close after the client has disconnected does not fail anything"() {
+
+ given: "a client that is disconnecting"
+ def client = new StubIOClient()
+ client.connectionStatus = ConnectionStatus.DISCONNECTING
+ def connectAttempt = new CompletableFuture<Void>()
+ client.connectFuture = connectAttempt
+
+ and: "a channel carrying the client's close handling"
+ def channel = new EmbeddedChannel()
+ client.initChannel(channel)
+
+ when: "the channel closes as part of that disconnect"
+ channel.close().sync()
+
+ then: "the close is left alone, since a deliberate disconnect must not trigger a reconnect"
+ !connectAttempt.isDone()
+ }
+}
diff --git a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy
index 9c3607fa6..fe5c3a9a0 100644
--- a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy
@@ -1533,14 +1533,29 @@ class GatewayTest extends Specification implements ManagerContainerTrait {
given: "some polling conditions and services from the container"
def gatewayService = container.getService(GatewayService.class)
def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
+ def identityProvider = container.getService(ManagerIdentityService.class).identityProvider as ManagerKeycloakIdentityProvider
+
+ and: "a user in the building realm holding the tunnel read role the endpoint requires"
+ def tunnelReader = identityProvider.createUpdateUser(
+ managerTestSetup.realmBuildingName,
+ new User().setUsername("gatewayrealmtunnelreader").setEnabled(true),
+ "gatewayrealmtunnelreader",
+ true
+ )
+ identityProvider.updateUserClientRoles(
+ managerTestSetup.realmBuildingName,
+ tunnelReader.id,
+ KEYCLOAK_CLIENT_ID,
+ READ_TUNNELS_ROLE
+ )
- and: "an authenticated user in the building realm"
+ and: "an authenticated session for that user"
def accessToken = authenticate(
container,
managerTestSetup.realmBuildingName,
KEYCLOAK_CLIENT_ID,
- "testuser2",
- "testuser2"
+ "gatewayrealmtunnelreader",
+ "gatewayrealmtunnelreader"
)
and: "the gateway service resource"
@@ -1557,10 +1572,13 @@ class GatewayTest extends Specification implements ManagerContainerTrait {
then: "the building user should not be able to see any tunnels for the fake gateway ID in the smart city realm"
assert activeTunnels.length == 0
- cleanup: "the fake tunnel is removed"
+ cleanup: "the fake tunnel and the user are removed"
if (gatewayService != null) {
[email protected]("fake-tunnel-id")
}
+ if (tunnelReader != null) {
+ identityProvider.deleteUser(managerTestSetup.realmBuildingName, tunnelReader.id)
+ }
}
def "Verify gateway websocket cannot write attributes outside gateway realm"() {
diff --git a/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy b/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy
index e492520d5..14241d833 100644
--- a/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy
@@ -681,6 +681,10 @@ class UserResourceTest extends Specification implements ManagerContainerTrait {
then: "the cross-realm reset should be rejected"
thrown(NotAllowedException)
+
+ cleanup: "the fixture is given the live secret, since later specs authenticate as this service user"
+ keycloakTestSetup.serviceUser.secret =
+ adminUserResource.resetSecret(null, keycloakTestSetup.realmBuilding.name, keycloakTestSetup.serviceUser.id)
}
def "Update user client roles requires realm administration rights"() {