100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix IO client reconnect and broken integration tests (#3346)
Koen0764c5d077ee
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.ymlindex 88df04254..f745867c7 100644--- a/.github/workflows/integration-tests.yml+++ b/.github/workflows/integration-tests.yml@@ -25,7 +25,7 @@ jobs:- name: Pull Docker imagesrun: docker compose -f profile/dev-testing.yml pull # Only Keycloak and PostgreSQL for running integration tests- name: Run integration tests- timeout-minutes: 20+ timeout-minutes: 40run: |# Run Integration Testsdiff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.javaindex 1856f42f9..d7d98351d 100644--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java@@ -28,6 +28,7 @@ import io.netty.channel.*;import io.netty.handler.codec.DecoderException;import io.netty.handler.codec.EncoderException;import jakarta.validation.constraints.NotNull;+import java.io.IOException;import java.time.Duration;import java.util.ArrayList;import java.util.Arrays;@@ -298,14 +299,29 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {if (ex != null) {// Cleanup resourcesdisconnect();- } else {- synchronized (this) {- if (connectionStatus == ConnectionStatus.CONNECTING) {+ return;+ }++ boolean reconnect = false;++ synchronized (this) {+ if (connectionStatus == ConnectionStatus.CONNECTING) {+ if (isChannelReady()) {LOG.fine("Connection attempt success: " + getClientUri());onConnectionStatusChanged(ConnectionStatus.CONNECTED);+ } else {+ // The channel closed between the attempt succeeding and this callback running, so+ // reporting it as connected would leave the client stuck on a dead channel with no+ // further close event to act on+ LOG.fine("Connection closed before it could be reported: " + getClientUri());+ reconnect = true;}}}++ if (reconnect) {+ doReconnect();+ }},executorService);@@ -482,6 +498,7 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {.addListener(closedFuture -> {boolean reconnect = false;+ CompletableFuture<Void> failedAttempt = null;if (!closedFuture.isSuccess() && closedFuture.cause() != null) {LOG.info(@@ -495,9 +512,20 @@ public abstract class AbstractNettyIOClient<T> implements NettyIOClient<T> {if (connectionStatus == ConnectionStatus.CONNECTED) {onConnectionStatusChanged(ConnectionStatus.CONNECTING);reconnect = true;+ } else if (connectionStatus == ConnectionStatus.CONNECTING) {+ // Closed before this attempt reported success, which happens when the peer+ // rejects the connection as soon as it is opened. Fail the attempt so the retry+ // policy handles it, otherwise the attempt goes on to report itself connected on+ // a channel that has already gone and nothing ever reconnects.+ failedAttempt = connectFuture;}}+ if (failedAttempt != null) {+ failedAttempt.completeExceptionally(+ new IOException("Connection closed while connecting: " + getClientUri()));+ }+if (reconnect) {doReconnect();}diff --git a/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovy b/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovynew file mode 100644index 000000000..991733871--- /dev/null+++ b/agent/src/test/groovy/org/openremote/agent/protocol/io/AbstractNettyIOClientTest.groovy@@ -0,0 +1,98 @@+/*+ * Copyright 2026, OpenRemote Inc.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <https://www.gnu.org/licenses/>.+ *+ * SPDX-License-Identifier: AGPL-3.0-or-later+ */+package org.openremote.agent.protocol.io++import io.netty.channel.Channel+import io.netty.channel.EventLoopGroup+import io.netty.channel.embedded.EmbeddedChannel+import org.openremote.model.asset.agent.ConnectionStatus+import spock.lang.Specification++import java.util.concurrent.CompletableFuture++class AbstractNettyIOClientTest extends Specification {++ /** Enough of a client to own a channel; nothing here connects to anything. */+ private static class StubIOClient extends AbstractNettyIOClient<String> {++ @Override+ protected Class<? extends Channel> getChannelClass() {+ return EmbeddedChannel+ }++ @Override+ protected EventLoopGroup getWorkerGroup() {+ return null+ }++ @Override+ protected CompletableFuture<Void> startChannel() {+ return new CompletableFuture<Void>()+ }++ @Override+ String getClientUri() {+ return "stub://test"+ }+ }++ def "a peer that closes the channel before the attempt reports success fails that attempt"() {++ given: "a client part way through a connection attempt"+ def client = new StubIOClient()+ client.connectionStatus = ConnectionStatus.CONNECTING+ def connectAttempt = new CompletableFuture<Void>()+ client.connectFuture = connectAttempt++ and: "a channel carrying the client's close handling"+ def channel = new EmbeddedChannel()+ client.initChannel(channel)++ when: "the peer closes it before the attempt has reported success"+ channel.close().sync()++ then: "the attempt fails, so the retry policy reconnects instead of the close being dropped"+ // A close seen while the status is still CONNECTING used to fall through silently, and the+ // attempt then reported itself connected on a channel that had already gone. Nothing closes a+ // second time, so the client stayed CONNECTED forever and never reconnected.+ connectAttempt.isCompletedExceptionally()++ and: "the client is not left claiming to be connected"+ client.connectionStatus != ConnectionStatus.CONNECTED+ }++ def "a close after the client has disconnected does not fail anything"() {++ given: "a client that is disconnecting"+ def client = new StubIOClient()+ client.connectionStatus = ConnectionStatus.DISCONNECTING+ def connectAttempt = new CompletableFuture<Void>()+ client.connectFuture = connectAttempt++ and: "a channel carrying the client's close handling"+ def channel = new EmbeddedChannel()+ client.initChannel(channel)++ when: "the channel closes as part of that disconnect"+ channel.close().sync()++ then: "the close is left alone, since a deliberate disconnect must not trigger a reconnect"+ !connectAttempt.isDone()+ }+}diff --git a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovyindex 9c3607fa6..fe5c3a9a0 100644--- a/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy+++ b/test/src/test/groovy/org/openremote/test/gateway/GatewayTest.groovy@@ -1533,14 +1533,29 @@ class GatewayTest extends Specification implements ManagerContainerTrait {given: "some polling conditions and services from the container"def gatewayService = container.getService(GatewayService.class)def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)+ def identityProvider = container.getService(ManagerIdentityService.class).identityProvider as ManagerKeycloakIdentityProvider++ and: "a user in the building realm holding the tunnel read role the endpoint requires"+ def tunnelReader = identityProvider.createUpdateUser(+ managerTestSetup.realmBuildingName,+ new User().setUsername("gatewayrealmtunnelreader").setEnabled(true),+ "gatewayrealmtunnelreader",+ true+ )+ identityProvider.updateUserClientRoles(+ managerTestSetup.realmBuildingName,+ tunnelReader.id,+ KEYCLOAK_CLIENT_ID,+ READ_TUNNELS_ROLE+ )- and: "an authenticated user in the building realm"+ and: "an authenticated session for that user"def accessToken = authenticate(container,managerTestSetup.realmBuildingName,KEYCLOAK_CLIENT_ID,- "testuser2",- "testuser2"+ "gatewayrealmtunnelreader",+ "gatewayrealmtunnelreader")and: "the gateway service resource"@@ -1557,10 +1572,13 @@ class GatewayTest extends Specification implements ManagerContainerTrait {then: "the building user should not be able to see any tunnels for the fake gateway ID in the smart city realm"assert activeTunnels.length == 0- cleanup: "the fake tunnel is removed"+ cleanup: "the fake tunnel and the user are removed"if (gatewayService != null) {[email protected]("fake-tunnel-id")}+ if (tunnelReader != null) {+ identityProvider.deleteUser(managerTestSetup.realmBuildingName, tunnelReader.id)+ }}def "Verify gateway websocket cannot write attributes outside gateway realm"() {diff --git a/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy b/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovyindex e492520d5..14241d833 100644--- a/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy+++ b/test/src/test/groovy/org/openremote/test/users/UserResourceTest.groovy@@ -681,6 +681,10 @@ class UserResourceTest extends Specification implements ManagerContainerTrait {then: "the cross-realm reset should be rejected"thrown(NotAllowedException)++ cleanup: "the fixture is given the live secret, since later specs authenticate as this service user"+ keycloakTestSetup.serviceUser.secret =+ adminUserResource.resetSecret(null, keycloakTestSetup.realmBuilding.name, keycloakTestSetup.serviceUser.id)}def "Update user client roles requires realm administration rights"() {