100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Bug fix race condition in AbstractNettyIOClient connection logic (#2409)
Rich Turnerd7fe222a152c
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.javaindex 5899e58b4..ccd500a39 100644--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java@@ -235,7 +235,14 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen.withMaxRetries(Integer.MAX_VALUE).build();- connectRetry = Failsafe.with(retryPolicy).with(executorService).runAsyncExecution((execution) -> {+ CompletableFuture<Void> future = Failsafe.with(retryPolicy).with(executorService).runAsyncExecution((execution) -> {++ synchronized (this) {+ if (connectionStatus == ConnectionStatus.DISCONNECTED || connectionStatus == ConnectionStatus.DISCONNECTING) {+ execution.recordResult(null);+ return;+ }+ }LOG.fine("Connection attempt '" + (execution.getAttemptCount()+1) + "' for: " + getClientUri());// Connection future should timeout so we just wait for it but add additional timeout just in case@@ -244,7 +251,7 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemenexecution.recordResult(null);});- connectRetry.whenComplete((result, ex) -> {+ future.whenComplete((result, ex) -> {if (ex != null) {// Cleanup resourcesdisconnect();@@ -257,6 +264,14 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen}}});++ synchronized (this) {+ if (connectionStatus == ConnectionStatus.DISCONNECTED || connectionStatus == ConnectionStatus.DISCONNECTING) {+ future.cancel(true);+ } else {+ connectRetry = future;+ }+ }}protected Void waitForConnectFuture(Future<Void> connectFuture) throws Exception {@@ -279,14 +294,15 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemenLOG.fine("Disconnecting IO client: " + getClientUri());onConnectionStatusChanged(ConnectionStatus.DISCONNECTING);++ try {+ if (connectRetry != null) {+ connectRetry.cancel(true);+ connectRetry = null;+ }+ } catch (Exception ignored) {}}- try {- if (connectRetry != null) {- connectRetry.cancel(true);- connectRetry = null;- }- } catch (Exception ignored) {}doDisconnect();try {if (workerGroup != null) {diff --git a/test/build.gradle b/test/build.gradleindex b99d869ff..3eeb2bade 100644--- a/test/build.gradle+++ b/test/build.gradle@@ -30,6 +30,7 @@ dependencies {testImplementation "com.google.firebase:firebase-admin:$firebaseAdminVersion"testImplementation "org.shredzone.commons:commons-suncalc:$suncalcVersion"testImplementation "com.icegreen:greenmail:$greenmailVersion"+ testImplementation "dev.failsafe:failsafe:$failsafeVersion"testImplementation "net.solarnetwork.common:nifty-modbus-tcp:$niftyModbusVersion"testImplementation "net.solarnetwork.common:nifty-modbus-rtu-jsc:$niftyModbusVersion"diff --git a/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovynew file mode 100644index 000000000..5503dd830--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovy@@ -0,0 +1,103 @@+package org.openremote.test.protocol.io+++import io.netty.util.concurrent.Future+import io.netty.util.concurrent.GlobalEventExecutor+import io.netty.util.concurrent.Promise+import org.apache.http.client.utils.URIBuilder+import org.openremote.agent.protocol.io.AbstractNettyIOClient+import org.openremote.manager.gateway.GatewayIOClient+import org.openremote.model.asset.agent.ConnectionStatus+import org.openremote.model.auth.OAuthClientCredentialsGrant+import org.openremote.test.ManagerContainerTrait+import spock.lang.Specification+import spock.util.concurrent.PollingConditions++import java.util.concurrent.CountDownLatch+import java.util.concurrent.Executors+import java.util.concurrent.atomic.AtomicInteger++class NettyIOClientTest extends Specification implements ManagerContainerTrait {++ def "Concurrency test for reconnect logic"() {++ given: "A NettyIOClient that pauses before scheduling connection"+ // Speed up the retry interval for the test+ def oldDelay = AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS+ AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = 10++ def pauseLatch = new CountDownLatch(1)+ def startChannelCount = new AtomicInteger(0)++ def client = new GatewayIOClient(+ new URIBuilder("ws://127.0.0.1:$serverPort/websocket/events?Realm=random").build(),+ null,+ new OAuthClientCredentialsGrant("http://127.0.0.1:$serverPort/auth/realms/random/protocol/openid-connect/token",+ 'abcd',+ 'abcd',+ null).setBasicAuthHeader(true)+ ) {+ @Override+ protected Future<Void> startChannel() {+ startChannelCount.incrementAndGet()+ // Simulate failure because resources (bootstrap) are destroyed by disconnect()+ if (this.bootstrap == null) {+ throw new NullPointerException("Bootstrap is null")+ }+ // Return a dummy future (won't be reached if exception thrown above)+ Promise<Void> p = GlobalEventExecutor.INSTANCE.newPromise()+ p.setFailure(new Exception("Fail"))+ return p+ }++ @Override+ protected void scheduleDoConnect(long initialDelay) {+ // Pause here to simulate concurrency issue during connect/reconnect.+ // We pause here after connect() has set status to CONNECTING and initialized bootstrap,+ // but BEFORE the retry future is created and assigned to 'connectRetry'.+ try {+ pauseLatch.await()+ } catch (InterruptedException e) {+ e.printStackTrace()+ }+ super.scheduleDoConnect(initialDelay)+ }+ }++ // Use a dedicated executor for the retry logic+ client.executorService = Executors.newCachedThreadPool()++ when: "Connect is called in a separate thread"+ Thread.start {+ client.connect()+ }++ // Wait until the client enters the connecting state (and hits the pauseLatch)+ new PollingConditions().eventually {+ assert client.connectionStatus == ConnectionStatus.CONNECTING+ }+ // Small sleep to ensure we are waiting on the latch inside scheduleDoConnect+ Thread.sleep(100)++ and: "Disconnect is called immediately by the main thread"+ // This runs while the other thread is paused.+ // Crucially, 'connectRetry' is still null in the client, so disconnect() fails to cancel any future.+ // It destroys the bootstrap and sets status to DISCONNECTED.+ client.disconnect()++ then: "Client status is DISCONNECTED"+ client.connectionStatus == ConnectionStatus.DISCONNECTED++ when: "The paused connect thread is allowed to proceed"+ pauseLatch.countDown()++ then: "No attempt should be made to actually start the channel"+ new PollingConditions(initialDelay: 1, timeout: 5, delay: 2).eventually {+ assert startChannelCount.get() == 0+ }++ cleanup:+ AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = oldDelay+ client.executorService.shutdownNow()+ }+}