Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Bug fix race condition in AbstractNettyIOClient connection logic (#2409)

Rich Turnerd7fe222a152c
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
index 5899e58b4..ccd500a39 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
@@ -235,7 +235,14 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
.withMaxRetries(Integer.MAX_VALUE)
.build();
- connectRetry = Failsafe.with(retryPolicy).with(executorService).runAsyncExecution((execution) -> {
+ CompletableFuture<Void> future = Failsafe.with(retryPolicy).with(executorService).runAsyncExecution((execution) -> {
+
+ synchronized (this) {
+ if (connectionStatus == ConnectionStatus.DISCONNECTED || connectionStatus == ConnectionStatus.DISCONNECTING) {
+ execution.recordResult(null);
+ return;
+ }
+ }
LOG.fine("Connection attempt '" + (execution.getAttemptCount()+1) + "' for: " + getClientUri());
// Connection future should timeout so we just wait for it but add additional timeout just in case
@@ -244,7 +251,7 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
execution.recordResult(null);
});
- connectRetry.whenComplete((result, ex) -> {
+ future.whenComplete((result, ex) -> {
if (ex != null) {
// Cleanup resources
disconnect();
@@ -257,6 +264,14 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
}
}
});
+
+ synchronized (this) {
+ if (connectionStatus == ConnectionStatus.DISCONNECTED || connectionStatus == ConnectionStatus.DISCONNECTING) {
+ future.cancel(true);
+ } else {
+ connectRetry = future;
+ }
+ }
}
protected Void waitForConnectFuture(Future<Void> connectFuture) throws Exception {
@@ -279,14 +294,15 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
LOG.fine("Disconnecting IO client: " + getClientUri());
onConnectionStatusChanged(ConnectionStatus.DISCONNECTING);
+
+ try {
+ if (connectRetry != null) {
+ connectRetry.cancel(true);
+ connectRetry = null;
+ }
+ } catch (Exception ignored) {}
}
- try {
- if (connectRetry != null) {
- connectRetry.cancel(true);
- connectRetry = null;
- }
- } catch (Exception ignored) {}
doDisconnect();
try {
if (workerGroup != null) {
diff --git a/test/build.gradle b/test/build.gradle
index b99d869ff..3eeb2bade 100644
--- a/test/build.gradle
+++ b/test/build.gradle
@@ -30,6 +30,7 @@ dependencies {
testImplementation "com.google.firebase:firebase-admin:$firebaseAdminVersion"
testImplementation "org.shredzone.commons:commons-suncalc:$suncalcVersion"
testImplementation "com.icegreen:greenmail:$greenmailVersion"
+ testImplementation "dev.failsafe:failsafe:$failsafeVersion"
testImplementation "net.solarnetwork.common:nifty-modbus-tcp:$niftyModbusVersion"
testImplementation "net.solarnetwork.common:nifty-modbus-rtu-jsc:$niftyModbusVersion"
diff --git a/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovy
new file mode 100644
index 000000000..5503dd830
--- /dev/null
+++ b/test/src/test/groovy/org/openremote/test/protocol/io/NettyIOClientTest.groovy
@@ -0,0 +1,103 @@
+package org.openremote.test.protocol.io
+
+
+import io.netty.util.concurrent.Future
+import io.netty.util.concurrent.GlobalEventExecutor
+import io.netty.util.concurrent.Promise
+import org.apache.http.client.utils.URIBuilder
+import org.openremote.agent.protocol.io.AbstractNettyIOClient
+import org.openremote.manager.gateway.GatewayIOClient
+import org.openremote.model.asset.agent.ConnectionStatus
+import org.openremote.model.auth.OAuthClientCredentialsGrant
+import org.openremote.test.ManagerContainerTrait
+import spock.lang.Specification
+import spock.util.concurrent.PollingConditions
+
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicInteger
+
+class NettyIOClientTest extends Specification implements ManagerContainerTrait {
+
+ def "Concurrency test for reconnect logic"() {
+
+ given: "A NettyIOClient that pauses before scheduling connection"
+ // Speed up the retry interval for the test
+ def oldDelay = AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS
+ AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = 10
+
+ def pauseLatch = new CountDownLatch(1)
+ def startChannelCount = new AtomicInteger(0)
+
+ def client = new GatewayIOClient(
+ new URIBuilder("ws://127.0.0.1:$serverPort/websocket/events?Realm=random").build(),
+ null,
+ new OAuthClientCredentialsGrant("http://127.0.0.1:$serverPort/auth/realms/random/protocol/openid-connect/token",
+ 'abcd',
+ 'abcd',
+ null).setBasicAuthHeader(true)
+ ) {
+ @Override
+ protected Future<Void> startChannel() {
+ startChannelCount.incrementAndGet()
+ // Simulate failure because resources (bootstrap) are destroyed by disconnect()
+ if (this.bootstrap == null) {
+ throw new NullPointerException("Bootstrap is null")
+ }
+ // Return a dummy future (won't be reached if exception thrown above)
+ Promise<Void> p = GlobalEventExecutor.INSTANCE.newPromise()
+ p.setFailure(new Exception("Fail"))
+ return p
+ }
+
+ @Override
+ protected void scheduleDoConnect(long initialDelay) {
+ // Pause here to simulate concurrency issue during connect/reconnect.
+ // We pause here after connect() has set status to CONNECTING and initialized bootstrap,
+ // but BEFORE the retry future is created and assigned to 'connectRetry'.
+ try {
+ pauseLatch.await()
+ } catch (InterruptedException e) {
+ e.printStackTrace()
+ }
+ super.scheduleDoConnect(initialDelay)
+ }
+ }
+
+ // Use a dedicated executor for the retry logic
+ client.executorService = Executors.newCachedThreadPool()
+
+ when: "Connect is called in a separate thread"
+ Thread.start {
+ client.connect()
+ }
+
+ // Wait until the client enters the connecting state (and hits the pauseLatch)
+ new PollingConditions().eventually {
+ assert client.connectionStatus == ConnectionStatus.CONNECTING
+ }
+ // Small sleep to ensure we are waiting on the latch inside scheduleDoConnect
+ Thread.sleep(100)
+
+ and: "Disconnect is called immediately by the main thread"
+ // This runs while the other thread is paused.
+ // Crucially, 'connectRetry' is still null in the client, so disconnect() fails to cancel any future.
+ // It destroys the bootstrap and sets status to DISCONNECTED.
+ client.disconnect()
+
+ then: "Client status is DISCONNECTED"
+ client.connectionStatus == ConnectionStatus.DISCONNECTED
+
+ when: "The paused connect thread is allowed to proceed"
+ pauseLatch.countDown()
+
+ then: "No attempt should be made to actually start the channel"
+ new PollingConditions(initialDelay: 1, timeout: 5, delay: 2).eventually {
+ assert startChannelCount.get() == 0
+ }
+
+ cleanup:
+ AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = oldDelay
+ client.executorService.shutdownNow()
+ }
+}