Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Fixes for IO client reconnection

Rich Turner01613d10c42f
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractIOClientProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractIOClientProtocol.java
index a95a0106b..eae0168e9 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractIOClientProtocol.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractIOClientProtocol.java
@@ -142,7 +142,7 @@ public abstract class AbstractIOClientProtocol<T extends AbstractIOClientProtoco
LOG.fine("Stopping IO client for protocol: " + this);
client.removeAllMessageConsumers();
client.removeAllConnectionStatusConsumers();
- AbstractIOClientProtocol.LOG.info("Disconnecting IO client");
+ LOG.info("Disconnecting IO client");
client.disconnect();
}
client = null;
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
index ad1e17e4a..38e664462 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOClient.java
@@ -211,16 +211,28 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
onConnectionStatusChanged(ConnectionStatus.CONNECTING);
}
- scheduleDoConnect();
+ scheduleDoConnect(true);
}
- protected void scheduleDoConnect() {
+ protected void scheduleDoConnect(boolean immediate) {
+ long jitter = immediate ? 50 : Math.max(50, RECONNECT_DELAY_JITTER_MILLIS);
+ long delay = immediate ? 50 : Math.max(50, RECONNECT_DELAY_INITIAL_MILLIS);
+ long maxDelay = immediate ? 51 : Math.max(delay+1, Math.max(50, RECONNECT_DELAY_MAX_MILLIS));
+
RetryPolicy<Object> retryPolicy = RetryPolicy.builder()
- .withJitter(Duration.ofMillis(Math.max(50, RECONNECT_DELAY_JITTER_MILLIS)))
- .withBackoff(Duration.ofMillis(Math.max(50, RECONNECT_DELAY_INITIAL_MILLIS)), Duration.ofMillis(Math.max(100, RECONNECT_DELAY_MAX_MILLIS)))
+ .withJitter(Duration.ofMillis(jitter))
+ .withDelay(Duration.ofMillis(delay))
+ .withBackoff(Duration.ofMillis(delay), Duration.ofMillis(maxDelay))
+ .handle(RuntimeException.class)
+ .onRetryScheduled((execution) ->
+ LOG.fine("Re-connection scheduled in '" + execution.getDelay() + "' for: " + getClientUri()))
+ .onFailedAttempt((execution) ->
+ LOG.fine("Re-connection attempt failed '" + execution.getAttemptCount() + "' for: " + getClientUri()))
+ .onRetry((execution) ->
+ LOG.fine("Re-connection attempt '" + (execution.getAttemptCount()+1) + "' for: " + getClientUri()))
+ .withMaxRetries(Integer.MAX_VALUE)
.build();
-
connectRetry = Failsafe.with(retryPolicy).with(executorService).runAsyncExecution((execution) -> {
boolean success = doConnect().get();
@@ -235,6 +247,7 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
} else {
// Cleanup resources ready for next connection attempt
doDisconnect();
+ execution.recordFailure(new RuntimeException("Connection attempt failed"));
}
});
}
@@ -274,7 +287,7 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
LOG.info("Connection closed un-expectedly: " + getClientUri());
onConnectionStatusChanged(ConnectionStatus.CONNECTING);
doDisconnect();
- scheduleDoConnect();
+ scheduleDoConnect(false);
}
}
});
@@ -313,7 +326,7 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
return;
}
- LOG.finest("Disconnecting IO client: " + getClientUri());
+ LOG.fine("Disconnecting IO client: " + getClientUri());
onConnectionStatusChanged(ConnectionStatus.DISCONNECTING);
}
@@ -321,8 +334,9 @@ public abstract class AbstractNettyIOClient<T, U extends SocketAddress> implemen
connectRetry.cancel(false);
try {
connectRetry.get();
- } catch (InterruptedException | ExecutionException e) {
- LOG.log(Level.INFO, "Failed to wait for connection retry future: " + getClientUri());
+ } catch (CancellationException | InterruptedException | ExecutionException ignored) {
+ } catch (Exception e) {
+ LOG.log(Level.FINE, "Failed to wait for connection retry future: " + getClientUri());
}
connectRetry = null;
}
diff --git a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOServer.java b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOServer.java
index f895a8a1c..78e926411 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOServer.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/io/AbstractNettyIOServer.java
@@ -145,7 +145,7 @@ public abstract class AbstractNettyIOServer<T, U extends Channel, V extends Abst
return;
}
- LOG.finest("Stopping IO Server: " + getSocketAddressString());
+ LOG.fine("Stopping IO Server: " + getSocketAddressString());
onConnectionStatusChanged(ConnectionStatus.DISCONNECTING);
try {
diff --git a/test/src/test/groovy/org/openremote/test/protocol/TcpClientTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/TcpClientTest.groovy
index bb9f63d4b..a0d1042e3 100644
--- a/test/src/test/groovy/org/openremote/test/protocol/TcpClientTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/protocol/TcpClientTest.groovy
@@ -27,6 +27,8 @@ import org.openremote.agent.protocol.io.AbstractNettyIOClient
import org.openremote.agent.protocol.tcp.TCPIOClient
import org.openremote.agent.protocol.tcp.TCPStringServer
import org.openremote.model.asset.agent.ConnectionStatus
+import org.openremote.model.notification.AbstractNotificationMessage
+import org.openremote.model.notification.Notification
import org.openremote.test.ManagerContainerTrait
import spock.lang.Specification
import spock.util.concurrent.PollingConditions
@@ -41,14 +43,6 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
given: "expected conditions"
def conditions = new PollingConditions(timeout: 20, delay: 0.2)
- and: "the IO client reconnect time is set low for test purposes"
- def initialMillis = AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS
- def maxMillis = AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS
- def jitterMillis = AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = 0
-
and: "the container is started"
def container = startContainer(defaultConfig(), [])
@@ -60,6 +54,7 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
})
and: "a simple TCP client"
+ def connectAttempts = 0
TCPIOClient<String> client = new TCPIOClient<String>(
"127.0.0.1",
echoServerPort)
@@ -68,6 +63,11 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
new StringDecoder(CharsetUtil.UTF_8),
new AbstractNettyIOClient.MessageToMessageDecoder<String>(String.class, client)].toArray(new ChannelHandler[0])
})
+ client = Spy(client)
+ client.doConnect() >> {
+ connectAttempts++
+ callRealMethod()
+ }
and: "we add callback consumers to the client"
def connectionStatus = client.getConnectionStatus()
@@ -150,12 +150,19 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
}
when: "we lose connection to the server"
+ connectAttempts = 0
echoServer.stop()
- then: "the client status should change to CONNECTING"
+ then: "the server status should be DISCONNECTED"
+ conditions.eventually {
+ assert echoServer.connectionStatus == ConnectionStatus.DISCONNECTED
+ }
+
+ then: "the client status should change to CONNECTING and several re-connection attempts should be made"
conditions.eventually {
assert client.connectionStatus == ConnectionStatus.CONNECTING
assert connectionStatus == ConnectionStatus.CONNECTING
+ assert connectAttempts > 2
}
when: "the connection to the server is restored"
@@ -167,6 +174,9 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
retries++
}
+ then: "the server is connected"
+ assert echoServer.connectionStatus == ConnectionStatus.CONNECTED
+
then: "the client status should become CONNECTED"
conditions.eventually {
assert client.connectionStatus == ConnectionStatus.CONNECTED
@@ -200,9 +210,6 @@ class TcpClientTest extends Specification implements ManagerContainerTrait {
}
cleanup: "the server should be stopped"
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = initialMillis
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = maxMillis
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = jitterMillis
client.disconnect()
echoServer.stop()
}
diff --git a/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientTest.groovy
index 143d643ea..35b76ba7f 100644
--- a/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/protocol/udp/UdpClientTest.groovy
@@ -44,14 +44,6 @@ class UdpClientTest extends Specification implements ManagerContainerTrait {
given: "expected conditions"
def conditions = new PollingConditions(timeout: 20, delay: 0.2)
- and: "the IO client reconnect time is set low for test purposes"
- def initialMillis = AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS
- def maxMillis = AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS
- def jitterMillis = AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = 0
-
and: "the container is started"
def clientPort = findEphemeralPort()
def container = startContainer(defaultConfig(), [])
@@ -172,9 +164,6 @@ class UdpClientTest extends Specification implements ManagerContainerTrait {
}
cleanup: "the server should be stopped"
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = initialMillis
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = maxMillis
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = jitterMillis
client.disconnect()
echoServer.stop()
}
diff --git a/test/src/test/groovy/org/openremote/test/protocol/websocket/WebsocketClientTest.groovy b/test/src/test/groovy/org/openremote/test/protocol/websocket/WebsocketClientTest.groovy
index 010b97689..83d2fc2db 100644
--- a/test/src/test/groovy/org/openremote/test/protocol/websocket/WebsocketClientTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/protocol/websocket/WebsocketClientTest.groovy
@@ -75,13 +75,6 @@ class WebsocketClientTest extends Specification implements ManagerContainerTrait
given: "expected conditions"
def conditions = new PollingConditions(timeout: 20, delay: 0.2)
- and: "the IO client reconnect time is set low for test purposes"
- def initialMillis = AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS
- def maxMillis = AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS
- def jitterMillis = AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = 500
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = 0
and: "the container is started"
def container = startContainer(defaultConfig(), defaultServices())
@@ -216,9 +209,6 @@ class WebsocketClientTest extends Specification implements ManagerContainerTrait
}
cleanup: "the server should be stopped"
- AbstractNettyIOClient.RECONNECT_DELAY_INITIAL_MILLIS = initialMillis
- AbstractNettyIOClient.RECONNECT_DELAY_MAX_MILLIS = maxMillis
- AbstractNettyIOClient.RECONNECT_DELAY_JITTER_MILLIS = jitterMillis
if (client != null) {
client.disconnect()
}