Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Fix race condition for timers around engine stop and timer reschedule (#2662)

Eric Bariaux1cd80ef0c3a4
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
index 67294907f..5000bf322 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
@@ -48,6 +48,7 @@ import org.openremote.model.syslog.SyslogCategory;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -116,7 +117,7 @@ public class RulesEngine<T extends Ruleset> {
final protected RulesFacts facts;
final protected AbstractRulesEngine engine;
- protected boolean running;
+ protected volatile boolean running;
protected AtomicBoolean executing = new AtomicBoolean(false);
protected boolean previouslyFired;
protected long lastFireTimestamp;
@@ -349,21 +350,26 @@ public class RulesEngine<T extends Ruleset> {
}
public void stop() {
+ ScheduledFuture<?> currentFireTimer;
+ ScheduledFuture<?> currentStatsTimer;
+
synchronized (this) {
if (!running) {
return;
}
running = false;
+ currentFireTimer = fireTimer;
+ fireTimer = null;
+ currentStatsTimer = statsTimer;
+ statsTimer = null;
}
LOG.info("Stopping: " + id);
- if (fireTimer != null) {
- fireTimer.cancel(true);
- fireTimer = null;
+ if (currentFireTimer != null) {
+ currentFireTimer.cancel(true);
}
- if (statsTimer != null) {
- statsTimer.cancel(true);
- statsTimer = null;
+ if (currentStatsTimer != null) {
+ currentStatsTimer.cancel(true);
}
new HashSet<>(deployments.values()).forEach(this::stopRuleset);
@@ -404,19 +410,24 @@ public class RulesEngine<T extends Ruleset> {
* once within the guaranteed minimum expiration time.
*/
protected synchronized void scheduleFire(boolean quickFire) {
- boolean timerRunning = fireTimer != null && !fireTimer.isDone();
+ if (!running) {
+ return;
+ }
+
+ ScheduledFuture<?> currentTimer = fireTimer;
+ boolean timerRunning = currentTimer != null && !currentTimer.isDone();
if (timerRunning) {
if (!quickFire) {
// Firing is already scheduled
return;
} else {
- if (fireTimer.getDelay(TimeUnit.MILLISECONDS) <= rulesService.quickFireMillis) {
+ if (currentTimer.getDelay(TimeUnit.MILLISECONDS) <= rulesService.quickFireMillis) {
// Firing is already going to occur within time frame
return;
} else {
// Cancel the existing timer
- fireTimer.cancel(false);
+ currentTimer.cancel(false);
}
}
}
@@ -424,29 +435,47 @@ public class RulesEngine<T extends Ruleset> {
long fireTimeMillis = quickFire ? rulesService.quickFireMillis : rulesService.tempFactExpirationMillis;
LOG.finest("Scheduling rules firing in " + fireTimeMillis + "ms");
- fireTimer = scheduledExecutorService.schedule(
- () -> {
- if (!running) {
- return;
- }
+ AtomicReference<ScheduledFuture<?>> scheduledTimer = new AtomicReference<>();
+ scheduledTimer.set(scheduledExecutorService.schedule(
+ () -> onFireTimerTriggered(scheduledTimer.get()),
+ fireTimeMillis,
+ TimeUnit.MILLISECONDS
+ ));
+ fireTimer = scheduledTimer.get();
+ }
- // If we're not already executing rules, process rules for all deployments
- if (executing.compareAndSet(false, true)) {
- fireAllDeployments();
- executing.set(false);
- }
+ protected void onFireTimerTriggered(ScheduledFuture<?> scheduledTimer) {
+ if (!running) {
+ return;
+ }
- if (Thread.currentThread().isInterrupted()) {
- LOG.finest("Timer interrupted during rules execution - not scheduling next fire");
- return;
- }
+ // If we're not already executing rules, process rules for all deployments
+ if (executing.compareAndSet(false, true)) {
+ try {
+ fireAllDeployments();
+ } finally {
+ executing.set(false);
+ }
+ }
- fireTimer = null;
- scheduleFire(false);
- },
- fireTimeMillis,
- TimeUnit.MILLISECONDS
- );
+ if (!running) {
+ return;
+ }
+
+ if (Thread.currentThread().isInterrupted()) {
+ LOG.finest("Timer interrupted during rules execution - not scheduling next fire");
+ return;
+ }
+
+ synchronized (this) {
+ // Ignore stale timer callbacks after a stop/start cycle or a reschedule.
+ if (fireTimer != scheduledTimer) {
+ return;
+ }
+ fireTimer = null;
+ }
+
+ scheduleFire(false);
}
protected void fireAllDeployments() {
diff --git a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
index 921e811a3..de22a42da 100644
--- a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
@@ -20,6 +20,8 @@ import org.openremote.test.ManagerContainerTrait
import spock.lang.Specification
import spock.util.concurrent.PollingConditions
+import java.util.concurrent.ScheduledFuture
+import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicBoolean
import static org.openremote.model.rules.Ruleset.Lang.GROOVY
@@ -99,6 +101,51 @@ class RulesEngineFiringTest extends Specification implements ManagerContainerTra
assert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"
}
+ @SuppressWarnings("GroovyAccessibility")
+ def "Check schedule fire tolerates concurrent timer clearing"() {
+ given: "the container is started"
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+ def container = startContainer(defaultConfig(), defaultServices())
+ def rulesService = container.getService(RulesService.class)
+ def rulesetStorageService = container.getService(RulesetStorageService.class)
+ RulesEngine engine = null
+
+ and: "a simple realm ruleset is deployed"
+ Ruleset ruleset = new RealmRuleset(
+ Constants.MASTER_REALM,
+ "Concurrent timer clearing",
+ GROOVY,
+ getClass().getResource("/org/openremote/test/rules/BasicMatchAllAssetStates.groovy").text)
+ rulesetStorageService.merge(ruleset)
+
+ expect: "the rules engine to become available and running"
+ conditions.eventually {
+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)
+ assert engine != null
+ assert engine.isRunning()
+ }
+
+ when: "the timer reference is cleared after the running check but before the delay lookup"
+ engine.stop()
+ engine.@running = true
+
+ ScheduledFuture<?> timer = Mock(ScheduledFuture)
+ timer.isDone() >> {
+ engine.@fireTimer = null
+ false
+ }
+ timer.getDelay(TimeUnit.MILLISECONDS) >> 0L
+ engine.@fireTimer = timer
+
+ engine.scheduleFire(true)
+
+ then: "the quick fire check does not dereference a null timer"
+ noExceptionThrown()
+
+ cleanup:
+ stopContainer()
+ }
+
/*
The exact timing issue is:
- fireTimer triggers and is executing the rules (fireAllDeployments)