100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix race condition for timers around engine stop and timer reschedule (#2662)
Eric Bariaux1cd80ef0c3a4
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.javaindex 67294907f..5000bf322 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java@@ -48,6 +48,7 @@ import org.openremote.model.syslog.SyslogCategory;import java.util.*;import java.util.concurrent.*;import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;import java.util.logging.Level;import java.util.logging.Logger;@@ -116,7 +117,7 @@ public class RulesEngine<T extends Ruleset> {final protected RulesFacts facts;final protected AbstractRulesEngine engine;- protected boolean running;+ protected volatile boolean running;protected AtomicBoolean executing = new AtomicBoolean(false);protected boolean previouslyFired;protected long lastFireTimestamp;@@ -349,21 +350,26 @@ public class RulesEngine<T extends Ruleset> {}public void stop() {+ ScheduledFuture<?> currentFireTimer;+ ScheduledFuture<?> currentStatsTimer;+synchronized (this) {if (!running) {return;}running = false;+ currentFireTimer = fireTimer;+ fireTimer = null;+ currentStatsTimer = statsTimer;+ statsTimer = null;}LOG.info("Stopping: " + id);- if (fireTimer != null) {- fireTimer.cancel(true);- fireTimer = null;+ if (currentFireTimer != null) {+ currentFireTimer.cancel(true);}- if (statsTimer != null) {- statsTimer.cancel(true);- statsTimer = null;+ if (currentStatsTimer != null) {+ currentStatsTimer.cancel(true);}new HashSet<>(deployments.values()).forEach(this::stopRuleset);@@ -404,19 +410,24 @@ public class RulesEngine<T extends Ruleset> {* once within the guaranteed minimum expiration time.*/protected synchronized void scheduleFire(boolean quickFire) {- boolean timerRunning = fireTimer != null && !fireTimer.isDone();+ if (!running) {+ return;+ }++ ScheduledFuture<?> currentTimer = fireTimer;+ boolean timerRunning = currentTimer != null && !currentTimer.isDone();if (timerRunning) {if (!quickFire) {// Firing is already scheduledreturn;} else {- if (fireTimer.getDelay(TimeUnit.MILLISECONDS) <= rulesService.quickFireMillis) {+ if (currentTimer.getDelay(TimeUnit.MILLISECONDS) <= rulesService.quickFireMillis) {// Firing is already going to occur within time framereturn;} else {// Cancel the existing timer- fireTimer.cancel(false);+ currentTimer.cancel(false);}}}@@ -424,29 +435,47 @@ public class RulesEngine<T extends Ruleset> {long fireTimeMillis = quickFire ? rulesService.quickFireMillis : rulesService.tempFactExpirationMillis;LOG.finest("Scheduling rules firing in " + fireTimeMillis + "ms");- fireTimer = scheduledExecutorService.schedule(- () -> {- if (!running) {- return;- }+ AtomicReference<ScheduledFuture<?>> scheduledTimer = new AtomicReference<>();+ scheduledTimer.set(scheduledExecutorService.schedule(+ () -> onFireTimerTriggered(scheduledTimer.get()),+ fireTimeMillis,+ TimeUnit.MILLISECONDS+ ));+ fireTimer = scheduledTimer.get();+ }- // If we're not already executing rules, process rules for all deployments- if (executing.compareAndSet(false, true)) {- fireAllDeployments();- executing.set(false);- }+ protected void onFireTimerTriggered(ScheduledFuture<?> scheduledTimer) {+ if (!running) {+ return;+ }- if (Thread.currentThread().isInterrupted()) {- LOG.finest("Timer interrupted during rules execution - not scheduling next fire");- return;- }+ // If we're not already executing rules, process rules for all deployments+ if (executing.compareAndSet(false, true)) {+ try {+ fireAllDeployments();+ } finally {+ executing.set(false);+ }+ }- fireTimer = null;- scheduleFire(false);- },- fireTimeMillis,- TimeUnit.MILLISECONDS- );+ if (!running) {+ return;+ }++ if (Thread.currentThread().isInterrupted()) {+ LOG.finest("Timer interrupted during rules execution - not scheduling next fire");+ return;+ }++ synchronized (this) {+ // Ignore stale timer callbacks after a stop/start cycle or a reschedule.+ if (fireTimer != scheduledTimer) {+ return;+ }+ fireTimer = null;+ }++ scheduleFire(false);}protected void fireAllDeployments() {diff --git a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovyindex 921e811a3..de22a42da 100644--- a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy+++ b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy@@ -20,6 +20,8 @@ import org.openremote.test.ManagerContainerTraitimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions+import java.util.concurrent.ScheduledFuture+import java.util.concurrent.TimeUnitimport java.util.concurrent.atomic.AtomicBooleanimport static org.openremote.model.rules.Ruleset.Lang.GROOVY@@ -99,6 +101,51 @@ class RulesEngineFiringTest extends Specification implements ManagerContainerTraassert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"}+ @SuppressWarnings("GroovyAccessibility")+ def "Check schedule fire tolerates concurrent timer clearing"() {+ given: "the container is started"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def container = startContainer(defaultConfig(), defaultServices())+ def rulesService = container.getService(RulesService.class)+ def rulesetStorageService = container.getService(RulesetStorageService.class)+ RulesEngine engine = null++ and: "a simple realm ruleset is deployed"+ Ruleset ruleset = new RealmRuleset(+ Constants.MASTER_REALM,+ "Concurrent timer clearing",+ GROOVY,+ getClass().getResource("/org/openremote/test/rules/BasicMatchAllAssetStates.groovy").text)+ rulesetStorageService.merge(ruleset)++ expect: "the rules engine to become available and running"+ conditions.eventually {+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)+ assert engine != null+ assert engine.isRunning()+ }++ when: "the timer reference is cleared after the running check but before the delay lookup"+ engine.stop()+ engine.@running = true++ ScheduledFuture<?> timer = Mock(ScheduledFuture)+ timer.isDone() >> {+ engine.@fireTimer = null+ false+ }+ timer.getDelay(TimeUnit.MILLISECONDS) >> 0L+ engine.@fireTimer = timer++ engine.scheduleFire(true)++ then: "the quick fire check does not dereference a null timer"+ noExceptionThrown()++ cleanup:+ stopContainer()+ }+/*The exact timing issue is:- fireTimer triggers and is executing the rules (fireAllDeployments)