100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Bug fix for parallel rules execution (#2051)
Eric Bariaux2b2889c8e865
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.javaindex ad4c832f9..67294907f 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java@@ -47,6 +47,7 @@ import org.openremote.model.syslog.SyslogCategory;import java.util.*;import java.util.concurrent.*;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.logging.Level;import java.util.logging.Logger;@@ -116,6 +117,7 @@ public class RulesEngine<T extends Ruleset> {final protected AbstractRulesEngine engine;protected boolean running;+ protected AtomicBoolean executing = new AtomicBoolean(false);protected boolean previouslyFired;protected long lastFireTimestamp;protected boolean trackLocationPredicates;@@ -427,8 +429,18 @@ public class RulesEngine<T extends Ruleset> {if (!running) {return;}- // Process rules for all deployments- fireAllDeployments();++ // If we're not already executing rules, process rules for all deployments+ if (executing.compareAndSet(false, true)) {+ fireAllDeployments();+ executing.set(false);+ }++ if (Thread.currentThread().isInterrupted()) {+ LOG.finest("Timer interrupted during rules execution - not scheduling next fire");+ return;+ }+fireTimer = null;scheduleFire(false);},diff --git a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovyindex 5c82855f6..921e811a3 100644--- a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy+++ b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy@@ -89,6 +89,179 @@ class RulesEngineFiringTest extends Specification implements ManagerContainerTrafalse}+ then: "A new attribute event is sent"+ conditions.eventually {+ // The exact value does not matter, as the rule when is always true if the attribute exists+ // it just needs to be long enough for a potential co-execution to happen+ // and short enough to be reached before the timeout+ assert receivedEvents.size() == 4+ }+ assert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"+ }++ /*+ The exact timing issue is:+ - fireTimer triggers and is executing the rules (fireAllDeployments)+ - stop is called, cancelling the fireTimer and setting it to null+ - start is called, creating a new fireTimer+ - in this case, that new timer does not fire yet and fireAllDeployments returns,+ the fire lambda sets fireTimer to null and creates a new timer+ As a result, two timers are active at the same time+ */+ @SuppressWarnings("GroovyAccessibility")+ def "Check rules engine are not running in parallel on engine restart with new firing after rule execution"() {+ given: "the container is started"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def container = startContainer(defaultConfig(), defaultServices())+ def rulesService = container.getService(RulesService.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetProcessingService = container.getService(AssetProcessingService.class)+ def rulesetStorageService = container.getService(RulesetStorageService.class)+ RulesEngine engine = null++ and: "some rulesets with a long running rule"+ Ruleset ruleset = new RealmRuleset(+ Constants.MASTER_REALM,+ "Parallel Engine check rules",+ GROOVY,+ getClass().getResource("/org/openremote/test/rules/ParallelEngineCheck.groovy").text)+ Long masterRulesetId = rulesetStorageService.merge(ruleset).id++ and: "assets are added"+ def counterAsset = new ThingAsset("CounterAsset")+ .setId(UniqueIdentifierGenerator.generateId("CounterAsset"))+ .setRealm(Constants.MASTER_REALM)+ .addAttributes(+ new Attribute<>("count", ValueType.INTEGER, null).addMeta(new MetaItem<>(MetaItemType.RULE_STATE)),+ )+ counterAsset = assetStorageService.merge(counterAsset)++ expect: "the rule engines to become available and be running"+ conditions.eventually {+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)+ assert engine != null+ assert engine.isRunning()+ }++ when: "a mock rules engine is created"+ // We want to stop rules evaluation while doing this setup, otherwise our count of attribute event change is biased+ rulesService.stop(container)+ RulesEngine mockRulesEngine = Spy(engine)+ AtomicBoolean rulesExecuting = new AtomicBoolean(false)+ AtomicBoolean parallelExecutionOccurred = new AtomicBoolean(false)+ mockRulesEngine.fireAllDeployments() >> {+ println("Rules executing")+ if (rulesExecuting.getAndSet(true)) {+ parallelExecutionOccurred.set(true)+ }+ callRealMethod()+ rulesExecuting.set(false)+ }+ rulesService.realmEngines.put(Constants.MASTER_REALM, mockRulesEngine)+ rulesService.start(container)++ and: "An attribute has changed value"+ def receivedEvents = new ArrayList<AttributeEvent>()+ assetProcessingService.addEventInterceptor { EntityManager em, AttributeEvent event ->+ receivedEvents.add(event)+ false+ }+++ and: "the engine is restarted while rules are evaluating"+ conditions.eventually {+ assert(rulesExecuting.get())+ }+ mockRulesEngine.stop()+ // We don't want to start immediately, we want the start code to execute while the rules are evaluating+ // but the firing of the timer created by this call to happen after the rules have evaluated+ Thread.sleep(300)+ mockRulesEngine.start()++ then: "A new attribute event is sent"+ conditions.eventually {+ assert receivedEvents.size() == 4+ }+ assert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"+ }++ /*+ The exact timing issue is:+ - fireTimer triggers and is executing the rules (fireAllDeployments)+ - stop is called, cancelling the fireTimer and setting it to null+ - start is called, creating a new fireTimer+ - in this case, that new timer already fires while fireAllDeployments is still running+ As a result, the newly created timer triggers while the original rules execution is still happening+ */+ @SuppressWarnings("GroovyAccessibility")+ def "Check rules engine are not running in parallel on engine restart with new firing during current rule execution"() {+ given: "the container is started"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def container = startContainer(defaultConfig(), defaultServices())+ def rulesService = container.getService(RulesService.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetProcessingService = container.getService(AssetProcessingService.class)+ def rulesetStorageService = container.getService(RulesetStorageService.class)+ RulesEngine engine = null++ and: "some rulesets with a long running rule"+ Ruleset ruleset = new RealmRuleset(+ Constants.MASTER_REALM,+ "Parallel Engine check rules",+ GROOVY,+ getClass().getResource("/org/openremote/test/rules/ParallelEngineCheck.groovy").text)+ Long masterRulesetId = rulesetStorageService.merge(ruleset).id++ and: "assets are added"+ def counterAsset = new ThingAsset("CounterAsset")+ .setId(UniqueIdentifierGenerator.generateId("CounterAsset"))+ .setRealm(Constants.MASTER_REALM)+ .addAttributes(+ new Attribute<>("count", ValueType.INTEGER, null).addMeta(new MetaItem<>(MetaItemType.RULE_STATE)),+ )+ counterAsset = assetStorageService.merge(counterAsset)++ expect: "the rule engines to become available and be running"+ conditions.eventually {+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)+ assert engine != null+ assert engine.isRunning()+ }++ when: "a mock rules engine is created"+ // We want to stop rules evaluation while doing this setup, otherwise our count of attribute event change is biased+ rulesService.stop(container)+ RulesEngine mockRulesEngine = Spy(engine)+ AtomicBoolean rulesExecuting = new AtomicBoolean(false)+ AtomicBoolean parallelExecutionOccurred = new AtomicBoolean(false)+ mockRulesEngine.fireAllDeployments() >> {+ println("Rules executing")+ if (rulesExecuting.getAndSet(true)) {+ parallelExecutionOccurred.set(true)+ }+ callRealMethod()+ rulesExecuting.set(false)+ }+ rulesService.realmEngines.put(Constants.MASTER_REALM, mockRulesEngine)+ rulesService.start(container)++ and: "An attribute has changed value"+ def receivedEvents = new ArrayList<AttributeEvent>()+ assetProcessingService.addEventInterceptor { EntityManager em, AttributeEvent event ->+ receivedEvents.add(event)+ false+ }+++ and: "the engine is restarted while rules are evaluating"+ conditions.eventually {+ assert(rulesExecuting.get())+ }+ mockRulesEngine.stop()+ // With an immediate stop/start, the rules taking 600 ms to executing and the firing period being 500ms+ // the newly created timer should fire while the initial rules are still evaluating+ mockRulesEngine.start()+then: "A new attribute event is sent"conditions.eventually {assert receivedEvents.size() == 4diff --git a/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy b/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovyindex 7023554c4..e3e61b37d 100644--- a/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy+++ b/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy@@ -24,5 +24,10 @@ rules.add()def updates = facts.bound("updates") as List<AttributeEvent>updates.forEach { event ->assets.dispatch(event) }- Thread.sleep(600)++ // This used to be a Thread.sleep() but one test requires the wait to be non interruptible+ long startTime = System.currentTimeMillis()+ while ((System.currentTimeMillis() - startTime) < 600) {+ // Busy wait for 600ms - cannot be interrupted+ }})