Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Bug fix for parallel rules execution (#2051)

Eric Bariaux2b2889c8e865
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
index ad4c832f9..67294907f 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
@@ -47,6 +47,7 @@ import org.openremote.model.syslog.SyslogCategory;
import java.util.*;
import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -116,6 +117,7 @@ public class RulesEngine<T extends Ruleset> {
final protected AbstractRulesEngine engine;
protected boolean running;
+ protected AtomicBoolean executing = new AtomicBoolean(false);
protected boolean previouslyFired;
protected long lastFireTimestamp;
protected boolean trackLocationPredicates;
@@ -427,8 +429,18 @@ public class RulesEngine<T extends Ruleset> {
if (!running) {
return;
}
- // Process rules for all deployments
- fireAllDeployments();
+
+ // If we're not already executing rules, process rules for all deployments
+ if (executing.compareAndSet(false, true)) {
+ fireAllDeployments();
+ executing.set(false);
+ }
+
+ if (Thread.currentThread().isInterrupted()) {
+ LOG.finest("Timer interrupted during rules execution - not scheduling next fire");
+ return;
+ }
+
fireTimer = null;
scheduleFire(false);
},
diff --git a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
index 5c82855f6..921e811a3 100644
--- a/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/rules/RulesEngineFiringTest.groovy
@@ -89,6 +89,179 @@ class RulesEngineFiringTest extends Specification implements ManagerContainerTra
false
}
+ then: "A new attribute event is sent"
+ conditions.eventually {
+ // The exact value does not matter, as the rule when is always true if the attribute exists
+ // it just needs to be long enough for a potential co-execution to happen
+ // and short enough to be reached before the timeout
+ assert receivedEvents.size() == 4
+ }
+ assert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"
+ }
+
+ /*
+ The exact timing issue is:
+ - fireTimer triggers and is executing the rules (fireAllDeployments)
+ - stop is called, cancelling the fireTimer and setting it to null
+ - start is called, creating a new fireTimer
+ - in this case, that new timer does not fire yet and fireAllDeployments returns,
+ the fire lambda sets fireTimer to null and creates a new timer
+ As a result, two timers are active at the same time
+ */
+ @SuppressWarnings("GroovyAccessibility")
+ def "Check rules engine are not running in parallel on engine restart with new firing after rule execution"() {
+ given: "the container is started"
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+ def container = startContainer(defaultConfig(), defaultServices())
+ def rulesService = container.getService(RulesService.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
+ def assetProcessingService = container.getService(AssetProcessingService.class)
+ def rulesetStorageService = container.getService(RulesetStorageService.class)
+ RulesEngine engine = null
+
+ and: "some rulesets with a long running rule"
+ Ruleset ruleset = new RealmRuleset(
+ Constants.MASTER_REALM,
+ "Parallel Engine check rules",
+ GROOVY,
+ getClass().getResource("/org/openremote/test/rules/ParallelEngineCheck.groovy").text)
+ Long masterRulesetId = rulesetStorageService.merge(ruleset).id
+
+ and: "assets are added"
+ def counterAsset = new ThingAsset("CounterAsset")
+ .setId(UniqueIdentifierGenerator.generateId("CounterAsset"))
+ .setRealm(Constants.MASTER_REALM)
+ .addAttributes(
+ new Attribute<>("count", ValueType.INTEGER, null).addMeta(new MetaItem<>(MetaItemType.RULE_STATE)),
+ )
+ counterAsset = assetStorageService.merge(counterAsset)
+
+ expect: "the rule engines to become available and be running"
+ conditions.eventually {
+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)
+ assert engine != null
+ assert engine.isRunning()
+ }
+
+ when: "a mock rules engine is created"
+ // We want to stop rules evaluation while doing this setup, otherwise our count of attribute event change is biased
+ rulesService.stop(container)
+ RulesEngine mockRulesEngine = Spy(engine)
+ AtomicBoolean rulesExecuting = new AtomicBoolean(false)
+ AtomicBoolean parallelExecutionOccurred = new AtomicBoolean(false)
+ mockRulesEngine.fireAllDeployments() >> {
+ println("Rules executing")
+ if (rulesExecuting.getAndSet(true)) {
+ parallelExecutionOccurred.set(true)
+ }
+ callRealMethod()
+ rulesExecuting.set(false)
+ }
+ rulesService.realmEngines.put(Constants.MASTER_REALM, mockRulesEngine)
+ rulesService.start(container)
+
+ and: "An attribute has changed value"
+ def receivedEvents = new ArrayList<AttributeEvent>()
+ assetProcessingService.addEventInterceptor { EntityManager em, AttributeEvent event ->
+ receivedEvents.add(event)
+ false
+ }
+
+
+ and: "the engine is restarted while rules are evaluating"
+ conditions.eventually {
+ assert(rulesExecuting.get())
+ }
+ mockRulesEngine.stop()
+ // We don't want to start immediately, we want the start code to execute while the rules are evaluating
+ // but the firing of the timer created by this call to happen after the rules have evaluated
+ Thread.sleep(300)
+ mockRulesEngine.start()
+
+ then: "A new attribute event is sent"
+ conditions.eventually {
+ assert receivedEvents.size() == 4
+ }
+ assert !parallelExecutionOccurred.get(): "Rules engine triggered while rules are still executing"
+ }
+
+ /*
+ The exact timing issue is:
+ - fireTimer triggers and is executing the rules (fireAllDeployments)
+ - stop is called, cancelling the fireTimer and setting it to null
+ - start is called, creating a new fireTimer
+ - in this case, that new timer already fires while fireAllDeployments is still running
+ As a result, the newly created timer triggers while the original rules execution is still happening
+ */
+ @SuppressWarnings("GroovyAccessibility")
+ def "Check rules engine are not running in parallel on engine restart with new firing during current rule execution"() {
+ given: "the container is started"
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+ def container = startContainer(defaultConfig(), defaultServices())
+ def rulesService = container.getService(RulesService.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
+ def assetProcessingService = container.getService(AssetProcessingService.class)
+ def rulesetStorageService = container.getService(RulesetStorageService.class)
+ RulesEngine engine = null
+
+ and: "some rulesets with a long running rule"
+ Ruleset ruleset = new RealmRuleset(
+ Constants.MASTER_REALM,
+ "Parallel Engine check rules",
+ GROOVY,
+ getClass().getResource("/org/openremote/test/rules/ParallelEngineCheck.groovy").text)
+ Long masterRulesetId = rulesetStorageService.merge(ruleset).id
+
+ and: "assets are added"
+ def counterAsset = new ThingAsset("CounterAsset")
+ .setId(UniqueIdentifierGenerator.generateId("CounterAsset"))
+ .setRealm(Constants.MASTER_REALM)
+ .addAttributes(
+ new Attribute<>("count", ValueType.INTEGER, null).addMeta(new MetaItem<>(MetaItemType.RULE_STATE)),
+ )
+ counterAsset = assetStorageService.merge(counterAsset)
+
+ expect: "the rule engines to become available and be running"
+ conditions.eventually {
+ engine = rulesService.realmEngines.get(Constants.MASTER_REALM)
+ assert engine != null
+ assert engine.isRunning()
+ }
+
+ when: "a mock rules engine is created"
+ // We want to stop rules evaluation while doing this setup, otherwise our count of attribute event change is biased
+ rulesService.stop(container)
+ RulesEngine mockRulesEngine = Spy(engine)
+ AtomicBoolean rulesExecuting = new AtomicBoolean(false)
+ AtomicBoolean parallelExecutionOccurred = new AtomicBoolean(false)
+ mockRulesEngine.fireAllDeployments() >> {
+ println("Rules executing")
+ if (rulesExecuting.getAndSet(true)) {
+ parallelExecutionOccurred.set(true)
+ }
+ callRealMethod()
+ rulesExecuting.set(false)
+ }
+ rulesService.realmEngines.put(Constants.MASTER_REALM, mockRulesEngine)
+ rulesService.start(container)
+
+ and: "An attribute has changed value"
+ def receivedEvents = new ArrayList<AttributeEvent>()
+ assetProcessingService.addEventInterceptor { EntityManager em, AttributeEvent event ->
+ receivedEvents.add(event)
+ false
+ }
+
+
+ and: "the engine is restarted while rules are evaluating"
+ conditions.eventually {
+ assert(rulesExecuting.get())
+ }
+ mockRulesEngine.stop()
+ // With an immediate stop/start, the rules taking 600 ms to executing and the firing period being 500ms
+ // the newly created timer should fire while the initial rules are still evaluating
+ mockRulesEngine.start()
+
then: "A new attribute event is sent"
conditions.eventually {
assert receivedEvents.size() == 4
diff --git a/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy b/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy
index 7023554c4..e3e61b37d 100644
--- a/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy
+++ b/test/src/test/resources/org/openremote/test/rules/ParallelEngineCheck.groovy
@@ -24,5 +24,10 @@ rules.add()
def updates = facts.bound("updates") as List<AttributeEvent>
updates.forEach { event ->
assets.dispatch(event) }
- Thread.sleep(600)
+
+ // This used to be a Thread.sleep() but one test requires the wait to be non interruptible
+ long startTime = System.currentTimeMillis()
+ while ((System.currentTimeMillis() - startTime) < 600) {
+ // Busy wait for 600ms - cannot be interrupted
+ }
})