Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Stabilize flaky predicted datapoint deletion tests (#3064)

Koen778cf6c71970
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
index a4afa1d90..52345fbfb 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/ApplyPredictedDataPointsService.java
@@ -96,11 +96,11 @@ public class ApplyPredictedDataPointsService implements ContainerService {
@Override
public void stop(Container container) throws Exception {
- if (scheduledFuture != null) {
- scheduledFuture.cancel(true);
- scheduledFuture = null;
- }
synchronized (scheduleLock) {
+ if (scheduledFuture != null) {
+ scheduledFuture.cancel(true);
+ scheduledFuture = null;
+ }
scheduleQueue.clear();
scheduledEntries.clear();
}
@@ -116,9 +116,10 @@ public class ApplyPredictedDataPointsService implements ContainerService {
return;
}
+ // Events are delivered concurrently so their meta can be stale; it is only trusted for attributes that are not
+ // scheduled, otherwise the stored attribute is checked
MetaMap meta = event.getMeta();
- if (meta != null && !hasRequiredMeta(meta)) {
- removeScheduledEntry(event.getRef());
+ if (meta != null && !hasRequiredMeta(meta) && !isScheduled(event.getRef())) {
return;
}
@@ -150,22 +151,18 @@ public class ApplyPredictedDataPointsService implements ContainerService {
* Based on the next predicted data point to be applied, schedule the next execution of our processing.
*/
protected void scheduleNext() {
- long delayMillis = -1;
+ // Cancelling and scheduling must be atomic with reading the queue head, otherwise a concurrent caller can
+ // cancel a live schedule and leave a queued entry without one
synchronized (scheduleLock) {
- if (!scheduleQueue.isEmpty()) {
- long now = timerService.getCurrentTimeMillis();
- long nextTime = scheduleQueue.peek().timestamp;
- delayMillis = Math.max(0, nextTime - now);
+ if (scheduledFuture != null) {
+ scheduledFuture.cancel(false);
+ scheduledFuture = null;
}
- }
-
- if (scheduledFuture != null) {
- scheduledFuture.cancel(false);
- scheduledFuture = null;
- }
- if (delayMillis >= 0) {
- scheduledFuture = scheduledExecutorService.schedule(this::processDueEntries, delayMillis, TimeUnit.MILLISECONDS);
+ if (!scheduleQueue.isEmpty()) {
+ long delayMillis = Math.max(0, scheduleQueue.peek().timestamp - timerService.getCurrentTimeMillis());
+ scheduledFuture = scheduledExecutorService.schedule(this::processDueEntries, delayMillis, TimeUnit.MILLISECONDS);
+ }
}
}
@@ -255,6 +252,12 @@ public class ApplyPredictedDataPointsService implements ContainerService {
return meta != null && meta.has(HAS_PREDICTED_DATA_POINTS) && meta.has(APPLY_PREDICTED_DATA_POINTS);
}
+ protected boolean isScheduled(AttributeRef ref) {
+ synchronized (scheduleLock) {
+ return scheduledEntries.containsKey(ref);
+ }
+ }
+
protected void upsertScheduledEntry(AttributeRef ref, long timestamp, Object value) {
synchronized (scheduleLock) {
ScheduledEntry existing = scheduledEntries.remove(ref);
diff --git a/test/src/test/groovy/org/openremote/test/assets/ApplyPredictedDataPointsServiceTest.groovy b/test/src/test/groovy/org/openremote/test/assets/ApplyPredictedDataPointsServiceTest.groovy
index d63a29c4b..28e1e4fe1 100644
--- a/test/src/test/groovy/org/openremote/test/assets/ApplyPredictedDataPointsServiceTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/ApplyPredictedDataPointsServiceTest.groovy
@@ -6,9 +6,11 @@ import org.openremote.manager.datapoint.ApplyPredictedDataPointsService
import org.openremote.manager.datapoint.AssetPredictedDatapointService
import org.openremote.manager.event.ClientEventService
import org.openremote.manager.setup.SetupService
+import org.openremote.model.attribute.Attribute
import org.openremote.model.attribute.AttributeEvent
import org.openremote.model.attribute.AttributeRef
import org.openremote.model.attribute.MetaItem
+import org.openremote.model.attribute.MetaMap
import org.openremote.model.datapoint.ValueDatapoint
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.ManagerTestSetup
@@ -27,9 +29,23 @@ import static org.openremote.model.value.MetaItemType.HAS_PREDICTED_DATA_POINTS
class ApplyPredictedDataPointsServiceTest extends Specification implements ManagerContainerTrait {
- ScheduledExecutorService executor = Mock(ScheduledExecutorService)
- ScheduledFuture<?> future = Mock(ScheduledFuture)
- Long delay
+ // Delay and future of the last schedule as a single value, so a delay can never be paired with a stale future
+ volatile Map<String, ?> scheduled
+ ApplyPredictedDataPointsService mockedApplyService
+ ScheduledExecutorService originalExecutor
+
+ def cleanup() {
+ if (mockedApplyService != null) {
+ synchronized (mockedApplyService.scheduleLock) {
+ mockedApplyService.scheduledFuture?.cancel(true)
+ mockedApplyService.scheduledFuture = null
+ mockedApplyService.scheduledEntries.clear()
+ mockedApplyService.scheduleQueue.clear()
+ mockedApplyService.scheduledExecutorService = originalExecutor
+ }
+ mockedApplyService = null
+ }
+ }
def "Applies predicted datapoints when timestamps match"() {
given: "a running container and target attribute with required meta"
@@ -47,9 +63,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "the scheduled executor is mocked to allow manual triggering"
setupScheduler(applyService)
- and: "the attribute is tagged for predicted datapoint application"
- def attributeRef = createTestAttribute(assetStorageService, managerTestSetup.thingId, "predictedValue1")
- enablePredictedApplyMeta(assetStorageService, attributeRef.getId(), attributeRef.getName())
+ and: "the attribute is created tagged for predicted datapoint application"
+ def attributeRef = createTestAttributeWithApplyMeta(assetStorageService, managerTestSetup.thingId, "predictedValue1")
when: "predicted datapoints are added in the future"
def predictedDatapoints = [
@@ -63,8 +78,9 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
)
and: "time advances to the first datapoint"
+ awaitScheduled(applyService, attributeRef, conditions)
advancePseudoClock(1, TimeUnit.MINUTES, container)
- future.get()
+ scheduled.future.get()
then: "the first predicted value should be applied"
conditions.eventually {
def asset = assetStorageService.find(attributeRef.getId(), true)
@@ -74,8 +90,9 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
}
when: "time advances to the next datapoint"
+ awaitScheduled(applyService, attributeRef, conditions)
advancePseudoClock(1, TimeUnit.MINUTES, container)
- future.get()
+ scheduled.future.get()
then: "the second predicted value should be applied"
conditions.eventually {
@@ -233,8 +250,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "time advances beyond the datapoints"
advancePseudoClock(3, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "the attribute value should remain unchanged"
@@ -277,8 +294,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "time advances beyond the datapoints"
advancePseudoClock(3, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "the attribute value should remain unchanged"
@@ -321,8 +338,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "time advances beyond the datapoints"
advancePseudoClock(3, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "the attribute value should remain unchanged"
@@ -340,6 +357,7 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
def assetStorageService = container.getService(AssetStorageService.class)
def assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)
+ def clientEventService = container.getService(ClientEventService.class)
def applyService = container.getService(ApplyPredictedDataPointsService.class)
and: "the clock is stopped for deterministic scheduling"
@@ -349,9 +367,19 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "the scheduled executor is mocked to allow manual triggering"
setupScheduler(applyService)
+ and: "we capture attribute events"
+ List<AttributeEvent> events = new CopyOnWriteArrayList<>()
+ Consumer<AttributeEvent> consumer = { event -> events.add(event) }
+ clientEventService.addSubscription(AttributeEvent.class, null, consumer)
+
and: "a plain attribute exists without meta"
def attributeRef = createTestAttribute(assetStorageService, managerTestSetup.thingId, "predictedValue7")
+ and: "the creation event without meta has been delivered, so it cannot arrive after the meta is added"
+ conditions.eventually {
+ assert events.any { it.ref == attributeRef && !it.getMeta()?.has(APPLY_PREDICTED_DATA_POINTS) }
+ }
+
and: "predicted datapoints are added in the future"
assetPredictedDatapointService.updateValues(
attributeRef.getId(),
@@ -364,18 +392,18 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
when: "time advances to the first datapoint"
advancePseudoClock(1, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
and: "required meta is added"
enablePredictedApplyMeta(assetStorageService, attributeRef.getId(), attributeRef.getName())
and: "time advances to the second datapoint"
+ awaitScheduled(applyService, attributeRef, conditions)
advancePseudoClock(1, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
- }
+ scheduled.future.get()
+
then: "the second predicted value should be applied"
conditions.eventually {
def asset = assetStorageService.find(attributeRef.getId(), true)
@@ -385,7 +413,7 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
}
}
- def "Does not apply values after removing required meta from attribute"() {
+ def "Keeps the schedule when an attribute event carries stale meta"() {
given: "a running container and target attribute with required meta"
def container = startContainer(defaultConfig(), defaultServices())
def conditions = new PollingConditions(timeout: 10, delay: 0.2)
@@ -401,9 +429,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "the scheduled executor is mocked to allow manual triggering"
setupScheduler(applyService)
- and: "a plain attribute exists with required meta"
- def attributeRef = createTestAttribute(assetStorageService, managerTestSetup.thingId, "predictedValue8")
- enablePredictedApplyMeta(assetStorageService, attributeRef.getId(), attributeRef.getName())
+ and: "an attribute with the required meta is created in a single update"
+ def attributeRef = createTestAttributeWithApplyMeta(assetStorageService, managerTestSetup.thingId, "predictedValue11")
and: "predicted datapoints are added in the future"
assetPredictedDatapointService.updateValues(
@@ -415,11 +442,63 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
]
)
+ and: "the first datapoint is scheduled"
+ awaitScheduled(applyService, attributeRef, conditions)
+
+ when: "an event without the required meta is delivered, as can happen when events arrive out of order"
+ applyService.onAttributeEvent(
+ new AttributeEvent(attributeRef, 0d, now).setMeta(new MetaMap())
+ )
+
+ then: "the schedule should be kept because the stored attribute still has the required meta"
+ assert isScheduled(applyService, attributeRef)
+
when: "time advances to the first datapoint"
advancePseudoClock(1, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ scheduled.future.get()
+
+ then: "the first predicted value should still be applied"
+ conditions.eventually {
+ def asset = assetStorageService.find(attributeRef.getId(), true)
+ def attribute = asset.getAttribute(attributeRef.getName()).get()
+ assert attribute.getValue(Double.class).orElse(null) == 10d
+ assert attribute.getTimestamp().orElse(0L) == now + TimeUnit.MINUTES.toMillis(1)
}
+ }
+
+ def "Does not apply values after removing required meta from attribute"() {
+ given: "a running container and target attribute with required meta"
+ def container = startContainer(defaultConfig(), defaultServices())
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+ def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
+ def assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)
+ def applyService = container.getService(ApplyPredictedDataPointsService.class)
+
+ and: "the clock is stopped for deterministic scheduling"
+ stopPseudoClock()
+ def now = getClockTimeOf(container)
+
+ and: "the scheduled executor is mocked to allow manual triggering"
+ setupScheduler(applyService)
+
+ and: "an attribute with the required meta is created in a single update"
+ def attributeRef = createTestAttributeWithApplyMeta(assetStorageService, managerTestSetup.thingId, "predictedValue8")
+
+ and: "predicted datapoints are added in the future"
+ assetPredictedDatapointService.updateValues(
+ attributeRef.getId(),
+ attributeRef.getName(),
+ [
+ new ValueDatapoint<>(now + TimeUnit.MINUTES.toMillis(1), 10d),
+ new ValueDatapoint<>(now + TimeUnit.MINUTES.toMillis(2), 20d)
+ ]
+ )
+
+ when: "time advances to the first datapoint"
+ awaitScheduled(applyService, attributeRef, conditions)
+ advancePseudoClock(1, TimeUnit.MINUTES, container)
+ scheduled.future.get()
then: "the first predicted value should be applied"
conditions.eventually {
@@ -435,8 +514,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "time advances beyond the datapoints"
advancePseudoClock(2, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "the attribute value should remain unchanged"
@@ -464,15 +543,20 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "the scheduled executor is mocked to allow manual triggering"
setupScheduler(applyService)
- and: "a plain attribute exists with required meta"
- def attributeRef = createTestAttribute(assetStorageService, managerTestSetup.thingId, "predictedValue9")
- enablePredictedApplyMeta(assetStorageService, attributeRef.getId(), attributeRef.getName())
-
and: "we capture attribute events"
List<AttributeEvent> events = new CopyOnWriteArrayList<>()
Consumer<AttributeEvent> consumer = { event -> events.add(event) }
clientEventService.addSubscription(AttributeEvent.class, null, consumer)
+ and: "an attribute with the required meta is created in a single update"
+ def attributeRef = createTestAttributeWithApplyMeta(assetStorageService, managerTestSetup.thingId, "predictedValue9")
+
+ and: "the creation event has been delivered to the service before it is observed here"
+ conditions.eventually {
+ assert events.any { it.ref == attributeRef && it.getMeta()?.has(APPLY_PREDICTED_DATA_POINTS) }
+ }
+ events.clear()
+
and: "predicted datapoints are added in the future"
assetPredictedDatapointService.updateValues(
attributeRef.getId(),
@@ -486,8 +570,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
when: "time advances to the middle of the predicted datapoints"
advancePseudoClock(1, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "an attribute event should have been emitted for the applied value"
@@ -508,14 +592,17 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
when: "we clear events and advance time past all datapoints"
events.clear()
advancePseudoClock(3, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
- then: "no further attribute events should be emitted"
+ then: "the scheduled prediction for the attribute is cancelled"
conditions.eventually {
- assert events.isEmpty()
+ assert !applyService.scheduledEntries.containsKey(attributeRef)
}
+
+ and: "no further non-deletion attribute events are emitted for the attribute"
+ assert !events.any { it.ref == attributeRef && !it.deleted }
}
def "Does not emit attribute events after asset deletion"() {
@@ -535,15 +622,20 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
and: "the scheduled executor is mocked to allow manual triggering"
setupScheduler(applyService)
- and: "a plain attribute exists with required meta"
- def attributeRef = createTestAttribute(assetStorageService, managerTestSetup.thingId, "predictedValue10")
- enablePredictedApplyMeta(assetStorageService, attributeRef.getId(), attributeRef.getName())
-
and: "we capture attribute events"
List<AttributeEvent> events = new CopyOnWriteArrayList<>()
Consumer<AttributeEvent> consumer = { event -> events.add(event) }
clientEventService.addSubscription(AttributeEvent.class, null, consumer)
+ and: "an attribute with the required meta is created in a single update"
+ def attributeRef = createTestAttributeWithApplyMeta(assetStorageService, managerTestSetup.thingId, "predictedValue10")
+
+ and: "the creation event has been delivered to the service before it is observed here"
+ conditions.eventually {
+ assert events.any { it.ref == attributeRef && it.getMeta()?.has(APPLY_PREDICTED_DATA_POINTS) }
+ }
+ events.clear()
+
and: "predicted datapoints are added in the future"
assetPredictedDatapointService.updateValues(
attributeRef.getId(),
@@ -557,8 +649,8 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
when: "time advances to the middle of the predicted datapoints"
advancePseudoClock(1, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
then: "an attribute event should have been emitted for the applied value"
@@ -577,14 +669,17 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
when: "we clear events and advance time past all datapoints"
events.clear()
advancePseudoClock(3, TimeUnit.MINUTES, container)
- if (future != null) {
- future.get()
+ if (scheduled != null) {
+ scheduled.future.get()
}
- then: "no further attribute events should be emitted"
+ then: "the scheduled prediction for the attribute is cancelled"
conditions.eventually {
- assert events.isEmpty()
+ assert !applyService.scheduledEntries.containsKey(attributeRef)
}
+
+ and: "no further non-deletion attribute events are emitted for the attribute"
+ assert !events.any { it.ref == attributeRef && !it.deleted }
}
private static void enablePredictedApplyMeta(AssetStorageService assetStorageService, String assetId, String attributeName) {
@@ -606,6 +701,18 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
return new AttributeRef(assetId, attributeName)
}
+ private static AttributeRef createTestAttributeWithApplyMeta(AssetStorageService assetStorageService, String assetId, String attributeName) {
+ def asset = assetStorageService.find(assetId)
+ def attribute = new Attribute<>(attributeName, ValueType.NUMBER, 0d)
+ attribute.addOrReplaceMeta(
+ new MetaItem<>(HAS_PREDICTED_DATA_POINTS, true),
+ new MetaItem<>(APPLY_PREDICTED_DATA_POINTS, true)
+ )
+ asset.getAttributes().addOrReplace(attribute)
+ assetStorageService.merge(asset)
+ return new AttributeRef(assetId, attributeName)
+ }
+
private static void addMeta(AssetStorageService assetStorageService, AttributeRef attributeRef, MetaItem<?> metaItem) {
def asset = assetStorageService.find(attributeRef.getId())
def attribute = asset.getAttribute(attributeRef.getName()).get()
@@ -621,22 +728,43 @@ class ApplyPredictedDataPointsServiceTest extends Specification implements Manag
}
private void setupScheduler(ApplyPredictedDataPointsService applyService) {
- executor = Mock(ScheduledExecutorService)
- future = Mock(ScheduledFuture)
- delay = null
+ mockedApplyService = applyService
+ originalExecutor = applyService.scheduledExecutorService
+ def executor = Mock(ScheduledExecutorService)
+ scheduled = null
executor.schedule(_ as Runnable, _ as Long, _ as TimeUnit) >> { args ->
- delay = args[1] as Long
- future = Mock(ScheduledFuture)
- future.get() >> {
+ def scheduledFuture = Mock(ScheduledFuture)
+ scheduledFuture.get() >> {
(args[0] as Runnable).run()
return true
}
- return future
+ // Published only once the future is stubbed, so awaiting a schedule also awaits its future
+ scheduled = [delay: args[1] as Long, future: scheduledFuture]
+ return scheduledFuture
}
- applyService.scheduledFuture?.cancel(true)
- applyService.scheduledFuture = null
- applyService.scheduledExecutorService = executor
+ synchronized (applyService.scheduleLock) {
+ applyService.scheduledFuture?.cancel(true)
+ applyService.scheduledFuture = null
+ applyService.scheduledExecutorService = executor
+ }
+ }
+
+ /**
+ * The service schedules on a container thread in response to events, so the schedule must be awaited before the
+ * clock is advanced; without this the test can resolve a future from an earlier schedule, or none at all.
+ */
+ private void awaitScheduled(ApplyPredictedDataPointsService applyService, AttributeRef attributeRef, PollingConditions conditions) {
+ conditions.eventually {
+ assert isScheduled(applyService, attributeRef)
+ assert scheduled != null
+ }
+ }
+
+ private static boolean isScheduled(ApplyPredictedDataPointsService applyService, AttributeRef attributeRef) {
+ synchronized (applyService.scheduleLock) {
+ return applyService.scheduledEntries.containsKey(attributeRef)
+ }
}
}