Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Forecast test fix (#1037)

Rich Turner7cbc95c32bbe
diff --git a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java b/manager/src/main/java/org/openremote/manager/asset/ForecastService.java
index 1b8ff7d42..3f14aae3b 100644
--- a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java
+++ b/manager/src/main/java/org/openremote/manager/asset/ForecastService.java
@@ -420,7 +420,7 @@ public class ForecastService extends RouteBuilder implements ContainerService {
} else {
return Optional.<Number>empty();
}
- }).collect(Collectors.toList());
+ }).toList();
if (forecastTimestamps.size() >= forecastValues.size()) {
List<Pair<?, LocalDateTime>> datapoints = IntStream
diff --git a/model/src/main/java/org/openremote/model/attribute/Attribute.java b/model/src/main/java/org/openremote/model/attribute/Attribute.java
index b9ae434e2..9d2132c08 100644
--- a/model/src/main/java/org/openremote/model/attribute/Attribute.java
+++ b/model/src/main/java/org/openremote/model/attribute/Attribute.java
@@ -319,9 +319,11 @@ public class Attribute<T> extends AbstractNameValueHolder<T> implements MetaHold
@Override
public String toString() {
+ String valStr = value != null ? value.toString() : null;
+ valStr = valStr != null && valStr.length() > 100 ? valStr.substring(0, 100) : valStr;
return getClass().getSimpleName() + "{" +
"name='" + name + '\'' +
- ", value='" + value + '\'' +
+ ", value='" + valStr + '\'' +
", timestamp='" + getTimestamp().orElse(0L) + '\'' +
"} ";
}
diff --git a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
index b007fc042..02580aa19 100644
--- a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
+++ b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import org.openremote.agent.protocol.simulator.SimulatorAgent;
import org.openremote.agent.protocol.simulator.SimulatorAgentLink;
import org.openremote.container.util.UniqueIdentifierGenerator;
-import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.setup.ManagerSetup;
import org.openremote.model.Constants;
import org.openremote.model.Container;
@@ -36,22 +35,13 @@ import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.attribute.MetaItem;
import org.openremote.model.geo.GeoJSONPoint;
import org.openremote.model.security.Realm;
-import org.openremote.model.util.Pair;
import org.openremote.model.util.ValueUtil;
-import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage;
import org.openremote.model.value.ValueConstraint;
import org.openremote.model.value.ValueType;
import org.openremote.model.value.impl.ColourRGB;
-import java.time.Duration;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.stream.Collectors;
import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT;
import static org.openremote.model.Constants.*;
@@ -111,16 +101,7 @@ public class ManagerTestSetup extends ManagerSetup {
public String electricityBatteryAssetId;
public String light1Id;
public String light2Id;
- public String forecastThing1Id;
- public String forecastThing2Id;
- public String forecastAttributeName = "forecastAttribute";
- public ForecastConfigurationWeightedExponentialAverage forecastConfig = new ForecastConfigurationWeightedExponentialAverage(
- new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("P7D"),
- 3,
- new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("PT1H"),
- 4
- );
- public List<List<Double>> forecastTestHistoryData = new ArrayList<>();
+
public ManagerTestSetup(Container container) {
super(container);
@@ -307,27 +288,6 @@ public class ManagerTestSetup extends ManagerSetup {
electricitySupplierAsset = assetStorageService.merge(electricitySupplierAsset);
electricitySupplierAssetId = electricitySupplierAsset.getId();
-
- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.0, 3.0, 2.0}));
- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.1, 3.1, 2.1}));
- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.2, 3.2, 2.2}));
- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.3, 3.3, 2.3}));
- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.4, 3.4, 2.4}));
-
- forecastThing1Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);
- forecastThing2Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);
-
- Long offset = ((forecastConfig.getForecastPeriod().toMillis() * 2) / 3) * (-1);
- insertHistoryTestDataToDb(new AttributeRef(forecastThing1Id, forecastAttributeName), forecastTestHistoryData, 0, forecastConfig);
- insertHistoryTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestHistoryData, offset ,forecastConfig);
-
- List<Pair<Double, Long>> forecastTestData = new ArrayList<>();
- forecastTestData.add(new Pair<>(1.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() + offset));
- forecastTestData.add(new Pair<>(2.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() * 2 + offset));
-
- insertForecastTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestData);
-
-
// ################################ Assets for 'building' realm ###################################
BuildingAsset smartBuilding = new BuildingAsset("Smart building");
@@ -731,59 +691,4 @@ public class ManagerTestSetup extends ManagerSetup {
LightAsset lightController_3Asset = createDemoLightControllerAsset("LightController 3", assetArea3, new GeoJSONPoint(5.487478, 51.446979));
lightController_3Asset = assetStorageService.merge(lightController_3Asset);
}
-
- public void insertHistoryTestDataToDb(AttributeRef attributeRef, List<List<Double>> data, long offset, ForecastConfigurationWeightedExponentialAverage config) {
- for (int pastPeriodIndex = config.getPastCount(); pastPeriodIndex > 0; pastPeriodIndex--) {
- List<Double> values = new ArrayList<>(data.size());
- for (int forecastPeriodIndex = 0; forecastPeriodIndex < data.size(); forecastPeriodIndex++) {
- values.add(data.get(forecastPeriodIndex).get(config.getPastCount() - pastPeriodIndex));
- }
- insertHistoryTestDataToDb(attributeRef, pastPeriodIndex, values, offset, config);
- }
- }
-
- public void insertHistoryTestDataToDb(AttributeRef attributeRef, int pastPeriodIndex, List<Double> values, long offset, ForecastConfigurationWeightedExponentialAverage config) {
- LocalDateTime now = LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis()), ZoneId.systemDefault());
- long nowInMillis = now.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
- List<Pair<?, LocalDateTime>> history = new ArrayList<>();
- for (int forecastPeriodIndex = 0; forecastPeriodIndex < values.size(); forecastPeriodIndex++) {
- long timespan = config.getPastPeriod().toMillis() * pastPeriodIndex -
- config.getForecastPeriod().toMillis() * (forecastPeriodIndex + 1) +
- Duration.ofMinutes(2).toMillis() - offset;
- history.add(
- new Pair<>(
- Double.valueOf(values.get(forecastPeriodIndex)),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() - timespan), ZoneId.systemDefault())
- )
- );
- }
- assetDatapointService.upsertValues(attributeRef.getId(), attributeRef.getName(), history);
- }
-
- @SuppressWarnings("unchecked")
- public void insertForecastTestDataToDb(AttributeRef attributeRef, List<Pair<Double, Long>> data) {
- List<Pair<Double, LocalDateTime>> valuesAndTimestamps = data
- .stream()
- .map(p -> new Pair<Double,LocalDateTime>(
- p.getKey(),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(p.getValue()), ZoneId.systemDefault())))
- .collect(Collectors.toList());
- assetPredictedDatapointService.updateValues(attributeRef.getId(), attributeRef.getName(), (List)valuesAndTimestamps);
- }
-
- public String addForecastTestAsset(AssetStorageService assetStorageService, String realm, String attributeName, ForecastConfigurationWeightedExponentialAverage config) {
- Asset<?> thing = new ThingAsset("Forecast Test Thing");
- thing.setRealm(realm);
- thing.getAttributes().addOrReplace(
- new Attribute<>(attributeName, NUMBER)
- .addOrReplaceMeta(
- new MetaItem<>(
- FORECAST,
- config
- )
- )
- );
- thing = assetStorageService.merge(thing);
- return thing.getId();
- }
}
diff --git a/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy
index 7545e9442..1e67be4cd 100644
--- a/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy
@@ -6,10 +6,13 @@ import org.openremote.manager.asset.ForecastService
import org.openremote.manager.datapoint.AssetDatapointService
import org.openremote.manager.datapoint.AssetPredictedDatapointService
import org.openremote.manager.setup.SetupService
+import org.openremote.model.asset.Asset
+import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
import org.openremote.model.attribute.AttributeRef
import org.openremote.model.attribute.MetaItem
import org.openremote.model.datapoint.AssetPredictedDatapoint
+import org.openremote.model.util.Pair
import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage
import org.openremote.setup.integration.ManagerTestSetup
import org.openremote.test.ManagerContainerTrait
@@ -19,6 +22,11 @@ import spock.lang.Specification
import spock.util.concurrent.PollingConditions
import java.time.Duration
+import java.time.Instant
+import java.time.LocalDateTime
+import java.time.ZoneId
+import java.time.temporal.ChronoUnit
+import java.util.stream.IntStream
import static java.util.concurrent.TimeUnit.MILLISECONDS
import static org.openremote.model.value.MetaItemType.FORECAST
@@ -39,17 +47,40 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
static ForecastService forecastService
@Shared
static TimerService timerService
-
- def setupSpec() {
- given: "the server container is started"
- def container = startContainer(defaultConfig(), defaultServices())
- managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
- assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)
- assetStorageService = container.getService(AssetStorageService.class)
- assetDatapointService = container.getService(AssetDatapointService.class)
- forecastService = container.getService(ForecastService.class)
- timerService = container.getService(TimerService.class)
- }
+ @Shared
+ static ForecastConfigurationWeightedExponentialAverage forecastConfig = new ForecastConfigurationWeightedExponentialAverage(
+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("P7D"),
+ 3,
+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("PT1H"),
+ 4
+ )
+ @Shared
+ // Outer list is hours
+ // Inner list is weekly values
+ static double[][] forecastHistoricalData = [
+ [4.0, 3.0, 2.0],
+ [4.1, 3.1, 2.1],
+ [4.2, 3.2, 2.2],
+ [4.3, 3.3, 2.3],
+ [4.4, 3.4, 2.4]
+ ]
+ @Shared
+ static ThingAsset forecastThing2
+ @Shared
+ static String forecastAttributeName = "forecastAttribute"
+
+ // TODO: RT - Not too sure the intention with the offset and seems to be used only in ignored tests so skip for now
+// def setupSpec() {
+// forecastThing2 = addForecastAsset(assetStorageService, managerTestSetup.realmEnergyName, forecastAttributeName, forecastConfig)
+// Long offset = ((forecastConfig.getForecastPeriod().toMillis() * 2) / 3) * (-1)
+// insertHistoryTestDataToDb(new AttributeRef(forecastThing2.id, forecastAttributeName), forecastHistoricalData, offset, forecastConfig)
+//
+// // Insert forecast test data
+// assetPredictedDatapointService.updateValues(forecastThing2.id, forecastAttributeName, [
+// new Pair<>(1.0d, LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() + offset), ZoneId.systemDefault())),
+// new Pair<>(2.0d, LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() * 2 + offset), ZoneId.systemDefault()))
+// ])
+// }
@Ignore
def "Test forecast calculation after server restart - no initial forecast data"() {
@@ -121,37 +152,44 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
}
def "Test adding updating and deleting an asset"() {
- given: "no assets with forecast configuration"
+ given: "the server container is started"
+ def container = startContainer(defaultConfig(), defaultServices())
def conditions = new PollingConditions(timeout: 10, delay: 0.2)
- def config = managerTestSetup.forecastConfig
- def attributeName = "forecastTestAttribute"
+ managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
+ assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)
+ assetStorageService = container.getService(AssetStorageService.class)
+ assetDatapointService = container.getService(AssetDatapointService.class)
+ forecastService = container.getService(ForecastService.class)
+ timerService = container.getService(TimerService.class)
- and:
+ when: "the system time is stopped and set exactly on the next hour"
stopPseudoClock()
+ def now = Instant.ofEpochMilli(timerService.getCurrentTimeMillis())
+ now = now.truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS)
+ advancePseudoClock(now.toEpochMilli()-timerService.getCurrentTimeMillis(), MILLISECONDS, container)
- when: "asset is added"
- String assetId = managerTestSetup.addForecastTestAsset(
- assetStorageService, managerTestSetup.realmEnergyName, attributeName, config
- )
- AttributeRef attributeRef = new AttributeRef(assetId, attributeName)
+ and: "forecast asset is added"
+ def forecastThing1 = addForecastAsset(assetStorageService, managerTestSetup.realmEnergyName, forecastAttributeName, forecastConfig)
+ def attributeRef = new AttributeRef(forecastThing1.id, forecastAttributeName)
then: "attribute should be registered at the forecast service"
conditions.eventually {
assert forecastService.forecastTaskManager.containsAttribute(attributeRef)
}
- when: "test data is inserted to the database"
+ when: "the forecast service is stopped"
forecastService.forecastTaskManager.stop(3000)
- managerTestSetup.insertHistoryTestDataToDb(
- attributeRef, managerTestSetup.forecastTestHistoryData, config.forecastPeriod.toMillis(), config
- )
+
+ and: "historical data is added for the forecast attribute"
+ // Offset historical data by two hours to match up with test intervals
+ insertHistoryTestDataToDb(new AttributeRef(forecastThing1.id, forecastAttributeName), forecastHistoricalData, 2*3600000, forecastConfig)
and: "time has passed"
advancePseudoClock(
- config.forecastPeriod.toMillis(), MILLISECONDS, container
+ forecastConfig.forecastPeriod.toMillis(), MILLISECONDS, container
)
- and: "forecast service is restarted"
+ and: "the forecast service is started again"
forecastService.forecastTaskManager.start(timerService.currentTimeMillis)
then: "initial forecast values should be calculated"
@@ -159,45 +197,26 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService
.getDatapoints(attributeRef)
.sort {it.timestamp}
- assert predictedDatapoints.size() == config.forecastCount
+ assert predictedDatapoints.size() == forecastConfig.forecastCount
for (int i = 0; i < predictedDatapoints.size(); i++) {
- assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()
- assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + (i + 1) * config.forecastPeriod.toMillis()
- }
- }
-
- when: "time has passed"
- forecastService.forecastTaskManager.stop(3000)
- advancePseudoClock(
- config.forecastPeriod.toMillis(), MILLISECONDS, container
- )
- forecastService.forecastTaskManager.start(timerService.currentTimeMillis)
-
- then: "forecast values should be updated"
- conditions.eventually {
- List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService
- .getDatapoints(attributeRef)
- .sort {it.timestamp}
- assert predictedDatapoints.size() == config.forecastCount
- for (int i = 1; i < predictedDatapoints.size(); i++) {
- assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()
- assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + i * config.forecastPeriod.toMillis()
+ assert predictedDatapoints[i].value == calculateForecast(forecastHistoricalData[i][0..forecastConfig.pastCount - 1]).get()
+ assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + (i + 1) * forecastConfig.forecastPeriod.toMillis()
}
}
when: "new forecast attribute is added"
- def asset = assetStorageService.find(assetId)
- asset.getAttributes().addOrReplace(
- new Attribute<>("newForecastAttribute", NUMBER)
- .addOrReplaceMeta(
- new MetaItem<>(
- FORECAST,
- config
- )
- )
+ forecastThing1 = assetStorageService.find(forecastThing1.id)
+ forecastThing1.getAttributes().addOrReplace(
+ new Attribute<>("newForecastAttribute", NUMBER)
+ .addOrReplaceMeta(
+ new MetaItem<>(
+ FORECAST,
+ forecastConfig
+ )
+ )
)
- assetStorageService.merge(asset)
- def newAttributeRef = new AttributeRef(assetId, "newForecastAttribute")
+ forecastThing1 = assetStorageService.merge(forecastThing1)
+ def newAttributeRef = new AttributeRef(forecastThing1.id, "newForecastAttribute")
then: "new attribute should be registered at the forecast service"
conditions.eventually {
@@ -205,35 +224,35 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
}
when: "forecast configuration is updated"
- asset = assetStorageService.find(assetId)
- def updatedPastCount = config.pastCount + 1
+ forecastThing1 = assetStorageService.find(forecastThing1.id)
+ def updatedPastCount = forecastConfig.pastCount + 1
def updatedConfig = new ForecastConfigurationWeightedExponentialAverage(
- config.pastPeriod, updatedPastCount, config.forecastPeriod, config.forecastCount
+ forecastConfig.pastPeriod, updatedPastCount, forecastConfig.forecastPeriod, forecastConfig.forecastCount
)
- asset.getAttributes().addOrReplace(
- new Attribute<>("newForecastAttribute", NUMBER)
- .addOrReplaceMeta(
- new MetaItem<>(
- FORECAST,
- updatedConfig
- )
- )
+ forecastThing1.getAttributes().addOrReplace(
+ new Attribute<>("newForecastAttribute", NUMBER)
+ .addOrReplaceMeta(
+ new MetaItem<>(
+ FORECAST,
+ updatedConfig
+ )
+ )
)
- assetStorageService.merge(asset)
+ forecastThing1 = assetStorageService.merge(forecastThing1)
then: "forecast service should contain updated forecast configuration"
conditions.eventually {
ForecastService.ForecastAttribute forecastAttribute = forecastService.forecastTaskManager.getAttribute(newAttributeRef)
assert forecastAttribute != null
- assert updatedPastCount == forecastAttribute.config.pastCount
+ assert updatedPastCount == (forecastAttribute.config as ForecastConfigurationWeightedExponentialAverage).pastCount
}
when: "forecast configuration is deleted"
- asset = assetStorageService.find(assetId)
- asset.getAttributes().addOrReplace(
- new Attribute<>("newForecastAttribute", NUMBER)
+ forecastThing1 = assetStorageService.find(forecastThing1.id)
+ forecastThing1.getAttributes().addOrReplace(
+ new Attribute<>("newForecastAttribute", NUMBER)
)
- assetStorageService.merge(asset)
+ forecastThing1 = assetStorageService.merge(forecastThing1)
then: "attribute should be unregistered from forecast service"
conditions.eventually {
@@ -241,7 +260,7 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
}
when: "asset is deleted"
- assetStorageService.delete([assetId])
+ assetStorageService.delete([forecastThing1.id])
then: "attribute should be unregistered from forecast service"
conditions.eventually {
@@ -250,10 +269,48 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait
}
private Optional<Double> calculateForecast(List<Double> values) {
- double R = values.size();
- double a = 2 / (R + 1);
- return values.stream().reduce((olderValue, oldValue) -> {
- return (oldValue * a) + (olderValue * (1 - a));
- });
+ double R = values.size()
+ double a = 2 / (R + 1)
+ return values.stream().reduce((result, periodValue) -> (periodValue * a) + (result * (1 - a)))
+ }
+
+ protected Asset<?> addForecastAsset(AssetStorageService assetStorageService, String realm, String attributeName, ForecastConfigurationWeightedExponentialAverage config) {
+ Asset<?> thing = new ThingAsset("Forecast Test Thing")
+ thing.setRealm(realm)
+ thing.getAttributes().addOrReplace(
+ new Attribute<>(attributeName, NUMBER)
+ .addOrReplaceMeta(
+ new MetaItem<>(
+ FORECAST,
+ config
+ )
+ )
+ );
+ thing = assetStorageService.merge(thing)
+ return thing
+ }
+
+ void insertHistoryTestDataToDb(AttributeRef attributeRef, double[][] data, long offsetMillis, ForecastConfigurationWeightedExponentialAverage config) {
+ def values = IntStream.range(0, data.length).mapToObj { hourIndex ->
+ {
+ def hourData = data[hourIndex]
+
+ return IntStream.range(0, config.getPastCount()).mapToObj { periodIndex ->
+ {
+ def periodHourValue = hourData[periodIndex]
+ def timeOffset =
+ config.getPastPeriod().toMillis() * (config.getPastCount() - periodIndex) - // THE DAY OF THE WEEK
+ config.getForecastPeriod().toMillis() * hourIndex - // THE HOUR OFFSET
+ offsetMillis
+
+ return new Pair<>(
+ periodHourValue,
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() - timeOffset), ZoneId.systemDefault()))
+ }
+ }
+ }
+ }.flatMap{it}.toList()
+
+ assetDatapointService.upsertValues(attributeRef.getId(), attributeRef.getName(), values)
}
}