100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Forecast test fix (#1037)
Rich Turner7cbc95c32bbe
diff --git a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java b/manager/src/main/java/org/openremote/manager/asset/ForecastService.javaindex 1b8ff7d42..3f14aae3b 100644--- a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java+++ b/manager/src/main/java/org/openremote/manager/asset/ForecastService.java@@ -420,7 +420,7 @@ public class ForecastService extends RouteBuilder implements ContainerService {} else {return Optional.<Number>empty();}- }).collect(Collectors.toList());+ }).toList();if (forecastTimestamps.size() >= forecastValues.size()) {List<Pair<?, LocalDateTime>> datapoints = IntStreamdiff --git a/model/src/main/java/org/openremote/model/attribute/Attribute.java b/model/src/main/java/org/openremote/model/attribute/Attribute.javaindex b9ae434e2..9d2132c08 100644--- a/model/src/main/java/org/openremote/model/attribute/Attribute.java+++ b/model/src/main/java/org/openremote/model/attribute/Attribute.java@@ -319,9 +319,11 @@ public class Attribute<T> extends AbstractNameValueHolder<T> implements MetaHold@Overridepublic String toString() {+ String valStr = value != null ? value.toString() : null;+ valStr = valStr != null && valStr.length() > 100 ? valStr.substring(0, 100) : valStr;return getClass().getSimpleName() + "{" +"name='" + name + '\'' +- ", value='" + value + '\'' ++ ", value='" + valStr + '\'' +", timestamp='" + getTimestamp().orElse(0L) + '\'' +"} ";}diff --git a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.javaindex b007fc042..02580aa19 100644--- a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java+++ b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import org.openremote.agent.protocol.simulator.SimulatorAgent;import org.openremote.agent.protocol.simulator.SimulatorAgentLink;import org.openremote.container.util.UniqueIdentifierGenerator;-import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.setup.ManagerSetup;import org.openremote.model.Constants;import org.openremote.model.Container;@@ -36,22 +35,13 @@ import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.MetaItem;import org.openremote.model.geo.GeoJSONPoint;import org.openremote.model.security.Realm;-import org.openremote.model.util.Pair;import org.openremote.model.util.ValueUtil;-import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage;import org.openremote.model.value.ValueConstraint;import org.openremote.model.value.ValueType;import org.openremote.model.value.impl.ColourRGB;-import java.time.Duration;-import java.time.Instant;-import java.time.LocalDateTime;-import java.time.ZoneId;-import java.time.temporal.ChronoUnit;-import java.util.ArrayList;import java.util.Arrays;import java.util.List;-import java.util.stream.Collectors;import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT;import static org.openremote.model.Constants.*;@@ -111,16 +101,7 @@ public class ManagerTestSetup extends ManagerSetup {public String electricityBatteryAssetId;public String light1Id;public String light2Id;- public String forecastThing1Id;- public String forecastThing2Id;- public String forecastAttributeName = "forecastAttribute";- public ForecastConfigurationWeightedExponentialAverage forecastConfig = new ForecastConfigurationWeightedExponentialAverage(- new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("P7D"),- 3,- new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("PT1H"),- 4- );- public List<List<Double>> forecastTestHistoryData = new ArrayList<>();+public ManagerTestSetup(Container container) {super(container);@@ -307,27 +288,6 @@ public class ManagerTestSetup extends ManagerSetup {electricitySupplierAsset = assetStorageService.merge(electricitySupplierAsset);electricitySupplierAssetId = electricitySupplierAsset.getId();-- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.0, 3.0, 2.0}));- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.1, 3.1, 2.1}));- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.2, 3.2, 2.2}));- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.3, 3.3, 2.3}));- forecastTestHistoryData.add(Arrays.asList(new Double[] {4.4, 3.4, 2.4}));-- forecastThing1Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);- forecastThing2Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);-- Long offset = ((forecastConfig.getForecastPeriod().toMillis() * 2) / 3) * (-1);- insertHistoryTestDataToDb(new AttributeRef(forecastThing1Id, forecastAttributeName), forecastTestHistoryData, 0, forecastConfig);- insertHistoryTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestHistoryData, offset ,forecastConfig);-- List<Pair<Double, Long>> forecastTestData = new ArrayList<>();- forecastTestData.add(new Pair<>(1.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() + offset));- forecastTestData.add(new Pair<>(2.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() * 2 + offset));-- insertForecastTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestData);--// ################################ Assets for 'building' realm ###################################BuildingAsset smartBuilding = new BuildingAsset("Smart building");@@ -731,59 +691,4 @@ public class ManagerTestSetup extends ManagerSetup {LightAsset lightController_3Asset = createDemoLightControllerAsset("LightController 3", assetArea3, new GeoJSONPoint(5.487478, 51.446979));lightController_3Asset = assetStorageService.merge(lightController_3Asset);}-- public void insertHistoryTestDataToDb(AttributeRef attributeRef, List<List<Double>> data, long offset, ForecastConfigurationWeightedExponentialAverage config) {- for (int pastPeriodIndex = config.getPastCount(); pastPeriodIndex > 0; pastPeriodIndex--) {- List<Double> values = new ArrayList<>(data.size());- for (int forecastPeriodIndex = 0; forecastPeriodIndex < data.size(); forecastPeriodIndex++) {- values.add(data.get(forecastPeriodIndex).get(config.getPastCount() - pastPeriodIndex));- }- insertHistoryTestDataToDb(attributeRef, pastPeriodIndex, values, offset, config);- }- }-- public void insertHistoryTestDataToDb(AttributeRef attributeRef, int pastPeriodIndex, List<Double> values, long offset, ForecastConfigurationWeightedExponentialAverage config) {- LocalDateTime now = LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis()), ZoneId.systemDefault());- long nowInMillis = now.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();- List<Pair<?, LocalDateTime>> history = new ArrayList<>();- for (int forecastPeriodIndex = 0; forecastPeriodIndex < values.size(); forecastPeriodIndex++) {- long timespan = config.getPastPeriod().toMillis() * pastPeriodIndex -- config.getForecastPeriod().toMillis() * (forecastPeriodIndex + 1) +- Duration.ofMinutes(2).toMillis() - offset;- history.add(- new Pair<>(- Double.valueOf(values.get(forecastPeriodIndex)),- LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() - timespan), ZoneId.systemDefault())- )- );- }- assetDatapointService.upsertValues(attributeRef.getId(), attributeRef.getName(), history);- }-- @SuppressWarnings("unchecked")- public void insertForecastTestDataToDb(AttributeRef attributeRef, List<Pair<Double, Long>> data) {- List<Pair<Double, LocalDateTime>> valuesAndTimestamps = data- .stream()- .map(p -> new Pair<Double,LocalDateTime>(- p.getKey(),- LocalDateTime.ofInstant(Instant.ofEpochMilli(p.getValue()), ZoneId.systemDefault())))- .collect(Collectors.toList());- assetPredictedDatapointService.updateValues(attributeRef.getId(), attributeRef.getName(), (List)valuesAndTimestamps);- }-- public String addForecastTestAsset(AssetStorageService assetStorageService, String realm, String attributeName, ForecastConfigurationWeightedExponentialAverage config) {- Asset<?> thing = new ThingAsset("Forecast Test Thing");- thing.setRealm(realm);- thing.getAttributes().addOrReplace(- new Attribute<>(attributeName, NUMBER)- .addOrReplaceMeta(- new MetaItem<>(- FORECAST,- config- )- )- );- thing = assetStorageService.merge(thing);- return thing.getId();- }}diff --git a/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovyindex 7545e9442..1e67be4cd 100644--- a/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy@@ -6,10 +6,13 @@ import org.openremote.manager.asset.ForecastServiceimport org.openremote.manager.datapoint.AssetDatapointServiceimport org.openremote.manager.datapoint.AssetPredictedDatapointServiceimport org.openremote.manager.setup.SetupService+import org.openremote.model.asset.Asset+import org.openremote.model.asset.impl.ThingAssetimport org.openremote.model.attribute.Attributeimport org.openremote.model.attribute.AttributeRefimport org.openremote.model.attribute.MetaItemimport org.openremote.model.datapoint.AssetPredictedDatapoint+import org.openremote.model.util.Pairimport org.openremote.model.value.ForecastConfigurationWeightedExponentialAverageimport org.openremote.setup.integration.ManagerTestSetupimport org.openremote.test.ManagerContainerTrait@@ -19,6 +22,11 @@ import spock.lang.Specificationimport spock.util.concurrent.PollingConditionsimport java.time.Duration+import java.time.Instant+import java.time.LocalDateTime+import java.time.ZoneId+import java.time.temporal.ChronoUnit+import java.util.stream.IntStreamimport static java.util.concurrent.TimeUnit.MILLISECONDSimport static org.openremote.model.value.MetaItemType.FORECAST@@ -39,17 +47,40 @@ class ForecastServiceTest extends Specification implements ManagerContainerTraitstatic ForecastService forecastService@Sharedstatic TimerService timerService-- def setupSpec() {- given: "the server container is started"- def container = startContainer(defaultConfig(), defaultServices())- managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)- assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)- assetStorageService = container.getService(AssetStorageService.class)- assetDatapointService = container.getService(AssetDatapointService.class)- forecastService = container.getService(ForecastService.class)- timerService = container.getService(TimerService.class)- }+ @Shared+ static ForecastConfigurationWeightedExponentialAverage forecastConfig = new ForecastConfigurationWeightedExponentialAverage(+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("P7D"),+ 3,+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("PT1H"),+ 4+ )+ @Shared+ // Outer list is hours+ // Inner list is weekly values+ static double[][] forecastHistoricalData = [+ [4.0, 3.0, 2.0],+ [4.1, 3.1, 2.1],+ [4.2, 3.2, 2.2],+ [4.3, 3.3, 2.3],+ [4.4, 3.4, 2.4]+ ]+ @Shared+ static ThingAsset forecastThing2+ @Shared+ static String forecastAttributeName = "forecastAttribute"++ // TODO: RT - Not too sure the intention with the offset and seems to be used only in ignored tests so skip for now+// def setupSpec() {+// forecastThing2 = addForecastAsset(assetStorageService, managerTestSetup.realmEnergyName, forecastAttributeName, forecastConfig)+// Long offset = ((forecastConfig.getForecastPeriod().toMillis() * 2) / 3) * (-1)+// insertHistoryTestDataToDb(new AttributeRef(forecastThing2.id, forecastAttributeName), forecastHistoricalData, offset, forecastConfig)+//+// // Insert forecast test data+// assetPredictedDatapointService.updateValues(forecastThing2.id, forecastAttributeName, [+// new Pair<>(1.0d, LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() + offset), ZoneId.systemDefault())),+// new Pair<>(2.0d, LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() * 2 + offset), ZoneId.systemDefault()))+// ])+// }@Ignoredef "Test forecast calculation after server restart - no initial forecast data"() {@@ -121,37 +152,44 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait}def "Test adding updating and deleting an asset"() {- given: "no assets with forecast configuration"+ given: "the server container is started"+ def container = startContainer(defaultConfig(), defaultServices())def conditions = new PollingConditions(timeout: 10, delay: 0.2)- def config = managerTestSetup.forecastConfig- def attributeName = "forecastTestAttribute"+ managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)+ assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)+ assetStorageService = container.getService(AssetStorageService.class)+ assetDatapointService = container.getService(AssetDatapointService.class)+ forecastService = container.getService(ForecastService.class)+ timerService = container.getService(TimerService.class)- and:+ when: "the system time is stopped and set exactly on the next hour"stopPseudoClock()+ def now = Instant.ofEpochMilli(timerService.getCurrentTimeMillis())+ now = now.truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS)+ advancePseudoClock(now.toEpochMilli()-timerService.getCurrentTimeMillis(), MILLISECONDS, container)- when: "asset is added"- String assetId = managerTestSetup.addForecastTestAsset(- assetStorageService, managerTestSetup.realmEnergyName, attributeName, config- )- AttributeRef attributeRef = new AttributeRef(assetId, attributeName)+ and: "forecast asset is added"+ def forecastThing1 = addForecastAsset(assetStorageService, managerTestSetup.realmEnergyName, forecastAttributeName, forecastConfig)+ def attributeRef = new AttributeRef(forecastThing1.id, forecastAttributeName)then: "attribute should be registered at the forecast service"conditions.eventually {assert forecastService.forecastTaskManager.containsAttribute(attributeRef)}- when: "test data is inserted to the database"+ when: "the forecast service is stopped"forecastService.forecastTaskManager.stop(3000)- managerTestSetup.insertHistoryTestDataToDb(- attributeRef, managerTestSetup.forecastTestHistoryData, config.forecastPeriod.toMillis(), config- )++ and: "historical data is added for the forecast attribute"+ // Offset historical data by two hours to match up with test intervals+ insertHistoryTestDataToDb(new AttributeRef(forecastThing1.id, forecastAttributeName), forecastHistoricalData, 2*3600000, forecastConfig)and: "time has passed"advancePseudoClock(- config.forecastPeriod.toMillis(), MILLISECONDS, container+ forecastConfig.forecastPeriod.toMillis(), MILLISECONDS, container)- and: "forecast service is restarted"+ and: "the forecast service is started again"forecastService.forecastTaskManager.start(timerService.currentTimeMillis)then: "initial forecast values should be calculated"@@ -159,45 +197,26 @@ class ForecastServiceTest extends Specification implements ManagerContainerTraitList<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService.getDatapoints(attributeRef).sort {it.timestamp}- assert predictedDatapoints.size() == config.forecastCount+ assert predictedDatapoints.size() == forecastConfig.forecastCountfor (int i = 0; i < predictedDatapoints.size(); i++) {- assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()- assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + (i + 1) * config.forecastPeriod.toMillis()- }- }-- when: "time has passed"- forecastService.forecastTaskManager.stop(3000)- advancePseudoClock(- config.forecastPeriod.toMillis(), MILLISECONDS, container- )- forecastService.forecastTaskManager.start(timerService.currentTimeMillis)-- then: "forecast values should be updated"- conditions.eventually {- List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService- .getDatapoints(attributeRef)- .sort {it.timestamp}- assert predictedDatapoints.size() == config.forecastCount- for (int i = 1; i < predictedDatapoints.size(); i++) {- assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()- assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + i * config.forecastPeriod.toMillis()+ assert predictedDatapoints[i].value == calculateForecast(forecastHistoricalData[i][0..forecastConfig.pastCount - 1]).get()+ assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + (i + 1) * forecastConfig.forecastPeriod.toMillis()}}when: "new forecast attribute is added"- def asset = assetStorageService.find(assetId)- asset.getAttributes().addOrReplace(- new Attribute<>("newForecastAttribute", NUMBER)- .addOrReplaceMeta(- new MetaItem<>(- FORECAST,- config- )- )+ forecastThing1 = assetStorageService.find(forecastThing1.id)+ forecastThing1.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ forecastConfig+ )+ ))- assetStorageService.merge(asset)- def newAttributeRef = new AttributeRef(assetId, "newForecastAttribute")+ forecastThing1 = assetStorageService.merge(forecastThing1)+ def newAttributeRef = new AttributeRef(forecastThing1.id, "newForecastAttribute")then: "new attribute should be registered at the forecast service"conditions.eventually {@@ -205,35 +224,35 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait}when: "forecast configuration is updated"- asset = assetStorageService.find(assetId)- def updatedPastCount = config.pastCount + 1+ forecastThing1 = assetStorageService.find(forecastThing1.id)+ def updatedPastCount = forecastConfig.pastCount + 1def updatedConfig = new ForecastConfigurationWeightedExponentialAverage(- config.pastPeriod, updatedPastCount, config.forecastPeriod, config.forecastCount+ forecastConfig.pastPeriod, updatedPastCount, forecastConfig.forecastPeriod, forecastConfig.forecastCount)- asset.getAttributes().addOrReplace(- new Attribute<>("newForecastAttribute", NUMBER)- .addOrReplaceMeta(- new MetaItem<>(- FORECAST,- updatedConfig- )- )+ forecastThing1.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ updatedConfig+ )+ ))- assetStorageService.merge(asset)+ forecastThing1 = assetStorageService.merge(forecastThing1)then: "forecast service should contain updated forecast configuration"conditions.eventually {ForecastService.ForecastAttribute forecastAttribute = forecastService.forecastTaskManager.getAttribute(newAttributeRef)assert forecastAttribute != null- assert updatedPastCount == forecastAttribute.config.pastCount+ assert updatedPastCount == (forecastAttribute.config as ForecastConfigurationWeightedExponentialAverage).pastCount}when: "forecast configuration is deleted"- asset = assetStorageService.find(assetId)- asset.getAttributes().addOrReplace(- new Attribute<>("newForecastAttribute", NUMBER)+ forecastThing1 = assetStorageService.find(forecastThing1.id)+ forecastThing1.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER))- assetStorageService.merge(asset)+ forecastThing1 = assetStorageService.merge(forecastThing1)then: "attribute should be unregistered from forecast service"conditions.eventually {@@ -241,7 +260,7 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait}when: "asset is deleted"- assetStorageService.delete([assetId])+ assetStorageService.delete([forecastThing1.id])then: "attribute should be unregistered from forecast service"conditions.eventually {@@ -250,10 +269,48 @@ class ForecastServiceTest extends Specification implements ManagerContainerTrait}private Optional<Double> calculateForecast(List<Double> values) {- double R = values.size();- double a = 2 / (R + 1);- return values.stream().reduce((olderValue, oldValue) -> {- return (oldValue * a) + (olderValue * (1 - a));- });+ double R = values.size()+ double a = 2 / (R + 1)+ return values.stream().reduce((result, periodValue) -> (periodValue * a) + (result * (1 - a)))+ }++ protected Asset<?> addForecastAsset(AssetStorageService assetStorageService, String realm, String attributeName, ForecastConfigurationWeightedExponentialAverage config) {+ Asset<?> thing = new ThingAsset("Forecast Test Thing")+ thing.setRealm(realm)+ thing.getAttributes().addOrReplace(+ new Attribute<>(attributeName, NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ config+ )+ )+ );+ thing = assetStorageService.merge(thing)+ return thing+ }++ void insertHistoryTestDataToDb(AttributeRef attributeRef, double[][] data, long offsetMillis, ForecastConfigurationWeightedExponentialAverage config) {+ def values = IntStream.range(0, data.length).mapToObj { hourIndex ->+ {+ def hourData = data[hourIndex]++ return IntStream.range(0, config.getPastCount()).mapToObj { periodIndex ->+ {+ def periodHourValue = hourData[periodIndex]+ def timeOffset =+ config.getPastPeriod().toMillis() * (config.getPastCount() - periodIndex) - // THE DAY OF THE WEEK+ config.getForecastPeriod().toMillis() * hourIndex - // THE HOUR OFFSET+ offsetMillis++ return new Pair<>(+ periodHourValue,+ LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() - timeOffset), ZoneId.systemDefault()))+ }+ }+ }+ }.flatMap{it}.toList()++ assetDatapointService.upsertValues(attributeRef.getId(), attributeRef.getName(), values)}}