100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Added forecast service implementation (#526) (#996)
rainerhitzc517f54e486d
diff --git a/manager/src/main/java/org/openremote/manager/asset/ForecastService.java b/manager/src/main/java/org/openremote/manager/asset/ForecastService.javanew file mode 100644index 000000000..1b8ff7d42--- /dev/null+++ b/manager/src/main/java/org/openremote/manager/asset/ForecastService.java@@ -0,0 +1,858 @@+/*+ * Copyright 2023, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.manager.asset;++import org.apache.camel.builder.RouteBuilder;+import org.openremote.container.message.MessageBrokerService;+import org.openremote.container.persistence.PersistenceService;+import org.openremote.container.timer.TimerService;+import org.openremote.manager.datapoint.AssetDatapointService;+import org.openremote.manager.datapoint.AssetPredictedDatapointService;+import org.openremote.manager.gateway.GatewayService;+import org.openremote.model.Container;+import org.openremote.model.ContainerService;+import org.openremote.model.PersistenceEvent;+import org.openremote.model.asset.Asset;+import org.openremote.model.attribute.Attribute;+import org.openremote.model.attribute.AttributeMap;+import org.openremote.model.attribute.AttributeRef;+import org.openremote.model.datapoint.AssetDatapoint;+import org.openremote.model.datapoint.AssetPredictedDatapoint;+import org.openremote.model.query.AssetQuery;+import org.openremote.model.query.filter.AttributePredicate;+import org.openremote.model.query.filter.NameValuePredicate;+import org.openremote.model.query.filter.StringPredicate;+import org.openremote.model.util.Pair;+import org.openremote.model.value.ForecastConfiguration;+import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage;+import org.openremote.model.value.MetaItemType;++import jakarta.persistence.TypedQuery;+import java.math.BigDecimal;+import java.math.BigInteger;+import java.time.Duration;+import java.time.Instant;+import java.time.LocalDateTime;+import java.time.ZoneId;+import java.util.ArrayList;+import java.util.Collections;+import java.util.Date;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.OptionalLong;+import java.util.Set;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;+import java.util.logging.Level;+import java.util.logging.Logger;+import java.util.stream.Collectors;+import java.util.stream.IntStream;++import static java.util.stream.Collectors.toList;+import static org.openremote.container.concurrent.GlobalLock.withLockReturning;+import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC;+import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType;+import static org.openremote.manager.gateway.GatewayService.isNotForGateway;+import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttributes;+import static org.openremote.model.util.TextUtil.requireNonNullAndNonEmpty;+import static org.openremote.model.value.MetaItemType.FORECAST;++/**+ * Calculates forecast values for asset attributes with an attached {@link MetaItemType#FORECAST}+ * configuration like {@link ForecastConfigurationWeightedExponentialAverage}.+ */+public class ForecastService extends RouteBuilder implements ContainerService {++ private static final Logger LOG = Logger.getLogger(ForecastService.class.getName());+ private static long STOP_TIMEOUT = Duration.ofSeconds(5).toMillis();++ protected TimerService timerService;+ protected GatewayService gatewayService;+ protected AssetStorageService assetStorageService;+ protected AssetDatapointService assetDatapointService;+ protected PersistenceService persistenceService;+ protected AssetPredictedDatapointService assetPredictedDatapointService;+ protected ScheduledExecutorService executorService;+ protected ForecastTaskManager forecastTaskManager = new ForecastTaskManager();++ @Override+ public void init(Container container) throws Exception {+ timerService = container.getService(TimerService.class);+ gatewayService = container.getService(GatewayService.class);+ assetStorageService = container.getService(AssetStorageService.class);+ assetDatapointService = container.getService(AssetDatapointService.class);+ persistenceService = container.getService(PersistenceService.class);+ assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class);+ executorService = container.getExecutorService();+ }++ @Override+ public void start(Container container) throws Exception {+ container.getService(MessageBrokerService.class).getContext().addRoutes(this);++ LOG.fine("Loading forecast asset attributes...");++ List<Asset<?>> assets = getForecastAssets();++ Set<ForecastAttribute> forecastAttributes = assets+ .stream()+ .flatMap(asset -> asset.getAttributes()+ .stream()+ .filter(attr -> {+ if (attr.hasMeta(FORECAST)) {+ Optional<ForecastConfiguration> forecastConfig = attr.getMetaValue(FORECAST);+ return forecastConfig.isPresent() &&+ ForecastConfigurationWeightedExponentialAverage.TYPE.equals(forecastConfig.get().getType());+ }+ return false;+ })+ .map(attr -> new ForecastAttribute(asset, attr)))+ .collect(Collectors.toSet());++ LOG.fine("Found forecast asset attributes count = " + forecastAttributes.size());++ forecastTaskManager.init(forecastAttributes);+ }++ @Override+ public void stop(Container container) throws Exception {+ forecastTaskManager.stop(STOP_TIMEOUT);+ }++ @SuppressWarnings("unchecked")+ @Override+ public void configure() throws Exception {+ from(PERSISTENCE_TOPIC)+ .routeId("ForecastConfigurationPersistenceChanges")+ .filter(isPersistenceEventForEntityType(Asset.class))+ .filter(isNotForGateway(gatewayService))+ .process(exchange -> {+ PersistenceEvent<Asset<?>> persistenceEvent = (PersistenceEvent<Asset<?>>)exchange.getIn().getBody(PersistenceEvent.class);+ processAssetChange(persistenceEvent);+ });+ }++ protected List<Asset<?>> getForecastAssets() {+ return withLockReturning(getClass().getSimpleName() + "::getForecastAssets", () -> {+ return assetStorageService.findAll(+ new AssetQuery().attributes(+ new AttributePredicate().meta(+ new NameValuePredicate(+ FORECAST,+ new StringPredicate(AssetQuery.Match.CONTAINS, true, "type")+ )+ )+ )+ );+ });+ }++ protected void processAssetChange(PersistenceEvent<Asset<?>> persistenceEvent) {++ Asset<?> asset = persistenceEvent.getEntity();+ Set<ForecastAttribute> forecastAttributes = null;++ switch (persistenceEvent.getCause()) {+ case CREATE:+ forecastAttributes = asset.getAttributes()+ .stream()+ .filter(attr -> {+ if (attr.hasMeta(FORECAST)) {+ Optional<ForecastConfiguration> forecastConfig = attr.getMetaValue(FORECAST);+ return forecastConfig.isPresent() &&+ ForecastConfigurationWeightedExponentialAverage.TYPE.equals(forecastConfig.get().getType());+ }+ return false;+ })+ .map(attr -> new ForecastAttribute(asset, attr))+ .collect(Collectors.toSet());++ forecastTaskManager.add(forecastAttributes);++ break;+ case UPDATE:+ if (persistenceEvent.getPropertyNames() == null || persistenceEvent.getPropertyNames().indexOf("attributes") < 0) {+ return;+ }++ List<Attribute<?>> oldAttributes = ((AttributeMap)persistenceEvent.getPreviousState("attributes"))+ .stream()+ .filter(attr -> attr.hasMeta(FORECAST))+ .collect(toList());+ List<Attribute<?>> newAttributes = ((AttributeMap) persistenceEvent.getCurrentState("attributes"))+ .stream()+ .filter(attr -> attr.hasMeta(FORECAST))+ .collect(Collectors.toList());++ List<Attribute<?>> newOrModifiedAttributes = getAddedOrModifiedAttributes(oldAttributes, newAttributes).collect(toList());++ Set<ForecastAttribute> attributesToDelete = newOrModifiedAttributes+ .stream()+ .map(attr -> new ForecastAttribute(asset, attr))+ .filter(attr -> forecastTaskManager.containsAttribute(attr))+ .collect(Collectors.toSet());++ attributesToDelete.addAll(oldAttributes+ .stream()+ .filter(oldAttr -> {+ return newAttributes+ .stream()+ .filter(newAttr -> newAttr.getName().equals(oldAttr.getName()))+ .count() == 0;+ })+ .map(attr -> new ForecastAttribute(asset, attr))+ .collect(Collectors.toList())+ );++ forecastTaskManager.delete(attributesToDelete);++ forecastAttributes = newOrModifiedAttributes+ .stream()+ .filter(attr -> {+ if (attr.hasMeta(FORECAST)) {+ Optional<ForecastConfiguration> forecastConfig = attr.getMetaValue(FORECAST);+ return forecastConfig.isPresent() &&+ ForecastConfigurationWeightedExponentialAverage.TYPE.equals(forecastConfig.get().getType());+ }+ return false;+ })+ .map(attr -> new ForecastAttribute(asset, attr))+ .collect(Collectors.toSet());++ forecastTaskManager.add(forecastAttributes);++ break;+ case DELETE:+ forecastAttributes = asset.getAttributes()+ .stream()+ .filter(attr -> attr.hasMeta(FORECAST))+ .map(attr -> new ForecastAttribute(asset, attr))+ .collect(Collectors.toSet());++ forecastTaskManager.delete(forecastAttributes);++ break;+ }+ }++ private class ForecastTaskManager {++ private static long DELAY_MIN_TO_CANCEL_SAFELY = Duration.ofSeconds(2).toMillis();+ private static long DEFAULT_SCHEDULE_DELAY = Duration.ofMinutes(15).toMillis();++ protected ScheduledFuture<?> scheduledFuture;+ protected Map<ForecastAttribute, Long> nextForecastCalculationMap = new HashMap<>();+ protected Set<ForecastAttribute> forecastAttributes = new HashSet<>();++ public synchronized void init(Set<ForecastAttribute> attributes) {+ if (attributes == null || attributes.size() == 0) {+ return;+ }+ long now = timerService.getCurrentTimeMillis();+ attributes.forEach(attr -> {+ if (attr.isValidConfig()) {+ attr.setForecastTimestamps(loadForecastTimestampsFromDb(attr.getAttributeRef(), now));+ forecastAttributes.add(attr);+ }+ });+ start(now, true);+ }++ public synchronized void add(Set<ForecastAttribute> attributes) {+ if (attributes == null || attributes.size() == 0) {+ return;+ }+ attributes.forEach(attr -> {+ if (attr.isValidConfig()) {+ LOG.fine("Adding asset attribute to forecast calculation service: " + attr.getAttributeRef());+ forecastAttributes.add(attr);+ }+ });+ long now = timerService.getCurrentTimeMillis();+ if (scheduledFuture != null) {+ if (scheduledFuture.getDelay(TimeUnit.MILLISECONDS) > DELAY_MIN_TO_CANCEL_SAFELY) {+ scheduledFuture.cancel(false);+ scheduledFuture = null;+ start(now);+ }+ } else {+ start(now);+ }+ }++ public synchronized void delete(Set<ForecastAttribute> attributes) {+ attributes.forEach(attr -> delete(attr));+ }++ public synchronized void delete(ForecastAttribute attribute) {+ LOG.fine("Removing asset attribute from forecast calculation service: " + attribute.getAttributeRef());++ nextForecastCalculationMap.remove(attribute);+ forecastAttributes.remove(attribute);++ assetPredictedDatapointService.purgeValues(attribute.getAttributeRef().getId(), attribute.getAttributeRef().getName());+ }++ public synchronized boolean containsAttribute(ForecastAttribute attribute) {+ return forecastAttributes.contains(attribute);+ }++ public synchronized boolean containsAttribute(AttributeRef attributeRef) {+ return forecastAttributes+ .stream()+ .filter(attr -> attr.getAttributeRef().equals(attributeRef))+ .findFirst()+ .isPresent();+ }++ public synchronized ForecastAttribute getAttribute(AttributeRef attributeRef) {+ return forecastAttributes+ .stream()+ .filter(attr -> attr.getAttributeRef().equals(attributeRef))+ .findFirst()+ .orElse(null);+ }++ private boolean stop(long timeout) {+ long start = timerService.getCurrentTimeMillis();+ while (true) {+ synchronized (ForecastTaskManager.this) {+ if (scheduledFuture == null) {+ return true;+ } else if (scheduledFuture != null && scheduledFuture.getDelay(TimeUnit.MILLISECONDS) > DELAY_MIN_TO_CANCEL_SAFELY) {+ scheduledFuture.cancel(false);+ scheduledFuture = null;+ return true;+ } else {+ if (timerService.getCurrentTimeMillis() - start > timeout) {+ scheduledFuture.cancel(true);+ scheduledFuture = null;+ return false;+ }+ }+ }+ try {+ Thread.currentThread().sleep(300);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return false;+ }+ }+ }++ private synchronized void start(long now) {+ start(now, false);+ }++ private synchronized void start(long now, boolean isServerRestart) {+ if (scheduledFuture == null) {+ addForecastTimestamps(now, isServerRestart);+ updateNextForecastCalculationMap();+ scheduleForecastCalculation(now, Optional.empty());+ }+ }++ private void calculateForecasts() {+ final long now = timerService.getCurrentTimeMillis();+ List<ForecastAttribute> attributesToCalculate = new ArrayList<>();++ try {+ synchronized (ForecastTaskManager.this) {+ if (Thread.currentThread().isInterrupted()) {+ return;+ }+ purgeForecastTimestamps(now);+ addForecastTimestamps(now, false);+ purgeForecastTimestamps(now);++ nextForecastCalculationMap.forEach((attribute, nextForecastCalculationTimestamp) -> {+ if (nextForecastCalculationTimestamp <= now) {+ attributesToCalculate.add(attribute);+ }+ });+ }++ attributesToCalculate.forEach(attr -> {+ if (Thread.currentThread().isInterrupted()) {+ return;+ }+ List<Long> forecastTimestamps = attr.getForecastTimestamps();+ if (forecastTimestamps == null || forecastTimestamps.size() == 0) {+ return;+ }+ if (!(attr.getConfig() instanceof ForecastConfigurationWeightedExponentialAverage)) {+ return;+ }+ ForecastConfigurationWeightedExponentialAverage weaConfig = (ForecastConfigurationWeightedExponentialAverage)attr.getConfig();++ LOG.fine("Calculating forecast values for attribute: " + attr.getAttributeRef());++ Long offset = forecastTimestamps.get(0) - (now + weaConfig.getForecastPeriod().toMillis());+ List<List<Long>> allSampleTimestamps = calculateSampleTimestamps(weaConfig, offset);+ List<DatapointBucket> historyDatapointBuckets = getHistoryDataFromDb(attr.getAttributeRef(), weaConfig, offset);++ List<Optional<Number>> forecastValues = allSampleTimestamps.stream().map(sampleTimestamps -> {+ List<AssetDatapoint> sampleDatapoints = findSampleDatapoints(historyDatapointBuckets, sampleTimestamps);++ if (sampleDatapoints.size() == weaConfig.getPastCount()) {+ return calculateWeightedExponentialAverage(attr.getAttribute(), sampleDatapoints);+ } else {+ return Optional.<Number>empty();+ }+ }).collect(Collectors.toList());++ if (forecastTimestamps.size() >= forecastValues.size()) {+ List<Pair<?, LocalDateTime>> datapoints = IntStream+ .range(0, forecastValues.size())+ .filter(i -> forecastValues.get(i).isPresent())+ .mapToObj(i -> new Pair<>(+ forecastValues.get(i).get(),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(forecastTimestamps.get(i)), ZoneId.systemDefault()))+ )+ .collect(Collectors.toList());++ assetPredictedDatapointService.purgeValues(attr.getId(), attr.getName());++ if (datapoints.size() > 0) {+ LOG.fine("Updating forecast values for attribute: " + attr.getAttributeRef());+ assetPredictedDatapointService.updateValues(attr.getId(), attr.getName(), datapoints);+ }+ }+ });++ synchronized (ForecastTaskManager.this) {+ if (Thread.currentThread().isInterrupted()) {+ return;+ }+ updateNextForecastCalculationMap();+ scheduleForecastCalculation(timerService.getCurrentTimeMillis(), Optional.empty());+ }+ } catch (Exception e) {+ LOG.log(Level.SEVERE, "Exception while calculating and updating forecast values", e);++ scheduleForecastCalculation(+ timerService.getCurrentTimeMillis(),+ Optional.of(DEFAULT_SCHEDULE_DELAY)+ );+ }+ }++ private synchronized void scheduleForecastCalculation(long now, Optional<Long> fixedDelay) {+ Optional<Long> delay = fixedDelay;++ if (delay.isEmpty()) {+ delay = calculateScheduleDelay(now);+ }++ if (delay.isPresent()) {+ LOG.fine("Scheduling next forecast calculation in '" + delay.get() + " [ms]'.");+ scheduledFuture = executorService.schedule(() -> calculateForecasts(), delay.get(), TimeUnit.MILLISECONDS);+ } else {+ scheduledFuture = null;+ if (forecastAttributes.size() > 0) {+ LOG.fine("Scheduling next forecast calculation in '" + DEFAULT_SCHEDULE_DELAY + " [ms]'.");+ scheduleForecastCalculation(now, Optional.of(DEFAULT_SCHEDULE_DELAY));+ }+ }+ }++ private List<List<Long>> calculateSampleTimestamps(ForecastConfigurationWeightedExponentialAverage config, Long offset) {+ List<List<Long>> sampleTimestamps = new ArrayList<>(config.getForecastCount());+ long now = timerService.getCurrentTimeMillis();+ long pastPeriod = config.getPastPeriod().toMillis();+ long forecastPeriod = config.getForecastPeriod().toMillis();++ for (int forecastIndex = 1; forecastIndex <= config.getForecastCount(); forecastIndex++) {+ List<Long> timestamps = new ArrayList<>(config.getPastCount());+ for (int pastPeriodIndex = config.getPastCount(); pastPeriodIndex > 0; pastPeriodIndex--) {+ timestamps.add(now - (pastPeriod * pastPeriodIndex) + (forecastPeriod * forecastIndex) + offset);+ }+ sampleTimestamps.add(timestamps);+ }++ return sampleTimestamps;+ }++ private List<Long> calculateForecastTimestamps(long now, ForecastConfigurationWeightedExponentialAverage config) {+ List<Long> forecastTimestamps = new ArrayList<>(config.getForecastCount());+ long forecastPeriod = config.getForecastPeriod().toMillis();++ for (int forecastIndex = 1; forecastIndex <= config.getForecastCount(); forecastIndex++) {+ forecastTimestamps.add(now + forecastPeriod * forecastIndex);+ }++ return forecastTimestamps;+ }++ private List<AssetDatapoint> findSampleDatapoints(List<DatapointBucket> datapointBuckets, List<Long> sampleTimestamps) {+ List<AssetDatapoint> sampleDatapoints = new ArrayList<>(sampleTimestamps.size());++ for (Long timestamp : sampleTimestamps) {+ AssetDatapoint foundDatapoint = null;++ List<AssetDatapoint> datapoints = datapointBuckets+ .stream()+ .filter(bucket -> bucket.isInTimeRange(timestamp))+ .findFirst()+ .map(bucket -> bucket.getDatapoints())+ .orElse(null);++ if (datapoints == null) {+ continue;+ }++ for (AssetDatapoint assetDatapoint : datapoints) {+ if (assetDatapoint.getTimestamp() <= timestamp) {+ foundDatapoint = assetDatapoint;+ } else if (assetDatapoint.getTimestamp() > timestamp) {+ break;+ }+ }++ if (foundDatapoint != null) {+ sampleDatapoints.add(foundDatapoint);+ }+ }+ return sampleDatapoints;+ }++ private Optional<Number> calculateWeightedExponentialAverage(Attribute<?> attribute, List<AssetDatapoint> datapoints) {+ // a = 2 / (R + 1)+ // p: past period+ // Attr(t) = Attr(t-p) * a + Attr(t-2p) * (1 - a)+ List<Object> values = datapoints+ .stream()+ .map(dp -> dp.getValue())+ .collect(Collectors.toList());+ double R = datapoints.size();+ double a = 2 / (R + 1);++ Class<?> clazz = attribute.getType().getType();+ if (Long.class == clazz || Integer.class == clazz || Short.class == clazz || Byte.class == clazz ||+ Double.class == clazz || Float.class == clazz) {+ if (values.size() == 1) {+ values.add(0, Double.valueOf(0));+ }+ Optional<Number> value = values+ .stream()+ .map(v -> (Number)v)+ .reduce((olderValue, oldValue) ->+ Double.valueOf(oldValue.doubleValue() * a + olderValue.doubleValue() * (1 - a))+ );+ if (value.isPresent()) {+ if (clazz == Long.class) {+ value = Optional.of(Long.valueOf(value.get().longValue()));+ } else if (clazz == Integer.class) {+ value = Optional.of(Integer.valueOf(value.get().intValue()));+ } else if (clazz == Short.class) {+ value = Optional.of(Short.valueOf(value.get().shortValue()));+ } else if (clazz == Byte.class) {+ value = Optional.of(Byte.valueOf(value.get().byteValue()));+ } else if (clazz == Double.class) {+ value = Optional.of(Double.valueOf(value.get().doubleValue()));+ } else if (clazz == Float.class) {+ value = Optional.of(Float.valueOf(value.get().floatValue()));+ } else {+ value = Optional.empty();+ }+ }+ return value;+ } else if (attribute.getType().getType() == BigDecimal.class) {+ if (values.size() == 1) {+ values.add(0, BigDecimal.valueOf(0));+ }+ return values+ .stream()+ .map(v -> (Number)v)+ .reduce((olderValue, oldValue) ->+ ((BigDecimal)oldValue).multiply(BigDecimal.valueOf(a)).add(((BigDecimal)olderValue).multiply(BigDecimal.valueOf(1 - a)))+ );+ } else if (attribute.getType().getType() == BigInteger.class) {+ if (values.size() == 1) {+ values.add(0, BigInteger.valueOf(0));+ }+ // Attr(t) = Attr(t-p) * a + Attr(t-2p) * (1 - a)+ // Attr(t) = (Attr(t-p) * 2 / (R + 1) + Attr(t-2p) * (1 - 2 / (R + 1))) * (R + 1)/(R + 1)+ // Attr(t) = ((Attr(t-p) * 2 + Attr(t-2p) * (R - 1)) / (R + 1)+ return values+ .stream()+ .map(v -> (Number)v)+ .reduce((olderValue, oldValue) ->+ ((BigInteger)oldValue).multiply(BigInteger.valueOf(2))+ .add(((BigInteger)olderValue).multiply(BigInteger.valueOf((long)R - 1)))+ .divide(BigInteger.valueOf((long)R + 1))+ );+ }+ return Optional.empty();+ }++ private void updateNextForecastCalculationMap() {+ nextForecastCalculationMap.clear();+ forecastAttributes.forEach(attr -> {+ List<Long> timestamps = attr.getForecastTimestamps();+ if (timestamps != null && timestamps.size() > 0) {+ nextForecastCalculationMap.put(attr, timestamps.get(0));+ }+ });+ }++ private Optional<Long> calculateScheduleDelay(long now) {+ OptionalLong calculateForecastTimestamp = nextForecastCalculationMap.values().stream().mapToLong(v -> v).min();+ if (calculateForecastTimestamp.isPresent()) {+ long delay = calculateForecastTimestamp.getAsLong() - now;+ return Optional.of(delay < 0 ? 0 : delay);+ } else {+ return Optional.empty();+ }+ }++ private void addForecastTimestamps(long now, boolean isServerRestart) {+ forecastAttributes.forEach(attr -> {+ ForecastConfiguration config = attr.getConfig();+ ForecastConfigurationWeightedExponentialAverage weaConfig = null;+ if (config instanceof ForecastConfigurationWeightedExponentialAverage) {+ weaConfig = (ForecastConfigurationWeightedExponentialAverage)config;+ } else {+ return;+ }+ List<Long> newTimestamps = calculateForecastTimestamps(now, weaConfig);+ List<Long> oldTimestamps = attr.getForecastTimestamps();+ if (oldTimestamps == null || oldTimestamps.size() == 0) {+ if (newTimestamps.size() > 0) {+ // force immediate forecast calculation+ newTimestamps.add(0, now);+ }+ attr.setForecastTimestamps(newTimestamps);+ } else if (newTimestamps.size() > 0 && oldTimestamps.size() > 0) {+ long offset = oldTimestamps.get(0) - newTimestamps.get(0);+ List<Long> newShiftedTimestamps = newTimestamps.stream().map(timestamp -> timestamp + offset).collect(Collectors.toList());+ while(true) {+ if (newShiftedTimestamps.get(0) < now) {+ newShiftedTimestamps = newShiftedTimestamps+ .stream()+ .map(timestamp -> timestamp + ((ForecastConfigurationWeightedExponentialAverage)config).getForecastPeriod().toMillis())+ .collect(Collectors.toList());+ } else {+ break;+ }+ }+ if (isServerRestart && (oldTimestamps.get(0) < now || newTimestamps.size() > oldTimestamps.size())) {+ // force immediate forecast calculation+ newShiftedTimestamps.add(0, now);+ }+ attr.setForecastTimestamps(newShiftedTimestamps);+ }+ });+ }++ public void purgeForecastTimestamps(long now) {+ forecastAttributes.forEach(attr -> {+ List<Long> timestamps = attr.getForecastTimestamps();+ if (timestamps == null) {+ return;+ }+ int clearCount = 0;+ int index = Collections.binarySearch(timestamps, now);+ if (index >= 0) {+ clearCount = index + 1;+ } else {+ clearCount = index * (-1) -1;+ }+ if (clearCount > 0) {+ if (clearCount == timestamps.size()) {+ timestamps.clear();+ } else {+ try {+ timestamps.subList(0, clearCount).clear();+ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ }+ }+ });+ }++ private List<DatapointBucket> getHistoryDataFromDb(AttributeRef attributeRef, ForecastConfigurationWeightedExponentialAverage config, long offset) {+ List<DatapointBucket> datapointBuckets = new ArrayList<>(config.getPastCount());++ StringBuilder sb = new StringBuilder();+ sb.append(+ "select dp from " + AssetDatapoint.class.getSimpleName() + " dp " ++ "where dp.assetId = :assetId " ++ "and dp.attributeName = :attributeName "+ );+ for (int i = 1; i <= config.getPastCount(); i++) {+ sb.append(i == 1 ? "and (" : " or " );+ sb.append("(dp.timestamp >= :timestampMin" + i + " and dp.timestamp <= :timestampMax" + i + ")");+ if (i == config.getPastCount()) {+ sb.append(") ");+ }+ }+ sb.append("order by dp.timestamp asc");++ List<AssetDatapoint> datapoints = persistenceService.doReturningTransaction(entityManager -> {+ TypedQuery<AssetDatapoint> query = entityManager.createQuery(sb.toString(), AssetDatapoint.class)+ .setParameter("assetId", attributeRef.getId())+ .setParameter("attributeName", attributeRef.getName());+ long now = timerService.getCurrentTimeMillis();+ long pastPeriod = config.getPastPeriod().toMillis();+ long forecastPeriod = config.getForecastPeriod().toMillis();+ long totalForecastPeriod = Math.min(forecastPeriod * config.getForecastCount(), pastPeriod);++ for (int i = config.getPastCount(); i >= 1; i--) {+ long timestampMin = now - (pastPeriod * i) + offset;+ long timestampMax = now - (pastPeriod * i) + totalForecastPeriod + offset;++ datapointBuckets.add(new DatapointBucket(timestampMin, timestampMax));++ query.setParameter("timestampMin" + i, new Date(timestampMin));+ query.setParameter("timestampMax" + i, new Date(timestampMax));+ }+ return query.getResultList();+ });++ datapoints.forEach(datapoint -> {+ datapointBuckets.stream()+ .filter(bucket -> bucket.isInTimeRange(datapoint.getTimestamp()))+ .findFirst()+ .ifPresent(bucket -> bucket.add(datapoint));+ });++ return datapointBuckets;+ }++ private List<Long> loadForecastTimestampsFromDb(AttributeRef attributeRef, long now) {+ List<AssetPredictedDatapoint> datapoints = assetPredictedDatapointService.getDatapoints(attributeRef);+ List<Long> timestamps = datapoints+ .stream()+ .map(datapoint -> datapoint.getTimestamp())+ .filter(timestamp -> timestamp >= now)+ .sorted()+ .collect(Collectors.toList());;+ return timestamps;+ }+ }++ public static class ForecastAttribute {++ private String assetId;+ private AttributeRef attributeRef;+ private Attribute<?> attribute;+ private ForecastConfiguration config;+ private List<Long> forecastTimestamps = new ArrayList<>();++ public ForecastAttribute(Asset<?> asset, Attribute<?> attribute) {+ this(asset.getId(), attribute);+ }++ public ForecastAttribute(String assetId, Attribute<?> attribute) {+ requireNonNullAndNonEmpty(assetId);+ if (attribute == null) {+ throw new IllegalArgumentException("Attribute cannot be null");+ }+ this.assetId = assetId;+ this.attribute = attribute;+ this.attributeRef = new AttributeRef(assetId, attribute.getName());+ this.config = attribute.getMetaValue(FORECAST).orElse(null);+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (o == null || getClass() != o.getClass()) return false;+ ForecastAttribute that = (ForecastAttribute) o;+ return assetId.equals(that.assetId) && attribute.getName().equals(that.attribute.getName());+ }++ @Override+ public int hashCode() {+ int result = assetId.hashCode();+ result = 31 * result + attribute.getName().hashCode();+ return result;+ }++ public String getId() {+ return assetId;+ }++ public String getName() {+ return attribute.getName();+ }++ public AttributeRef getAttributeRef() {+ return attributeRef;+ }++ public Attribute<?> getAttribute() {+ return attribute;+ }++ public ForecastConfiguration getConfig() {+ return config;+ }++ public boolean isValidConfig() {+ return (config != null && config.isValid());+ }++ public void setForecastTimestamps(List<Long> timestamps) {+ this.forecastTimestamps = timestamps;+ }++ public List<Long> getForecastTimestamps() {+ return forecastTimestamps;+ }+ }++ private static class DatapointBucket {+ private long begin;+ private long end;+ private List<AssetDatapoint> datapoints = new ArrayList<>();++ public DatapointBucket(long begin, long end) {+ this.begin = begin;+ this.end = end;+ }++ public long getBegin() {+ return begin;+ }++ public long getEnd() {+ return end;+ }++ public List<AssetDatapoint> getDatapoints() {+ return datapoints;+ }++ public boolean isInTimeRange(long timestamp) {+ return timestamp >= begin && timestamp <= end;+ }++ public void add(AssetDatapoint datapoint) {+ datapoints.add(datapoint);+ }+ }+}diff --git a/manager/src/main/java/org/openremote/manager/setup/ManagerSetup.java b/manager/src/main/java/org/openremote/manager/setup/ManagerSetup.javaindex 1bf44f046..ea231572e 100644--- a/manager/src/main/java/org/openremote/manager/setup/ManagerSetup.java+++ b/manager/src/main/java/org/openremote/manager/setup/ManagerSetup.java@@ -19,6 +19,7 @@*/package org.openremote.manager.setup;+import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetProcessingService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.datapoint.AssetDatapointService;@@ -61,6 +62,7 @@ public class ManagerSetup implements Setup {protected static final Logger LOG = LoggerFactory.getLogger(ManagerSetup.class);protected Path provisionDocRoot;+ final protected TimerService timerService;final protected ScheduledExecutorService executorService;final protected ManagerPersistenceService persistenceService;final protected ManagerIdentityService identityService;@@ -73,6 +75,7 @@ public class ManagerSetup implements Setup {final protected MetaItem<?>[] EMPTY_META = new MetaItem<?>[0];public ManagerSetup(Container container) {+ this.timerService = container.getService(TimerService.class);this.executorService = container.getExecutorService();this.persistenceService = container.getService(ManagerPersistenceService.class);this.identityService = container.getService(ManagerIdentityService.class);diff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerServiceindex f4a2314f3..84959e501 100644--- a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService@@ -33,3 +33,4 @@ org.openremote.manager.provisioning.ProvisioningServiceorg.openremote.manager.dashboard.DashboardStorageServiceorg.openremote.manager.security.AuthorisationServiceorg.openremote.manager.app.ConfigurationService+org.openremote.manager.asset.ForecastServicediff --git a/model/src/main/java/org/openremote/model/value/ForecastConfiguration.java b/model/src/main/java/org/openremote/model/value/ForecastConfiguration.javanew file mode 100644index 000000000..0144947c2--- /dev/null+++ b/model/src/main/java/org/openremote/model/value/ForecastConfiguration.java@@ -0,0 +1,51 @@+/*+ * Copyright 2023, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.model.value;++import com.fasterxml.jackson.annotation.JsonProperty;+import com.fasterxml.jackson.annotation.JsonSubTypes;+import com.fasterxml.jackson.annotation.JsonTypeInfo;++import java.io.Serializable;++@JsonTypeInfo(+ use = JsonTypeInfo.Id.NAME,+ property = "type",+ include = JsonTypeInfo.As.EXISTING_PROPERTY+)+@JsonSubTypes({+ @JsonSubTypes.Type(ForecastConfigurationWeightedExponentialAverage.class)+})++public abstract class ForecastConfiguration implements Serializable {++ @JsonProperty+ protected String type;++ protected ForecastConfiguration(String type) {+ this.type = type;+ }++ public String getType() {+ return type;+ }++ public abstract boolean isValid();+}diff --git a/model/src/main/java/org/openremote/model/value/ForecastConfigurationWeightedExponentialAverage.java b/model/src/main/java/org/openremote/model/value/ForecastConfigurationWeightedExponentialAverage.javanew file mode 100644index 000000000..888e9311c--- /dev/null+++ b/model/src/main/java/org/openremote/model/value/ForecastConfigurationWeightedExponentialAverage.java@@ -0,0 +1,128 @@+/*+ * Copyright 2023, OpenRemote Inc.+ *+ * See the CONTRIBUTORS.txt file in the distribution for a+ * full listing of individual contributors.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <http://www.gnu.org/licenses/>.+ */+package org.openremote.model.value;++import com.fasterxml.jackson.annotation.JsonCreator;+import com.fasterxml.jackson.annotation.JsonProperty;+import com.fasterxml.jackson.annotation.JsonTypeName;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;+import com.fasterxml.jackson.databind.annotation.JsonSerialize;+import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;+import com.fasterxml.jackson.databind.util.StdConverter;+import org.openremote.model.Constants;+import org.openremote.model.value.impl.PeriodAndDuration;++import jakarta.validation.constraints.NotNull;+import jakarta.validation.constraints.Positive;+import java.time.Duration;+import java.time.Period;+import java.time.temporal.ChronoUnit;+import java.util.regex.Pattern;++import static org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage.TYPE;++@JsonTypeName(TYPE)+public class ForecastConfigurationWeightedExponentialAverage extends ForecastConfiguration {++ public static final String TYPE = "wea";++ protected static final Pattern iso8601Pattern = Pattern.compile(Constants.ISO8601_DURATION_REGEXP);++ @NotNull+ @JsonSerialize(using = ToStringSerializer.class)+ @JsonDeserialize(converter = PeriodAndDurationConverter.class)+ protected ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration pastPeriod;+ @NotNull+ @Positive+ protected Integer pastCount;+ @NotNull+ @JsonSerialize(using = ToStringSerializer.class)+ @JsonDeserialize(converter = PeriodAndDurationConverter.class)+ protected ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration forecastPeriod;+ @NotNull+ @Positive+ protected Integer forecastCount;++ @JsonCreator+ public ForecastConfigurationWeightedExponentialAverage(@JsonProperty("pastPeriod") ExtendedPeriodAndDuration pastPeriod, @JsonProperty("pastCount") Integer pastCount, @JsonProperty("forecastPeriod") ExtendedPeriodAndDuration forecastPeriod, @JsonProperty("forecastCount") Integer forecastCount) {+ super(TYPE);+ this.pastPeriod = pastPeriod;+ this.pastCount = pastCount;+ this.forecastPeriod = forecastPeriod;+ this.forecastCount = forecastCount;+ }++ @Override+ public boolean isValid() {+ return pastCount != null && pastCount > 0 &&+ forecastCount != null && forecastCount > 0 &&+ pastPeriod != null && pastPeriod.toMillis() > 0 &&+ forecastPeriod != null && forecastPeriod.toMillis() > 0;+ }++ public ExtendedPeriodAndDuration getPastPeriod() {+ return pastPeriod;+ }++ public Integer getPastCount() {+ return pastCount;+ }++ public ExtendedPeriodAndDuration getForecastPeriod() {+ return forecastPeriod;+ }++ public Integer getForecastCount() {+ return forecastCount;+ }++ public static class ExtendedPeriodAndDuration extends PeriodAndDuration {+ public ExtendedPeriodAndDuration(String str) {+ super(str);+ if (!iso8601Pattern.matcher(str).matches()) {+ throw new IllegalArgumentException("Invalid ISO 8601 format");+ }+ }++ public long toMillis() {+ Period period = (getPeriod() != null ? getPeriod() : Period.ZERO);+ Duration duration = (getDuration() != null ? getDuration() : Duration.ZERO);+ return duration.plus(durationFromPeriod(period)).toMillis();+ }++ private Duration durationFromPeriod(Period period) {+ Duration years = ChronoUnit.YEARS.getDuration().multipliedBy(period.getYears());+ Duration months = ChronoUnit.MONTHS.getDuration().multipliedBy(period.getMonths());+ Duration days = ChronoUnit.DAYS.getDuration().multipliedBy(period.getDays());+ return years.plus(months).plus(days);+ }+ }++ public static class PeriodAndDurationConverter extends StdConverter<JsonNode, ExtendedPeriodAndDuration> {+ @Override+ public ExtendedPeriodAndDuration convert(JsonNode value) {+ if (value.isTextual()) {+ return new ExtendedPeriodAndDuration(value.asText());+ }+ return null;+ }+ }+}diff --git a/model/src/main/java/org/openremote/model/value/MetaItemType.java b/model/src/main/java/org/openremote/model/value/MetaItemType.javaindex 08180c07e..9ba0e7a23 100644--- a/model/src/main/java/org/openremote/model/value/MetaItemType.java+++ b/model/src/main/java/org/openremote/model/value/MetaItemType.java@@ -98,6 +98,10 @@ public final class MetaItemType {// TODO: Re-evaluate this can this info be retrieved automatically using prediction servicepublic static final MetaItemDescriptor<Boolean> HAS_PREDICTED_DATA_POINTS = new MetaItemDescriptor<>("hasPredictedDataPoints", ValueType.BOOLEAN);+ /**+ * The forecast service calculates predicted data points based on the forecast configuration.+ */+ public static final MetaItemDescriptor<ForecastConfiguration> FORECAST = new MetaItemDescriptor<>("forecast", ValueType.FORECAST_CONFIGURATION);/* RULE META */diff --git a/model/src/main/java/org/openremote/model/value/ValueType.java b/model/src/main/java/org/openremote/model/value/ValueType.javaindex 9b48a08db..292fa529c 100644--- a/model/src/main/java/org/openremote/model/value/ValueType.java+++ b/model/src/main/java/org/openremote/model/value/ValueType.java@@ -19,12 +19,9 @@*/package org.openremote.model.value;-import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.BaseJsonNode;import com.fasterxml.jackson.databind.node.ObjectNode;-import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject;-import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaString;import org.openremote.model.Constants;import org.openremote.model.asset.agent.AgentLink;import org.openremote.model.asset.agent.ConnectionStatus;@@ -208,6 +205,8 @@ public final class ValueType {public static final ValueDescriptor<AssetQuery> ASSET_QUERY = new ValueDescriptor<>("assetQuery", AssetQuery.class);+ public static final ValueDescriptor<ForecastConfiguration> FORECAST_CONFIGURATION = new ValueDescriptor<>("forecastConfiguration", ForecastConfiguration.class);+protected ValueType() {}}diff --git a/model/src/main/java/org/openremote/model/value/impl/PeriodAndDuration.java b/model/src/main/java/org/openremote/model/value/impl/PeriodAndDuration.javaindex a56e89984..4f3a2f813 100644--- a/model/src/main/java/org/openremote/model/value/impl/PeriodAndDuration.java+++ b/model/src/main/java/org/openremote/model/value/impl/PeriodAndDuration.java@@ -46,7 +46,7 @@ public class PeriodAndDuration implements Serializable {String[] periodAndDuration = str.split("T");if (periodAndDuration.length == 2 && periodAndDuration[1].length() > 0) {- duration = Duration.parse("P" + periodAndDuration[1]);+ duration = Duration.parse("PT" + periodAndDuration[1]);}if (periodAndDuration[0].length() > 1) {period = Period.parse(periodAndDuration[0]);diff --git a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.javaindex dfcee03f1..b007fc042 100644--- a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java+++ b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import org.openremote.agent.protocol.simulator.SimulatorAgent;import org.openremote.agent.protocol.simulator.SimulatorAgentLink;import org.openremote.container.util.UniqueIdentifierGenerator;+import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.setup.ManagerSetup;import org.openremote.model.Constants;import org.openremote.model.Container;@@ -35,13 +36,22 @@ import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.MetaItem;import org.openremote.model.geo.GeoJSONPoint;import org.openremote.model.security.Realm;+import org.openremote.model.util.Pair;import org.openremote.model.util.ValueUtil;+import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage;import org.openremote.model.value.ValueConstraint;import org.openremote.model.value.ValueType;import org.openremote.model.value.impl.ColourRGB;+import java.time.Duration;+import java.time.Instant;+import java.time.LocalDateTime;+import java.time.ZoneId;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;import java.util.Arrays;import java.util.List;+import java.util.stream.Collectors;import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT;import static org.openremote.model.Constants.*;@@ -101,6 +111,16 @@ public class ManagerTestSetup extends ManagerSetup {public String electricityBatteryAssetId;public String light1Id;public String light2Id;+ public String forecastThing1Id;+ public String forecastThing2Id;+ public String forecastAttributeName = "forecastAttribute";+ public ForecastConfigurationWeightedExponentialAverage forecastConfig = new ForecastConfigurationWeightedExponentialAverage(+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("P7D"),+ 3,+ new ForecastConfigurationWeightedExponentialAverage.ExtendedPeriodAndDuration("PT1H"),+ 4+ );+ public List<List<Double>> forecastTestHistoryData = new ArrayList<>();public ManagerTestSetup(Container container) {super(container);@@ -288,6 +308,25 @@ public class ManagerTestSetup extends ManagerSetup {electricitySupplierAssetId = electricitySupplierAsset.getId();+ forecastTestHistoryData.add(Arrays.asList(new Double[] {4.0, 3.0, 2.0}));+ forecastTestHistoryData.add(Arrays.asList(new Double[] {4.1, 3.1, 2.1}));+ forecastTestHistoryData.add(Arrays.asList(new Double[] {4.2, 3.2, 2.2}));+ forecastTestHistoryData.add(Arrays.asList(new Double[] {4.3, 3.3, 2.3}));+ forecastTestHistoryData.add(Arrays.asList(new Double[] {4.4, 3.4, 2.4}));++ forecastThing1Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);+ forecastThing2Id = addForecastTestAsset(assetStorageService, realmEnergyName, forecastAttributeName, forecastConfig);++ Long offset = ((forecastConfig.getForecastPeriod().toMillis() * 2) / 3) * (-1);+ insertHistoryTestDataToDb(new AttributeRef(forecastThing1Id, forecastAttributeName), forecastTestHistoryData, 0, forecastConfig);+ insertHistoryTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestHistoryData, offset ,forecastConfig);++ List<Pair<Double, Long>> forecastTestData = new ArrayList<>();+ forecastTestData.add(new Pair<>(1.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() + offset));+ forecastTestData.add(new Pair<>(2.0, timerService.getCurrentTimeMillis() + forecastConfig.getForecastPeriod().toMillis() * 2 + offset));++ insertForecastTestDataToDb(new AttributeRef(forecastThing2Id, forecastAttributeName), forecastTestData);+// ################################ Assets for 'building' realm ###################################@@ -692,4 +731,59 @@ public class ManagerTestSetup extends ManagerSetup {LightAsset lightController_3Asset = createDemoLightControllerAsset("LightController 3", assetArea3, new GeoJSONPoint(5.487478, 51.446979));lightController_3Asset = assetStorageService.merge(lightController_3Asset);}++ public void insertHistoryTestDataToDb(AttributeRef attributeRef, List<List<Double>> data, long offset, ForecastConfigurationWeightedExponentialAverage config) {+ for (int pastPeriodIndex = config.getPastCount(); pastPeriodIndex > 0; pastPeriodIndex--) {+ List<Double> values = new ArrayList<>(data.size());+ for (int forecastPeriodIndex = 0; forecastPeriodIndex < data.size(); forecastPeriodIndex++) {+ values.add(data.get(forecastPeriodIndex).get(config.getPastCount() - pastPeriodIndex));+ }+ insertHistoryTestDataToDb(attributeRef, pastPeriodIndex, values, offset, config);+ }+ }++ public void insertHistoryTestDataToDb(AttributeRef attributeRef, int pastPeriodIndex, List<Double> values, long offset, ForecastConfigurationWeightedExponentialAverage config) {+ LocalDateTime now = LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis()), ZoneId.systemDefault());+ long nowInMillis = now.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();+ List<Pair<?, LocalDateTime>> history = new ArrayList<>();+ for (int forecastPeriodIndex = 0; forecastPeriodIndex < values.size(); forecastPeriodIndex++) {+ long timespan = config.getPastPeriod().toMillis() * pastPeriodIndex -+ config.getForecastPeriod().toMillis() * (forecastPeriodIndex + 1) ++ Duration.ofMinutes(2).toMillis() - offset;+ history.add(+ new Pair<>(+ Double.valueOf(values.get(forecastPeriodIndex)),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(timerService.getCurrentTimeMillis() - timespan), ZoneId.systemDefault())+ )+ );+ }+ assetDatapointService.upsertValues(attributeRef.getId(), attributeRef.getName(), history);+ }++ @SuppressWarnings("unchecked")+ public void insertForecastTestDataToDb(AttributeRef attributeRef, List<Pair<Double, Long>> data) {+ List<Pair<Double, LocalDateTime>> valuesAndTimestamps = data+ .stream()+ .map(p -> new Pair<Double,LocalDateTime>(+ p.getKey(),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(p.getValue()), ZoneId.systemDefault())))+ .collect(Collectors.toList());+ assetPredictedDatapointService.updateValues(attributeRef.getId(), attributeRef.getName(), (List)valuesAndTimestamps);+ }++ public String addForecastTestAsset(AssetStorageService assetStorageService, String realm, String attributeName, ForecastConfigurationWeightedExponentialAverage config) {+ Asset<?> thing = new ThingAsset("Forecast Test Thing");+ thing.setRealm(realm);+ thing.getAttributes().addOrReplace(+ new Attribute<>(attributeName, NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ config+ )+ )+ );+ thing = assetStorageService.merge(thing);+ return thing.getId();+ }}diff --git a/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovynew file mode 100644index 000000000..7545e9442--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/assets/ForecastServiceTest.groovy@@ -0,0 +1,259 @@+package org.openremote.test.assets++import org.openremote.container.timer.TimerService+import org.openremote.manager.asset.AssetStorageService+import org.openremote.manager.asset.ForecastService+import org.openremote.manager.datapoint.AssetDatapointService+import org.openremote.manager.datapoint.AssetPredictedDatapointService+import org.openremote.manager.setup.SetupService+import org.openremote.model.attribute.Attribute+import org.openremote.model.attribute.AttributeRef+import org.openremote.model.attribute.MetaItem+import org.openremote.model.datapoint.AssetPredictedDatapoint+import org.openremote.model.value.ForecastConfigurationWeightedExponentialAverage+import org.openremote.setup.integration.ManagerTestSetup+import org.openremote.test.ManagerContainerTrait+import spock.lang.Ignore+import spock.lang.Shared+import spock.lang.Specification+import spock.util.concurrent.PollingConditions++import java.time.Duration++import static java.util.concurrent.TimeUnit.MILLISECONDS+import static org.openremote.model.value.MetaItemType.FORECAST+import static org.openremote.model.value.ValueType.NUMBER+import static spock.util.matcher.HamcrestMatchers.closeTo++class ForecastServiceTest extends Specification implements ManagerContainerTrait {++ @Shared+ static ManagerTestSetup managerTestSetup+ @Shared+ static AssetPredictedDatapointService assetPredictedDatapointService+ @Shared+ static AssetStorageService assetStorageService+ @Shared+ static AssetDatapointService assetDatapointService+ @Shared+ static ForecastService forecastService+ @Shared+ static TimerService timerService++ def setupSpec() {+ given: "the server container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)+ assetPredictedDatapointService = container.getService(AssetPredictedDatapointService.class)+ assetStorageService = container.getService(AssetStorageService.class)+ assetDatapointService = container.getService(AssetDatapointService.class)+ forecastService = container.getService(ForecastService.class)+ timerService = container.getService(TimerService.class)+ }++ @Ignore+ def "Test forecast calculation after server restart - no initial forecast data"() {+ given: "attribute with forecast configuration"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def attributeRef = new AttributeRef(managerTestSetup.forecastThing1Id, managerTestSetup.forecastAttributeName)+ def config = managerTestSetup.forecastConfig++ and:+ stopPseudoClock()++ expect: "test datapoints have been added to the database"+ conditions.eventually {+ def count = assetDatapointService.getDatapointsCount(attributeRef)+ assert count == config.pastCount * managerTestSetup.forecastTestHistoryData.size()+ }++ and: "asset attribute has been registered at the forecast service"+ conditions.eventually {+ assert forecastService.forecastTaskManager.containsAttribute(attributeRef)+ }++ and: "initial forecast values have been calculated"+ conditions.eventually {+ List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService+ .getDatapoints(attributeRef)+ .sort {it.timestamp}+ assert predictedDatapoints.size() == config.forecastCount+ for (int i = 0; i < predictedDatapoints.size(); i++) {+ assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()+ assert predictedDatapoints[i].timestamp, closeTo(timerService.currentTimeMillis + (i + 1) * config.forecastPeriod.toMillis(), Duration.ofSeconds(5).toMillis())+ }+ }+ }++ @Ignore+ def "Test forecast calculation after server restart - forecast data partially available"() {+ given: "attribute with forecast configuration"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def attributeRef = new AttributeRef(managerTestSetup.forecastThing2Id, managerTestSetup.forecastAttributeName)+ def config = managerTestSetup.forecastConfig++ and:+ stopPseudoClock()++ expect: "test datapoints have been added to the database"+ conditions.eventually {+ def count = assetDatapointService.getDatapointsCount(attributeRef)+ assert count == config.pastCount * managerTestSetup.forecastTestHistoryData.size()+ }++ and: "asset attribute has been registered at the forecast service"+ conditions.eventually {+ assert forecastService.forecastTaskManager.containsAttribute(attributeRef)+ }++ and: "partially existing forecast values should have have been completed and updated"+ conditions.eventually {+ List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService+ .getDatapoints(attributeRef)+ .sort {it.timestamp}+ assert predictedDatapoints.size() == config.forecastCount+ Long offset = ((config.forecastPeriod.toMillis() * 2) / 3) * (-1)+ for (int i = 0; i < predictedDatapoints.size(); i++) {+ assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()+ assert predictedDatapoints[i].timestamp, closeTo(timerService.currentTimeMillis + (i + 1) * config.forecastPeriod.toMillis() + offset, Duration.ofSeconds(10).toMillis())+ }+ }+ }++ def "Test adding updating and deleting an asset"() {+ given: "no assets with forecast configuration"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def config = managerTestSetup.forecastConfig+ def attributeName = "forecastTestAttribute"++ and:+ stopPseudoClock()++ when: "asset is added"+ String assetId = managerTestSetup.addForecastTestAsset(+ assetStorageService, managerTestSetup.realmEnergyName, attributeName, config+ )+ AttributeRef attributeRef = new AttributeRef(assetId, attributeName)++ then: "attribute should be registered at the forecast service"+ conditions.eventually {+ assert forecastService.forecastTaskManager.containsAttribute(attributeRef)+ }++ when: "test data is inserted to the database"+ forecastService.forecastTaskManager.stop(3000)+ managerTestSetup.insertHistoryTestDataToDb(+ attributeRef, managerTestSetup.forecastTestHistoryData, config.forecastPeriod.toMillis(), config+ )++ and: "time has passed"+ advancePseudoClock(+ config.forecastPeriod.toMillis(), MILLISECONDS, container+ )++ and: "forecast service is restarted"+ forecastService.forecastTaskManager.start(timerService.currentTimeMillis)++ then: "initial forecast values should be calculated"+ conditions.eventually {+ List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService+ .getDatapoints(attributeRef)+ .sort {it.timestamp}+ assert predictedDatapoints.size() == config.forecastCount+ for (int i = 0; i < predictedDatapoints.size(); i++) {+ assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()+ assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + (i + 1) * config.forecastPeriod.toMillis()+ }+ }++ when: "time has passed"+ forecastService.forecastTaskManager.stop(3000)+ advancePseudoClock(+ config.forecastPeriod.toMillis(), MILLISECONDS, container+ )+ forecastService.forecastTaskManager.start(timerService.currentTimeMillis)++ then: "forecast values should be updated"+ conditions.eventually {+ List<AssetPredictedDatapoint> predictedDatapoints = assetPredictedDatapointService+ .getDatapoints(attributeRef)+ .sort {it.timestamp}+ assert predictedDatapoints.size() == config.forecastCount+ for (int i = 1; i < predictedDatapoints.size(); i++) {+ assert predictedDatapoints[i].value == calculateForecast(managerTestSetup.forecastTestHistoryData[i][0..config.pastCount - 1]).get()+ assert predictedDatapoints[i].timestamp == timerService.currentTimeMillis + i * config.forecastPeriod.toMillis()+ }+ }++ when: "new forecast attribute is added"+ def asset = assetStorageService.find(assetId)+ asset.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ config+ )+ )+ )+ assetStorageService.merge(asset)+ def newAttributeRef = new AttributeRef(assetId, "newForecastAttribute")++ then: "new attribute should be registered at the forecast service"+ conditions.eventually {+ assert forecastService.forecastTaskManager.containsAttribute(newAttributeRef)+ }++ when: "forecast configuration is updated"+ asset = assetStorageService.find(assetId)+ def updatedPastCount = config.pastCount + 1+ def updatedConfig = new ForecastConfigurationWeightedExponentialAverage(+ config.pastPeriod, updatedPastCount, config.forecastPeriod, config.forecastCount+ )+ asset.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER)+ .addOrReplaceMeta(+ new MetaItem<>(+ FORECAST,+ updatedConfig+ )+ )+ )+ assetStorageService.merge(asset)++ then: "forecast service should contain updated forecast configuration"+ conditions.eventually {+ ForecastService.ForecastAttribute forecastAttribute = forecastService.forecastTaskManager.getAttribute(newAttributeRef)+ assert forecastAttribute != null+ assert updatedPastCount == forecastAttribute.config.pastCount+ }++ when: "forecast configuration is deleted"+ asset = assetStorageService.find(assetId)+ asset.getAttributes().addOrReplace(+ new Attribute<>("newForecastAttribute", NUMBER)+ )+ assetStorageService.merge(asset)++ then: "attribute should be unregistered from forecast service"+ conditions.eventually {+ assert !forecastService.forecastTaskManager.containsAttribute(newAttributeRef)+ }++ when: "asset is deleted"+ assetStorageService.delete([assetId])++ then: "attribute should be unregistered from forecast service"+ conditions.eventually {+ assert !forecastService.forecastTaskManager.containsAttribute(attributeRef)+ }+ }++ private Optional<Double> calculateForecast(List<Double> values) {+ double R = values.size();+ double a = 2 / (R + 1);+ return values.stream().reduce((olderValue, oldValue) -> {+ return (oldValue * a) + (olderValue * (1 - a));+ });+ }+}