100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Overhaul of asset datapoint services and energy optimisation updates
Rich Turner205c4919d122
diff --git a/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java b/container/src/main/java/org/openremote/container/web/WebServiceExceptions.javaindex 83ce0747c..1caa92323 100644--- a/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java+++ b/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java@@ -222,6 +222,12 @@ public class WebServiceExceptions {if ("java.io.IOException: Connection reset by peer".equals(getRootCause(throwable).toString()))return;+ if (throwable instanceof WebApplicationException && ((WebApplicationException) throwable).getResponse().getStatus() == 404) {+ // Don't stack trace 404s just want request uri+ LOG.log(Level.FINE, "Web service exception (404) in '" + origin + "' for '" + info + "'");+ return;+ }+if (LOG.isLoggable(Level.FINE)) {LOG.log(Level.FINE, "Web service exception in '" + origin + "' for '" + info + "'", throwable);} else {diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.javaindex 352a0f68e..ab79f7d97 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java@@ -1708,7 +1708,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicStringBuilder attributeBuilder = new StringBuilder();- if (nameValuePredicate.negated) {+ // Only append not on the outer filter if value predicate is not set otherwise it will match any attribute with different name and value+ if (nameValuePredicate.negated && (nameValuePredicate.value == null || nameValuePredicate.name == null)) {attributeBuilder.append("NOT (");}@@ -1729,6 +1730,10 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicif (nameValuePredicate.name != null) {attributeBuilder.append(" and ");++ if (nameValuePredicate.negated) {+ attributeBuilder.append(" NOT(");+ }}// Inserts the SQL string and adds the parametersdiff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javanew file mode 100644index 000000000..ea1baa169--- /dev/null+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -0,0 +1,384 @@+package org.openremote.manager.datapoint;++import org.hibernate.Session;+import org.hibernate.jdbc.AbstractReturningWork;+import org.openremote.model.Container;+import org.openremote.model.ContainerService;+import org.openremote.container.persistence.PersistenceService;+import org.openremote.container.timer.TimerService;+import org.openremote.manager.asset.AssetProcessingException;+import org.openremote.manager.asset.AssetStorageService;+import org.openremote.manager.asset.AssetUpdateProcessor;+import org.openremote.manager.security.ManagerIdentityService;+import org.openremote.manager.web.ManagerWebService;+import org.openremote.model.asset.Asset;+import org.openremote.model.attribute.Attribute;+import org.openremote.model.attribute.AttributeEvent.Source;+import org.openremote.model.attribute.AttributeRef;+import org.openremote.model.attribute.AttributeWriteFailure;+import org.openremote.model.datapoint.AssetDatapoint;+import org.openremote.model.datapoint.Datapoint;+import org.openremote.model.datapoint.DatapointInterval;+import org.openremote.model.datapoint.ValueDatapoint;+import org.openremote.model.query.AssetQuery;+import org.openremote.model.query.filter.AttributePredicate;+import org.openremote.model.query.filter.NameValuePredicate;+import org.openremote.model.util.Pair;+import org.openremote.model.value.MetaItemType;+import org.openremote.model.value.Values;+import org.postgresql.util.PGInterval;+import org.postgresql.util.PGobject;++import javax.persistence.EntityManager;+import javax.persistence.TypedQuery;+import java.sql.*;+import java.time.Duration;+import java.time.Instant;+import java.time.LocalDateTime;+import java.time.temporal.ChronoUnit;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;+import java.util.logging.Level;+import java.util.logging.Logger;+import java.util.stream.Collectors;++import static java.time.temporal.ChronoUnit.DAYS;+import static java.util.stream.Collectors.groupingBy;+import static java.util.stream.Collectors.toList;+import static org.openremote.container.util.MapAccess.getInteger;+import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;++/**+ * Base class for all classes that store and retrieve {@link org.openremote.model.datapoint.Datapoint}.+ */+public abstract class AbstractDatapointService<T extends Datapoint> implements ContainerService {++ public static final int PRIORITY = AssetStorageService.PRIORITY + 100;+ protected PersistenceService persistenceService;+ protected AssetStorageService assetStorageService;+ protected TimerService timerService;+ protected ScheduledExecutorService executorService;+ protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;++ @Override+ public int getPriority() {+ return PRIORITY;+ }++ @Override+ public void init(Container container) throws Exception {+ persistenceService = container.getService(PersistenceService.class);+ assetStorageService = container.getService(AssetStorageService.class);+ timerService = container.getService(TimerService.class);+ executorService = container.getExecutorService();+ }++ @Override+ public void stop(Container container) throws Exception {+ if (dataPointsPurgeScheduledFuture != null) {+ dataPointsPurgeScheduledFuture.cancel(true);+ }+ }++ public void upsertValue(String assetId, String attributeName, Object value, LocalDateTime timestamp) throws IllegalStateException {+ persistenceService.doTransaction(em ->+ em.unwrap(Session.class).doWork(connection -> {+ PreparedStatement st;++ try {+ st = getUpsertPreparedStatement(connection);+ setUpsertValues(st, assetId, attributeName, value, timestamp);+ st.executeUpdate();+ } catch (Exception e) {+ String msg = "Failed to insert/update data point: ";+ getLogger().log(Level.WARNING, msg, e);+ throw new IllegalStateException(msg, e);+ }+ }));+ }++ public void upsertValues(String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) throws IllegalStateException {+ persistenceService.doTransaction(em ->+ em.unwrap(Session.class).doWork(connection -> {++ getLogger().finest("Storing datapoints for: id=" + assetId + ", name=" + attributeName + ", count=" + valuesAndTimestamps.size());+ PreparedStatement st;++ try {+ st = getUpsertPreparedStatement(connection);++ for (Pair<?, LocalDateTime> valueAndTimestamp : valuesAndTimestamps) {+ setUpsertValues(st, assetId, attributeName, valueAndTimestamp.key, valueAndTimestamp.value);+ st.addBatch();+ }+ st.executeBatch();+ } catch (Exception e) {+ String msg = "Failed to insert/update data points: " + assetId + ", name=" + attributeName + ", count=" + valuesAndTimestamps.size();+ getLogger().log(Level.WARNING, msg, e);+ throw new IllegalStateException(msg, e);+ }+ }));+ }++ public List<T> getDatapoints(AttributeRef attributeRef) {+ return persistenceService.doReturningTransaction(entityManager ->+ entityManager.createQuery(+ "select dp from " + getDatapointClass().getSimpleName() + " dp " ++ "where dp.assetId = :assetId " ++ "and dp.attributeName = :attributeName " ++ "order by dp.timestamp desc",+ getDatapointClass())+ .setParameter("assetId", attributeRef.getId())+ .setParameter("attributeName", attributeRef.getName())+ .getResultList());+ }++ public long getDatapointsCount() {+ return getDatapointsCount(null);+ }++ public long getDatapointsCount(AttributeRef attributeRef) {+ return persistenceService.doReturningTransaction(entityManager -> {++ String queryStr = attributeRef == null ?+ "select count(dp) from " + getDatapointClass().getSimpleName() + " dp" :+ "select count(dp) from " + getDatapointClass().getSimpleName() + " dp where dp.assetId = :assetId and dp.attributeName = :attributeName";++ TypedQuery<Long> query = entityManager.createQuery(+ queryStr,+ Long.class);++ if (attributeRef != null) {+ query+ .setParameter("assetId", attributeRef.getId())+ .setParameter("attributeName", attributeRef.getName());+ }++ return query.getSingleResult();+ });+ }++ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,+ DatapointInterval datapointInterval,+ Integer stepSize,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp) {++ Asset<?> asset = assetStorageService.find(attributeRef.getId());+ if (asset == null) {+ throw new IllegalStateException("Asset not found: " + attributeRef.getId());+ }++ Attribute<?> assetAttribute = asset.getAttribute(attributeRef.getName())+ .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeRef.getName()));++ return getValueDatapoints(asset.getId(), assetAttribute, datapointInterval, stepSize, fromTimestamp, toTimestamp);+ }++ public ValueDatapoint<?>[] getValueDatapoints(String assetId,+ Attribute<?> attribute,+ DatapointInterval datapointInterval,+ final Integer stepSize,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp) {++ AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());++ getLogger().finer("Getting datapoints for: " + attributeRef);++ return persistenceService.doReturningTransaction(entityManager ->++ entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {+ @Override+ public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {++ Class<?> attributeType = attribute.getType().getType();+ boolean isNumber = Number.class.isAssignableFrom(attributeType);+ boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);+ StringBuilder query = new StringBuilder();+ boolean downsample = isNumber || isBoolean;++ String truncate = null;+ String part = null;+ String interval = null;+ String stepStr = null;+ String partQuery = "date_part(?, ?)::int";+ String partQuery2 = "date_part(?, TIMESTAMP)::int";+ int step = 1;++ if (downsample) {++ switch (datapointInterval) {++ case MINUTE:+ // This works with minutes of the day so not constrained to step size < 60+ step = stepSize == null ? 1 : Math.max(1, Math.min(1440, stepSize));+ truncate = "day";+ part = "min";+ interval = "min";+ partQuery = "(date_part('hour', ?)::int * 60 + date_part(?, ?)::int)";+ partQuery2 = "(date_part('hour', TIMESTAMP)::int * 60 + date_part(?, TIMESTAMP)::int)";+ break;+ case HOUR:+ step = stepSize == null ? 1 : Math.max(1, Math.min(24, stepSize));+ truncate = "day";+ part = "hour";+ interval = "hour";+ break;+ case DAY:+ step = stepSize == null ? 1 : Math.max(1, Math.min(365, stepSize));+ truncate = "year";+ part = "doy";+ interval = "day";+ break;+ case WEEK:+ step = stepSize == null ? 1 : Math.max(1, Math.min(53, stepSize));+ truncate = "year";+ part = "week";+ interval = "week";+ break;+ case MONTH:+ step = stepSize == null ? 1 : Math.max(1, Math.min(12, stepSize));+ truncate = "year";+ part = "month";+ interval = "month";+ break;+ case YEAR:+ step = stepSize == null ? 1 : Math.max(1, stepSize);+ truncate = "decade";+ part = "year";+ interval = "year";+ break;+ default:+ throw new UnsupportedOperationException("Can't handle interval: " + datapointInterval);+ }+ stepStr = step + " " + interval;++ // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)+ query.append("select PERIOD as X, AVG_VALUE as Y " ++ "from generate_series(date_trunc(?, ?) + " + partQuery + " / ? * ?, date_trunc(?, ?) + " + partQuery + " / ? * ?, ?) PERIOD left join ( " ++ "select (date_trunc(?, TIMESTAMP) + " + partQuery2 + " / ? * ?)::timestamp as TS, ");++ if (isNumber) {+ query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");+ } else {+ query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");+ }++ query.append("from " + getDatapointTableName() ++ " where TIMESTAMP >= date_trunc(?, ?) and TIMESTAMP < (date_trunc(?, ?) + ?) and ENTITY_ID = ? and ATTRIBUTE_NAME = ? group by TS) DP on DP.TS = PERIOD order by PERIOD asc");++ } else {+ query.append("select distinct TIMESTAMP AS X, value AS Y from " + getDatapointTableName() ++ " where " ++ "TIMESTAMP >= ?" ++ "and " ++ "TIMESTAMP <= ? " ++ "and " ++ "ENTITY_ID = ? and ATTRIBUTE_NAME = ?"+ );+ }++ try (PreparedStatement st = connection.prepareStatement(query.toString())) {++ if (downsample) {+ int counter = 1;+ boolean isMinute = datapointInterval == DatapointInterval.MINUTE;++ st.setString(counter++, truncate);+ st.setObject(counter++, fromTimestamp);+ if (isMinute) {+ st.setObject(counter++, fromTimestamp);+ }+ st.setString(counter++, part);+ st.setObject(counter++, fromTimestamp);+ st.setInt(counter++, step);+ st.setObject(counter++, new PGInterval(stepStr));+ st.setString(counter++, truncate);+ st.setObject(counter++, toTimestamp);+ if (isMinute) {+ st.setObject(counter++, toTimestamp);+ }+ st.setString(counter++, part);+ st.setObject(counter++, toTimestamp);+ st.setInt(counter++, step);+ st.setObject(counter++, new PGInterval(stepStr));+ st.setObject(counter++, new PGInterval(stepStr));+ st.setString(counter++, truncate);+ st.setString(counter++, part);+ st.setInt(counter++, step);+ st.setObject(counter++, new PGInterval(stepStr));+ st.setString(counter++, interval);+ st.setObject(counter++, fromTimestamp);+ st.setString(counter++, interval);+ st.setObject(counter++, toTimestamp);+ st.setObject(counter++, new PGInterval(stepStr));+ st.setString(counter++, attributeRef.getId());+ st.setString(counter++, attributeRef.getName());+ } else {+ st.setObject(1, fromTimestamp);+ st.setObject(2, toTimestamp);+ st.setString(3, attributeRef.getId());+ st.setString(4, attributeRef.getName());+ }++ try (ResultSet rs = st.executeQuery()) {+ List<ValueDatapoint<?>> result = new ArrayList<>();+ while (rs.next()) {+ Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null) : null;+ result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));+ }+ return result.toArray(new ValueDatapoint<?>[0]);+ }+ }+ }+ })+ );+ }++ protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {+ return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " ++ "VALUES (?, ?, ?, ?) " ++ "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE " ++ "SET value = excluded.value");+ }++ protected void setUpsertValues(PreparedStatement st, String assetId, String attributeName, Object value, LocalDateTime timestamp) throws Exception {+ PGobject pgJsonValue = new PGobject();+ pgJsonValue.setType("jsonb");+ pgJsonValue.setValue(Values.asJSON(value).orElse("null"));+ st.setString(1, assetId);+ st.setString(2, attributeName);+ st.setObject(3, pgJsonValue);+ st.setObject(4, timestamp);+ }++ protected abstract Class<T> getDatapointClass();++ protected abstract String getDatapointTableName();++ protected abstract Logger getLogger();++ protected void doPurge(String whereClause, LocalDateTime dateTime) {+ persistenceService.doTransaction(em -> em.createQuery(+ "delete from " + getDatapointClass().getSimpleName() + " dp " + whereClause+ ).setParameter("dt", dateTime).executeUpdate());+ }++ protected long getFirstPurgeMillis(Instant currentTime) {+ // Schedule purge at approximately 3AM daily+ return ChronoUnit.MILLIS.between(+ currentTime,+ currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));+ }++ @Override+ public String toString() {+ return getClass().getSimpleName() + "{" ++ '}';+ }+}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.javaindex 5204179db..8725aee9d 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java@@ -30,7 +30,9 @@ import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.http.RequestParams;+import javax.ws.rs.BadRequestException;import javax.ws.rs.BeanParam;+import javax.ws.rs.NotSupportedException;import javax.ws.rs.WebApplicationException;import javax.ws.rs.core.Response;import java.time.Instant;@@ -56,11 +58,12 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As@Overridepublic ValueDatapoint<?>[] getDatapoints(@BeanParam RequestParams requestParams,- String assetId,- String attributeName,- DatapointInterval interval,- long fromTimestamp,- long toTimestamp) {+ String assetId,+ String attributeName,+ DatapointInterval interval,+ Integer stepSize,+ long fromTimestamp,+ long toTimestamp) {try {if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), assetId)) {@@ -85,10 +88,13 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements Asreturn assetDatapointService.getValueDatapoints(assetId,attribute,interval,+ stepSize,LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault()),LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault()));} catch (IllegalStateException ex) {- throw new WebApplicationException(ex, Response.Status.BAD_REQUEST);+ throw new BadRequestException(ex);+ } catch (UnsupportedOperationException ex) {+ throw new NotSupportedException(ex);}}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex d110964ca..66f015fb3 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -1,45 +1,31 @@package org.openremote.manager.datapoint;-import org.hibernate.Session;-import org.hibernate.jdbc.AbstractReturningWork;-import org.openremote.model.Container;-import org.openremote.model.ContainerService;-import org.openremote.container.persistence.PersistenceService;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.asset.AssetUpdateProcessor;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;+import org.openremote.model.Container;import org.openremote.model.asset.Asset;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent.Source;-import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.AttributeWriteFailure;import org.openremote.model.datapoint.AssetDatapoint;-import org.openremote.model.datapoint.DatapointInterval;-import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.query.AssetQuery;import org.openremote.model.query.filter.AttributePredicate;import org.openremote.model.query.filter.NameValuePredicate;import org.openremote.model.util.Pair;import org.openremote.model.value.MetaItemType;-import org.openremote.model.value.Values;-import org.postgresql.util.PGInterval;-import org.postgresql.util.PGobject;import javax.persistence.EntityManager;-import javax.persistence.TypedQuery;-import java.sql.*;+import java.sql.Date;import java.time.Duration;import java.time.Instant;import java.time.LocalDateTime;-import java.time.temporal.ChronoUnit;-import java.util.ArrayList;+import java.time.ZoneId;import java.util.List;import java.util.Map;-import java.util.concurrent.ScheduledExecutorService;-import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;@@ -57,38 +43,24 @@ import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;* and {@link #DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #DATA_POINTS_MAX_AGE_DAYS_DEFAULT}* days.*/-public class AssetDatapointService implements ContainerService, AssetUpdateProcessor {+public class AssetDatapointService extends AbstractDatapointService<AssetDatapoint> implements AssetUpdateProcessor {public static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";public static final int DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());- public static final int PRIORITY = AssetStorageService.PRIORITY + 100;- protected PersistenceService persistenceService;- protected AssetStorageService assetStorageService;- protected TimerService timerService;- protected ScheduledExecutorService executorService;protected int maxDatapointAgeDays;- protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;-- @Override- public int getPriority() {- return PRIORITY;- }@Overridepublic void init(Container container) throws Exception {- persistenceService = container.getService(PersistenceService.class);- assetStorageService = container.getService(AssetStorageService.class);- timerService = container.getService(TimerService.class);- executorService = container.getExecutorService();+ super.init(container);container.getService(ManagerWebService.class).getApiSingletons().add(- new AssetDatapointResourceImpl(- container.getService(TimerService.class),- container.getService(ManagerIdentityService.class),- container.getService(AssetStorageService.class),- this- )+ new AssetDatapointResourceImpl(+ container.getService(TimerService.class),+ container.getService(ManagerIdentityService.class),+ container.getService(AssetStorageService.class),+ this+ ));maxDatapointAgeDays = getInteger(container.getConfig(), DATA_POINTS_MAX_AGE_DAYS, DATA_POINTS_MAX_AGE_DAYS_DEFAULT);@@ -103,19 +75,12 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProceif (maxDatapointAgeDays > 0) {dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(this::purgeDataPoints,- getFirstRunMillis(timerService.getNow()),+ getFirstPurgeMillis(timerService.getNow()),Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS);}}- @Override- public void stop(Container container) throws Exception {- if (dataPointsPurgeScheduledFuture != null) {- dataPointsPurgeScheduledFuture.cancel(true);- }- }-protected static boolean attributeIsStoreDatapoint(Attribute<?> attribute) {return attribute.getMetaValue(STORE_DATA_POINTS).orElse(attribute.hasMeta(MetaItemType.AGENT_LINK));}@@ -127,213 +92,28 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProceSource source) throws AssetProcessingException {if (attributeIsStoreDatapoint(attribute) && attribute.getValue().isPresent()) { // Don't store datapoints with null value-- // Perform upsert on datapoint (datapoint isn't immutable then really and tied to postgresql but prevents entire attribute event from failing)- LOG.finest("Storing datapoint for: " + attribute);-- PGobject pgJsonValue = new PGobject();- pgJsonValue.setType("jsonb");try {- pgJsonValue.setValue(Values.asJSON(attribute.getValue().orElse(null)).orElse("null"));- } catch (SQLException e) {- throw new AssetProcessingException(AttributeWriteFailure.STATE_STORAGE_FAILED, "Failed to insert or update asset data point for attribute: " + attribute);+ upsertValue(asset.getId(), attribute.getName(), attribute.getValue().orElse(null), LocalDateTime.ofInstant(Instant.ofEpochMilli(attribute.getTimestamp().orElseGet(timerService::getCurrentTimeMillis)), ZoneId.systemDefault()));+ } catch (Exception e) {+ throw new AssetProcessingException(AttributeWriteFailure.STATE_STORAGE_FAILED, "Failed to insert or update asset data point for attribute: " + attribute, e);}-- em.unwrap(Session.class).doWork(connection -> {- PreparedStatement st = connection.prepareStatement("INSERT INTO asset_datapoint (entity_id, attribute_name, value, timestamp) \n" +- "VALUES (?, ?, ?, ?)\n" +- "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE \n" +- " SET value = excluded.value");-- st.setString(1, asset.getId());- st.setString(2, attribute.getName());- st.setObject(3, pgJsonValue);- st.setTimestamp(4, new java.sql.Timestamp(attribute.getTimestamp().orElseGet(timerService::getCurrentTimeMillis)));- st.executeUpdate();- });}return false;}- public List<AssetDatapoint> getDatapoints(AttributeRef attributeRef) {- return persistenceService.doReturningTransaction(entityManager ->- entityManager.createQuery(- "select dp from AssetDatapoint dp " +- "where dp.assetId = :assetId " +- "and dp.attributeName = :attributeName " +- "order by dp.timestamp desc",- AssetDatapoint.class)- .setParameter("assetId", attributeRef.getId())- .setParameter("attributeName", attributeRef.getName())- .getResultList());- }--- public long getDatapointsCount() {- return getDatapointsCount(null);- }-- public long getDatapointsCount(AttributeRef attributeRef) {- return persistenceService.doReturningTransaction(entityManager -> {-- String queryStr = attributeRef == null ?- "select count(dp) from AssetDatapoint dp" :- "select count(dp) from AssetDatapoint dp where dp.assetId = :assetId and dp.attributeName = :attributeName";-- TypedQuery<Long> query = entityManager.createQuery(- queryStr,- Long.class);-- if (attributeRef != null) {- query- .setParameter("assetId", attributeRef.getId())- .setParameter("attributeName", attributeRef.getName());- }-- return query.getSingleResult();- });+ @Override+ protected Class<AssetDatapoint> getDatapointClass() {+ return AssetDatapoint.class;}- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,- DatapointInterval datapointInterval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp) {-- Asset<?> asset = assetStorageService.find(attributeRef.getId());- if (asset == null) {- throw new IllegalStateException("Asset not found: " + attributeRef.getId());- }-- Attribute<?> assetAttribute = asset.getAttribute(attributeRef.getName())- .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeRef.getName()));-- return getValueDatapoints(asset.getId(), assetAttribute, datapointInterval, fromTimestamp, toTimestamp);+ @Override+ protected String getDatapointTableName() {+ return AssetDatapoint.TABLE_NAME;}- public ValueDatapoint<?>[] getValueDatapoints(String assetId, Attribute<?> attribute,- DatapointInterval datapointInterval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp) {-- AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());-- LOG.finer("Getting datapoints for: " + attributeRef);-- return persistenceService.doReturningTransaction(entityManager ->- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {- @Override- public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {-- String truncateX;- String interval;-- switch (datapointInterval) {- case MINUTE:- truncateX = "minute";- interval = "1 minute";- break;- case HOUR:- truncateX = "hour";- interval = "1 hour";- break;- case DAY:- truncateX = "day";- interval = "1 day";- break;- case WEEK:- truncateX = "day";- interval = "7 day";- break;- case MONTH:- truncateX = "day";- interval = "1 month";- break;- case YEAR:- truncateX = "month";- interval = "1 year";- break;- default:- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);- }-- Class<?> attributeType = attribute.getType().getType();- boolean isNumber = Number.class.isAssignableFrom(attributeType);- boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);- StringBuilder query = new StringBuilder();- boolean downsample = isNumber || isBoolean;-- if (downsample) {-- // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)- query.append("select PERIOD.TS as X, coalesce(AVG_VALUE, null) as Y " +- " from ( " +- " select date_trunc(?, GS)::timestamp TS " +- " from generate_series(?, ?, ?) GS " +- " ) PERIOD " +- " left join ( " +- " select " +- " date_trunc(?, TIMESTAMP)::timestamp as TS, ");-- if (isNumber) {- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");- } else {- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");- }-- query.append(" from ASSET_DATAPOINT " +- " where " +- " TIMESTAMP >= ? " +- " and " +- " TIMESTAMP <= ? " +- " and " +- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +- " group by TS " +- " ) DP on DP.TS >= PERIOD.TS and DP.TS < PERIOD.TS + ? " +- " order by PERIOD.TS asc "- );- } else {- query.append("select distinct TIMESTAMP AS X, value AS Y from ASSET_DATAPOINT " +- "where " +- "TIMESTAMP >= ?" +- "and " +- "TIMESTAMP <= ? " +- "and " +- "ENTITY_ID = ? and ATTRIBUTE_NAME = ? "- );- }-- try (PreparedStatement st = connection.prepareStatement(query.toString())) {-- if (downsample) {- st.setString(1, truncateX);- st.setObject(2, fromTimestamp);- st.setObject(3, toTimestamp);- st.setObject(4, new PGInterval(interval));- st.setString(5, truncateX);- st.setObject(6, fromTimestamp);- st.setObject(7, toTimestamp);- st.setString(8, attributeRef.getId());- st.setString(9, attributeRef.getName());- st.setObject(10, new PGInterval(interval));- } else {- st.setObject(1, fromTimestamp);- st.setObject(2, toTimestamp);- st.setString(3, attributeRef.getId());- st.setString(4, attributeRef.getName());- }-- try (ResultSet rs = st.executeQuery()) {- List<ValueDatapoint<?>> result = new ArrayList<>();- while (rs.next()) {- Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null) : null;- result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));- }- return result.toArray(new ValueDatapoint<?>[0]);- }- }- }- })- );+ @Override+ protected Logger getLogger() {+ return LOG;}protected void purgeDataPoints() {@@ -401,22 +181,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce}String whereStr = attributes.stream()- .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")- .collect(Collectors.joining(","));+ .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")+ .collect(Collectors.joining(","));return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";}-- protected long getFirstRunMillis(Instant currentTime) {- // Schedule purge at approximately 3AM daily- return ChronoUnit.MILLIS.between(- currentTime,- currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));- }-- @Override- public String toString() {- return getClass().getSimpleName() + "{" +- '}';- }}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.javaindex 7ec39bfcf..11f8882eb 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java@@ -24,11 +24,11 @@ import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebResource;import org.openremote.model.asset.Asset;-import org.openremote.model.attribute.AttributeRef;+import org.openremote.model.attribute.Attribute;+import org.openremote.model.datapoint.AssetPredictedDatapointResource;import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.http.RequestParams;-import org.openremote.model.datapoint.AssetPredictedDatapointResource;import javax.ws.rs.BeanParam;import javax.ws.rs.WebApplicationException;@@ -56,11 +56,12 @@ public class AssetPredictedDatapointResourceImpl extends ManagerWebResource impl@Overridepublic ValueDatapoint<?>[] getPredictedDatapoints(@BeanParam RequestParams requestParams,- String assetId,- String attributeName,- DatapointInterval datapointInterval,- long fromTimestamp,- long toTimestamp) {+ String assetId,+ String attributeName,+ DatapointInterval interval,+ Integer stepSize,+ long fromTimestamp,+ long toTimestamp) {try {if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), assetId)) {@@ -78,9 +79,14 @@ public class AssetPredictedDatapointResourceImpl extends ManagerWebResource implthrow new WebApplicationException(Response.Status.FORBIDDEN);}- return assetPredictedDatapointService.getValueDatapoints(- new AttributeRef(assetId, attributeName),- datapointInterval,+ Attribute<?> attribute = asset.getAttribute(attributeName).orElseThrow(() ->+ new WebApplicationException(Response.Status.NOT_FOUND)+ );++ return assetPredictedDatapointService.getValueDatapoints(assetId,+ attribute,+ interval,+ stepSize,LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault()),LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault()));} catch (IllegalStateException ex) {diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.javaindex 24520475b..39c65799b 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java@@ -19,55 +19,32 @@*/package org.openremote.manager.datapoint;-import org.hibernate.Session;-import org.hibernate.jdbc.AbstractReturningWork;import org.openremote.agent.protocol.ProtocolPredictedAssetService;-import org.openremote.container.persistence.PersistenceService;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;import org.openremote.model.Container;-import org.openremote.model.ContainerService;-import org.openremote.model.asset.Asset;-import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeRef;-import org.openremote.model.datapoint.DatapointInterval;-import org.openremote.model.datapoint.ValueDatapoint;+import org.openremote.model.datapoint.AssetPredictedDatapoint;+import org.openremote.model.datapoint.Datapoint;import org.openremote.model.util.Pair;-import org.openremote.model.value.Values;-import org.postgresql.util.PGInterval;-import org.postgresql.util.PGobject;-import javax.persistence.EntityManager;-import javax.persistence.TypedQuery;-import java.sql.*;import java.time.Duration;import java.time.Instant;import java.time.LocalDateTime;-import java.time.temporal.ChronoUnit;-import java.util.ArrayList;-import java.util.Collections;+import java.time.ZoneId;import java.util.List;-import java.util.concurrent.ScheduledExecutorService;-import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;-import static java.time.temporal.ChronoUnit.DAYS;+import static java.time.temporal.ChronoUnit.HOURS;-public class AssetPredictedDatapointService implements ContainerService, ProtocolPredictedAssetService {+public class AssetPredictedDatapointService extends AbstractDatapointService<AssetPredictedDatapoint> implements ProtocolPredictedAssetService {- public static final int PRIORITY = AssetStorageService.PRIORITY + 300;private static final Logger LOG = Logger.getLogger(AssetPredictedDatapointService.class.getName());- protected PersistenceService persistenceService;- protected AssetStorageService assetStorageService;- protected TimerService timerService;- protected ScheduledExecutorService executorService;- protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;-@Overridepublic int getPriority() {return PRIORITY;@@ -75,10 +52,7 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco@Overridepublic void init(Container container) throws Exception {- persistenceService = container.getService(PersistenceService.class);- assetStorageService = container.getService(AssetStorageService.class);- timerService = container.getService(TimerService.class);- executorService = container.getExecutorService();+ super.init(container);container.getService(ManagerWebService.class).getApiSingletons().add(new AssetPredictedDatapointResourceImpl(@@ -94,250 +68,50 @@ public class AssetPredictedDatapointService implements ContainerService, Protocopublic void start(Container container) throws Exception {dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(this::purgeDataPoints,- getFirstRunMillis(timerService.getNow()),+ getFirstPurgeMillis(timerService.getNow()),Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS);}- @Override- public void stop(Container container) throws Exception {- if (dataPointsPurgeScheduledFuture != null) {- dataPointsPurgeScheduledFuture.cancel(true);- }- }-- public long getDatapointsCount() {- return getDatapointsCount(null);- }-- public long getDatapointsCount(AttributeRef attributeRef) {- return persistenceService.doReturningTransaction(entityManager -> {-- String queryStr = attributeRef == null ?- "select count(dp) from AssetPredictedDatapoint dp" :- "select count(dp) from AssetPredictedDatapoint dp where dp.assetId = :assetId and dp.attributeName = :attributeName";-- TypedQuery<Long> query = entityManager.createQuery(- queryStr,- Long.class);-- if (attributeRef != null) {- query- .setParameter("assetId", attributeRef.getId())- .setParameter("attributeName", attributeRef.getName());- }-- return query.getSingleResult();- });- }-- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,- String truncate,- String interval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp) {- String assetId = attributeRef.getId();- String attributeName = attributeRef.getName();-- Asset<?> asset = assetStorageService.find(assetId);- if (asset == null) {- throw new IllegalStateException("Asset not found: " + assetId);- }-- Attribute<?> attribute = asset.getAttribute(attributeName)- .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeName));-- LOG.finer("Getting predicted datapoints for: " + attributeName);-- return persistenceService.doReturningTransaction(entityManager ->- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {- @Override- public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {--- Class<?> attributeType = attribute.getType().getType();- boolean isNumber = Number.class.isAssignableFrom(attributeType);- boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);- StringBuilder query = new StringBuilder();- boolean downsample = isNumber || isBoolean;-- if (downsample) {- // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)- query.append("select PERIOD.TS as X, coalesce(AVG_VALUE, null) as Y " +- " from ( " +- " select date_trunc(?, GS)::timestamp TS " +- " from generate_series(?, ?, ?) GS " +- " ) PERIOD " +- " left join ( " +- " select " +- " date_trunc(?, TIMESTAMP)::timestamp as TS, ");-- if (isNumber) {- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");- } else {- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");- }-- query.append(" from ASSET_PREDICTED_DATAPOINT " +- " where " +- " TIMESTAMP >= ? " +- " and " +- " TIMESTAMP <= ? " +- " and " +- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +- " group by TS " +- " ) DP on DP.TS >= PERIOD.TS and DP.TS < PERIOD.TS + ? " +- " order by PERIOD.TS asc "- );-- } else {- query.append("select distinct TIMESTAMP AS X, value AS Y from ASSET_PREDICTED_DATAPOINT " +- "where " +- "TIMESTAMP >= ? " +- "and " +- "TIMESTAMP < ? " +- "and " +- "ENTITY_ID = ? and ATTRIBUTE_NAME = ? "- );- }-- try (PreparedStatement st = connection.prepareStatement(query.toString())) {-- if (downsample) {- st.setString(1, truncate);- st.setObject(2, fromTimestamp);- st.setObject(3, toTimestamp);- st.setObject(4, new PGInterval(interval));- st.setString(5, truncate);- st.setObject(6, fromTimestamp);- st.setObject(7, toTimestamp);- st.setString(8, assetId);- st.setString(9, attributeName);- st.setObject(10, new PGInterval(interval));- } else {- st.setObject(1, fromTimestamp);- st.setObject(2, toTimestamp);- st.setString(3, assetId);- st.setString(4, attributeName);- }-- try (ResultSet rs = st.executeQuery()) {- List<ValueDatapoint<?>> result = new ArrayList<>();- while (rs.next()) {- Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), attributeType).orElse(null) : null;- result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));- }- return result.toArray(new ValueDatapoint<?>[0]);- }- }- }- })- );- }-- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,- DatapointInterval datapointInterval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp) {- String truncateX;- String interval;-- switch (datapointInterval) {- case MINUTE:- truncateX = "minute";- interval = "1 minute";- break;- case HOUR:- truncateX = "hour";- interval = "1 hour";- break;- case DAY:- truncateX = "day";- interval = "1 day";- break;- case WEEK:- truncateX = "day";- interval = "7 day";- break;- case MONTH:- truncateX = "day";- interval = "1 month";- break;- case YEAR:- truncateX = "month";- interval = "1 year";- break;- default:- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);- }-- return getValueDatapoints(attributeRef, truncateX, interval, fromTimestamp, toTimestamp);- }-public void updateValue(AttributeRef attributeRef, Object value, LocalDateTime timestamp) {updateValue(attributeRef.getId(), attributeRef.getName(), value, timestamp);}public void updateValue(String assetId, String attributeName, Object value, LocalDateTime timestamp) {- updateValues(assetId, attributeName, Collections.singletonList(new Pair<>(value, timestamp)));+ upsertValue(assetId, attributeName, value, timestamp);}public void updateValues(String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) {- persistenceService.doTransaction(em -> upsertValue(em, assetId, attributeName, valuesAndTimestamps));+ persistenceService.doTransaction(em -> upsertValues(assetId, attributeName, valuesAndTimestamps));}- protected void upsertValue(EntityManager em, String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) {- em.unwrap(Session.class).doWork(connection -> {- PreparedStatement st = connection.prepareStatement("INSERT INTO asset_predicted_datapoint (entity_id, attribute_name, value, timestamp) \n" +- "VALUES (?, ?, ?, ?)\n" +- "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE \n" +- " SET value = excluded.value");-- valuesAndTimestamps.forEach(valueAndTimestamp -> {+ @Override+ protected Class<AssetPredictedDatapoint> getDatapointClass() {+ return AssetPredictedDatapoint.class;+ }- PGobject pgJsonValue = new PGobject();- pgJsonValue.setType("jsonb");- try {- pgJsonValue.setValue(Values.asJSON(valueAndTimestamp.key).orElse("null"));- st.setString(1, assetId);- st.setString(2, attributeName);- st.setObject(3, pgJsonValue);- st.setObject(4, valueAndTimestamp.value);- st.addBatch();+ @Override+ protected String getDatapointTableName() {+ return AssetPredictedDatapoint.TABLE_NAME;+ }- } catch (Exception e) {- String msg = "Failed to store predicted data point: " + new AttributeRef(assetId, attributeName);- LOG.log(Level.WARNING, msg, e);- throw new IllegalStateException(msg, e);- }- });+ @Override+ protected Logger getLogger() {+ return LOG;+ }- st.executeBatch();- });+ @Override+ protected long getFirstPurgeMillis(Instant currentTime) {+ return super.getFirstPurgeMillis(currentTime) - 1800000; // Run half hour before default}protected void purgeDataPoints() {- LOG.info("Starting data points purge daily task");-try {// Purge data points not in the above list using default duration- LOG.finer("Purging predicted data points older than 1 day");-- persistenceService.doTransaction(em -> em.createQuery(- "delete from AssetPredictedDatapoint dp " +- "where dp.timestamp < :dt"- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(1, DAYS))).executeUpdate());+ LOG.finer("Purging predicted data points older than now");+ doPurge("where dp.timestamp < :dt", LocalDateTime.ofInstant(timerService.getNow().truncatedTo(HOURS), ZoneId.systemDefault()));} catch (Exception e) {LOG.log(Level.WARNING, "Failed to run data points purge", e);}-- LOG.info("Finished data points purge daily task");- }-- protected long getFirstRunMillis(Instant currentTime) {- // Schedule purge at approximately 2AM daily- return ChronoUnit.MILLIS.between(- currentTime,- currentTime.truncatedTo(DAYS).plus(26, ChronoUnit.HOURS));}}diff --git a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.javaindex b66eeb950..1f1ccfbb0 100644--- a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java+++ b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java@@ -19,6 +19,7 @@*/package org.openremote.manager.energy;+import com.google.common.util.concurrent.AtomicDouble;import org.apache.camel.builder.RouteBuilder;import org.openremote.container.message.MessageBrokerService;import org.openremote.container.persistence.PersistenceEvent;@@ -36,6 +37,7 @@ import org.openremote.model.asset.impl.*;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.attribute.AttributeRef;+import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.query.AssetQuery;import org.openremote.model.query.LogicGroup;@@ -338,8 +340,8 @@ public class EnergyOptimisationService extends RouteBuilder implements Container.types(ElectricityStorageAsset.class).attributes(new AttributePredicate().name(new StringPredicate(ElectricityAsset.POWER.getName())),- new AttributePredicate(ElectricityStorageAsset.SUPPORTS_IMPORT.getName(), new BooleanPredicate(false)),- new AttributePredicate(ElectricityStorageAsset.SUPPORTS_EXPORT.getName(), new BooleanPredicate(false))+ new AttributePredicate(ElectricityStorageAsset.SUPPORTS_IMPORT.getName(), new BooleanPredicate(true), true, null),+ new AttributePredicate(ElectricityStorageAsset.SUPPORTS_EXPORT.getName(), new BooleanPredicate(true), true, null))).forEach(asset -> {@SuppressWarnings("OptionalGetWithoutIsPresent")@@ -470,16 +472,44 @@ public class EnergyOptimisationService extends RouteBuilder implements ContainerLocalDateTime timestamp = LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault());ValueDatapoint<?>[] predictedData = assetPredictedDatapointService.getValueDatapoints(ref,- "minute",- (long)(intervalSize * 60) + " minute",+ DatapointInterval.MINUTE,+ (int)(intervalSize * 60),timestamp,timestamp.plus(24, HOURS).minus((long)(intervalSize * 60), ChronoUnit.MINUTES));if (predictedData.length != values.length) {LOG.warning("Returned predicted data point count does not match interval count: Ref=" + ref + ", expected=" + values.length + ", actual=" + predictedData.length);} else {- IntStream.range(0, predictedData.length).forEach(i ->- values[i] = predictedData[i].getValue() != null ? (double)(Object)predictedData[i].getValue() : 0d);++ IntStream.range(0, predictedData.length).forEach(i -> {+ if (predictedData[i].getValue() != null) {+ values[i] = (double) (Object) predictedData[i].getValue();+ } else {+ // Average previous and next values to fill in gaps (goes up to 5 back and forward) - this fixes+ // issues with resolution differences between stored predicted data and optimisation interval+ Double previous = null;+ Double next = null;+ int j = i-1;+ while (previous == null && j >= 0) {+ previous = (Double) predictedData[j].getValue();+ j--;+ }+ j = i+1;+ while (next == null && j < predictedData.length) {+ next = (Double) predictedData[j].getValue();+ j++;+ }+ if (next == null) {+ next = previous;+ }+ if (previous == null) {+ previous = next;+ }+ if (next != null) {+ values[i] = (previous + next) / 2;+ }+ }+ });}} else {LOG.finer("Electricity asset doesn't have any predicted data for attribute: Ref=" + ref);@@ -575,19 +605,28 @@ public class EnergyOptimisationService extends RouteBuilder implements Container}LOG.finer(getLogPrefix(optimisationAssetId) + "Applying earning opportunities for storage asset: " + storageAsset.getId());+optimiser.applyEarningOpportunities(importCostAndPower, exportCostAndPower, energyLevelMins, energyLevelMaxs, powerSetpoints, energyLevelCalculator, powerImportMaxCalculator, powerExportMaxCalculator);if (LOG.isLoggable(Level.FINER)) {- LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity power set points = " + Arrays.toString(powerSetpoints));+ LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity power set points for storage asset: " + storageAsset.getId() + " = " + Arrays.toString(powerSetpoints));+ }++ if (LOG.isLoggable(Level.FINEST)) {++ double previousCost = 0d;+ double optimisedCost = 0d;++ for (int i = 0; i < powerSetpoints.length; i++) {+ double power = powerNets[i];+ previousCost += Math.abs(power) * (power > 0 ? costImports[i] : costExports[i]) * optimiser.intervalSize;+ power += powerSetpoints[i];+ optimisedCost += Math.abs(power) * (power > 0 ? costImports[i] : costExports[i]) * optimiser.intervalSize;+ optimisedCost += Math.abs(power) * (power > 0 ? storageAsset.getTariffImport().orElse(0d) : storageAsset.getTariffExport().orElse(0d)) * optimiser.intervalSize;+ }- double[][] finalImportCostAndPower = importCostAndPower;- double[][] finalExportCostAndPower = exportCostAndPower;- double totalCost = IntStream.range(0, powerSetpoints.length).mapToDouble(i -> {- double setpoint = powerSetpoints[i];- double cost = setpoint < 0d ? finalExportCostAndPower[i][0] : finalImportCostAndPower[i][0];- return cost * setpoint;- }).reduce(0, Double::sum);+ // TODO Include storage import/export tariff- LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity cost = " + totalCost);+ LOG.finest(getLogPrefix(optimisationAssetId) + "Optimisation result for storage asset: " + storageAsset.getId() + " [Previous cost=" + previousCost + ", new cost=" + optimisedCost + "]");}return powerSetpoints;diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.javaindex 2334b4b04..a55c9d092 100644--- a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java+++ b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java@@ -42,7 +42,12 @@ public class HistoricFacade<T extends Ruleset> extends HistoricDatapoints {}@Override- public ValueDatapoint<?>[] getValueDataPoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {- return assetDatapointService.getValueDatapoints(attributeRef, interval, fromTimestamp, toTimestamp);+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {+ return assetDatapointService.getValueDatapoints(attributeRef, interval, null, fromTimestamp, toTimestamp);+ }++ @Override+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, int stepSize, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {+ return assetDatapointService.getValueDatapoints(attributeRef, interval, stepSize, fromTimestamp, toTimestamp);}}diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.javaindex 746b84486..78f2640cd 100644--- a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java+++ b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java@@ -39,14 +39,15 @@ public class PredictedFacade<T extends Ruleset> extends PredictedDatapoints {this.assetPredictedDatapointService = assetPredictedDatapointService;}+@Override- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, String truncate, String interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {- return assetPredictedDatapointService.getValueDatapoints(attributeRef, truncate, interval, fromTimestamp, toTimestamp);+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {+ return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, null, fromTimestamp, toTimestamp);}@Override- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {- return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, fromTimestamp, toTimestamp);+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, int stepSize, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {+ return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, stepSize, fromTimestamp, toTimestamp);}@Overridediff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.javaindex e8f8546d3..3cc7d7a63 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java@@ -30,6 +30,7 @@ import javax.persistence.Table;@Table(name = "ASSET_DATAPOINT")@org.hibernate.annotations.Immutablepublic class AssetDatapoint extends Datapoint {+ public static final String TABLE_NAME = "asset_datapoint";public AssetDatapoint() {}diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.javaindex fbba0a575..6bb64228e 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java@@ -44,6 +44,7 @@ public interface AssetDatapointResource {@PathParam("assetId") String assetId,@PathParam("attributeName") String attributeName,@QueryParam("interval") DatapointInterval datapointInterval,+ @QueryParam("step") Integer stepSize,@QueryParam("fromTimestamp") long fromTimestamp,@QueryParam("toTimestamp") long toTimestamp);diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.javaindex 69da8253d..6f905292b 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java@@ -28,9 +28,10 @@ import javax.persistence.Entity;import javax.persistence.Table;@Entity-@Table(name = "ASSET_PREDICTED_DATAPOINT")+@Table(name = AssetPredictedDatapoint.TABLE_NAME)@org.hibernate.annotations.Immutablepublic class AssetPredictedDatapoint extends Datapoint {+ public static final String TABLE_NAME = "asset_predicted_datapoint";public AssetPredictedDatapoint() {}diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.javaindex 177608b31..89cc9e57a 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java@@ -20,8 +20,6 @@package org.openremote.model.datapoint;import org.openremote.model.Constants;-import org.openremote.model.datapoint.DatapointInterval;-import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.http.RequestParams;import javax.annotation.security.RolesAllowed;import javax.ws.rs.*;@@ -42,10 +40,11 @@ public interface AssetPredictedDatapointResource {@Produces(APPLICATION_JSON)@RolesAllowed({Constants.READ_ASSETS_ROLE})ValueDatapoint<?>[] getPredictedDatapoints(@BeanParam RequestParams requestParams,- @PathParam("assetId") String assetId,- @PathParam("attributeName") String attributeName,- @QueryParam("interval") DatapointInterval datapointInterval,- @QueryParam("fromTimestamp") long fromTimestamp,- @QueryParam("toTimestamp") long toTimestamp);+ @PathParam("assetId") String assetId,+ @PathParam("attributeName") String attributeName,+ @QueryParam("interval") DatapointInterval datapointInterval,+ @QueryParam("step") Integer stepSize,+ @QueryParam("fromTimestamp") long fromTimestamp,+ @QueryParam("toTimestamp") long toTimestamp);}diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java b/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.javaindex d0517bf17..f7125eb41 100644--- a/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java@@ -21,7 +21,6 @@ package org.openremote.model.datapoint;public enum DatapointInterval {- NONE,MINUTE,HOUR,DAY,diff --git a/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java b/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.javaindex 473c6897b..82f01bde1 100644--- a/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java+++ b/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java@@ -32,8 +32,14 @@ import java.time.LocalDateTime;@TsIgnorepublic abstract class HistoricDatapoints {- public abstract ValueDatapoint<?>[] getValueDataPoints(AttributeRef attributeRef,- DatapointInterval interval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp);+ public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,+ DatapointInterval interval,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp);++ public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,+ DatapointInterval interval,+ int stepSize,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp);}diff --git a/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java b/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.javaindex d31412a31..90a9c3169 100644--- a/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java+++ b/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java@@ -33,18 +33,16 @@ import java.time.LocalDateTime;@TsIgnorepublic abstract class PredictedDatapoints {-public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,- String truncate,- String interval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp);-+ DatapointInterval interval,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp);public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,- DatapointInterval interval,- LocalDateTime fromTimestamp,- LocalDateTime toTimestamp);+ DatapointInterval interval,+ int stepSize,+ LocalDateTime fromTimestamp,+ LocalDateTime toTimestamp);public abstract void updateValue(String assetId, String attributeName, Object value, LocalDateTime timestamp);public abstract void updateValue(AttributeRef attributeRef, Object value, LocalDateTime timestamp);diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovyindex 340081fd5..f368b942f 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy@@ -21,9 +21,11 @@ import java.time.temporal.ChronoUnitimport java.util.concurrent.TimeUnitimport static java.util.concurrent.TimeUnit.DAYS+import static java.util.concurrent.TimeUnit.MINUTESimport static java.util.concurrent.TimeUnit.SECONDSimport static org.openremote.manager.datapoint.AssetDatapointService.DATA_POINTS_MAX_AGE_DAYS_DEFAULTimport static org.openremote.test.setup.ManagerTestSetup.thingLightToggleAttributeName+import static spock.util.matcher.HamcrestMatchers.closeToclass AssetDatapointTest extends Specification implements ManagerContainerTrait {@@ -41,8 +43,9 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitdef assetProcessingService = container.getService(AssetProcessingService.class)def assetDatapointService = container.getService(AssetDatapointService.class)- and: "the clock is stopped for testing purposes"+ and: "the clock is stopped for testing purposes and advanced to the next hour"stopPseudoClock()+ advancePseudoClock(Instant.ofEpochMilli(getClockTimeOf(container)).truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS).toEpochMilli() - getClockTimeOf(container), TimeUnit.MILLISECONDS, container)then: "the simulator protocol instance should have been initialised and attributes linked"conditions.eventually {@@ -53,26 +56,32 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitwhen: "an attribute linked to the simulator agent receives some values"def simulatorProtocol = ((SimulatorProtocol) agentService.protocolInstanceMap.get(managerTestSetup.agentId))- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), null)- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)- and: "we move forward in time more than purge days and a simulated sensor receives a new value"- advancePseudoClock(datapointPurgeDays, DAYS, container)+ then: "the attribute should be updated"+ conditions.eventually {+ def thing = assetStorageService.find(managerTestSetup.thingId, true)+ assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.3d+ }++ when: "a simulated sensor receives a new value"+ advancePseudoClock(60, SECONDS, container)def datapoint1ExpectedTimestamp = getClockTimeOf(container)- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)+ simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.5d)then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerTestSetup.thingId, true)- assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.3d+ assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.5d}when: "a simulated sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)def datapoint2ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 14.4d)@@ -83,7 +92,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait}when: "a simulated sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)def datapoint3ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 15.5d)@@ -94,7 +103,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait}when: "a simulated sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(60, SECONDS, container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), null)then: "the attribute should be updated"@@ -117,7 +126,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitassert Values.getValue(datapoints.get(1).value, Double.class).orElse(null) == 14.4dassert datapoints.get(1).timestamp == datapoint2ExpectedTimestamp- assert Values.getValue(datapoints.get(2).value, Double.class).orElse(null) == 13.3d+ assert Values.getValue(datapoints.get(2).value, Double.class).orElse(null) == 13.5dassert datapoints.get(2).timestamp == datapoint1ExpectedTimestampassert Values.getValue(datapoints.get(3).value, Double.class).orElse(null) == 13.3d@@ -132,18 +141,43 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitthing.getId(),thing.getAttribute("light1PowerConsumption").orElseThrow({ new RuntimeException("Missing attribute") }),DatapointInterval.MINUTE,- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()),- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).plus(1, ChronoUnit.HOURS)+ null,+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()))assert aggregatedDatapoints.size() == 61+ assert aggregatedDatapoints[54].value == 13.3+ assert aggregatedDatapoints[55].value == null+ assert aggregatedDatapoints[56].value == 13.3+ assert aggregatedDatapoints[57].value == 13.5+ assert aggregatedDatapoints[58].value == 14.4+ assert aggregatedDatapoints[59].value == 15.5+ assert aggregatedDatapoints[60].value == null+ }++ and: "when the step size is set on the datapoint retrieval then the datapoints should match"+ conditions.eventually {+ def thing = assetStorageService.find(managerTestSetup.thingId, true)+ def aggregatedDatapoints = assetDatapointService.getValueDatapoints(+ thing.getId(),+ thing.getAttribute("light1PowerConsumption").orElseThrow({ new RuntimeException("Missing attribute") }),+ DatapointInterval.MINUTE,+ 5,+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault())+ )+ assert aggregatedDatapoints.size() == 13+ assert aggregatedDatapoints[11].value, closeTo(13.36666, 0.0001)+ assert aggregatedDatapoints[12].value == 14.95}+// ------------------------------------// Test boolean data point storage// ------------------------------------when: "a simulated boolean sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(1, MINUTES, container)datapoint1ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), false)@@ -154,7 +188,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait}when: "a simulated sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(1, MINUTES, container)datapoint2ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), true)@@ -165,7 +199,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait}when: "a simulated sensor receives a new value"- advancePseudoClock(10, SECONDS, container)+ advancePseudoClock(1, MINUTES, container)datapoint3ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), false)@@ -197,10 +231,14 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitthing.getId(),thing.getAttribute(thingLightToggleAttributeName).orElseThrow({ new RuntimeException("Missing attribute") }),DatapointInterval.MINUTE,- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()),- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).plus(1, ChronoUnit.HOURS)+ null,+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()))assert aggregatedDatapoints.size() == 61+ assert aggregatedDatapoints[58].value == 0+ assert aggregatedDatapoints[59].value == 1d+ assert aggregatedDatapoints[60].value == 0}// ------------------------------------@@ -232,7 +270,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait}when: "the clock advances to the next days purge routine execution time"- advancePseudoClock(assetDatapointService.getFirstRunMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)+ advancePseudoClock(assetDatapointService.getFirstPurgeMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)and: "the purge routine runs"assetDatapointService.purgeDataPoints()diff --git a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovyindex 053379f8e..52df4c4c5 100644--- a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy+++ b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy@@ -15,6 +15,7 @@ import org.openremote.model.asset.impl.ElectricityStorageAssetimport org.openremote.model.asset.impl.ElectricitySupplierAssetimport org.openremote.model.attribute.AttributeEventimport org.openremote.model.attribute.AttributeRef+import org.openremote.model.datapoint.DatapointIntervalimport org.openremote.test.ManagerContainerTraitimport org.openremote.test.setup.ManagerTestSetupimport spock.lang.Specification@@ -124,8 +125,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContaiassert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0ddef setpoints = assetPredictedDatapointService.getValueDatapoints(new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),- "minute",- (long)(optimiser.intervalSize * 60) + " minute",+ DatapointInterval.MINUTE,+ (int)(optimiser.intervalSize * 60),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES))@@ -152,8 +153,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContaiassert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0ddef setpoints = assetPredictedDatapointService.getValueDatapoints(new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),- "minute",- (long)(optimiser.intervalSize * 60) + " minute",+ DatapointInterval.MINUTE,+ (int)(optimiser.intervalSize * 60),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES))@@ -180,8 +181,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContaiassert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0ddef setpoints = assetPredictedDatapointService.getValueDatapoints(new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),- "minute",- (long)(optimiser.intervalSize * 60) + " minute",+ DatapointInterval.MINUTE,+ (int)(optimiser.intervalSize * 60),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES))