Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Overhaul of asset datapoint services and energy optimisation updates

Rich Turner205c4919d122
diff --git a/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java b/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java
index 83ce0747c..1caa92323 100644
--- a/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java
+++ b/container/src/main/java/org/openremote/container/web/WebServiceExceptions.java
@@ -222,6 +222,12 @@ public class WebServiceExceptions {
if ("java.io.IOException: Connection reset by peer".equals(getRootCause(throwable).toString()))
return;
+ if (throwable instanceof WebApplicationException && ((WebApplicationException) throwable).getResponse().getStatus() == 404) {
+ // Don't stack trace 404s just want request uri
+ LOG.log(Level.FINE, "Web service exception (404) in '" + origin + "' for '" + info + "'");
+ return;
+ }
+
if (LOG.isLoggable(Level.FINE)) {
LOG.log(Level.FINE, "Web service exception in '" + origin + "' for '" + info + "'", throwable);
} else {
diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
index 352a0f68e..ab79f7d97 100644
--- a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
+++ b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
@@ -1708,7 +1708,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
StringBuilder attributeBuilder = new StringBuilder();
- if (nameValuePredicate.negated) {
+ // Only append not on the outer filter if value predicate is not set otherwise it will match any attribute with different name and value
+ if (nameValuePredicate.negated && (nameValuePredicate.value == null || nameValuePredicate.name == null)) {
attributeBuilder.append("NOT (");
}
@@ -1729,6 +1730,10 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
if (nameValuePredicate.name != null) {
attributeBuilder.append(" and ");
+
+ if (nameValuePredicate.negated) {
+ attributeBuilder.append(" NOT(");
+ }
}
// Inserts the SQL string and adds the parameters
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
new file mode 100644
index 000000000..ea1baa169
--- /dev/null
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
@@ -0,0 +1,384 @@
+package org.openremote.manager.datapoint;
+
+import org.hibernate.Session;
+import org.hibernate.jdbc.AbstractReturningWork;
+import org.openremote.model.Container;
+import org.openremote.model.ContainerService;
+import org.openremote.container.persistence.PersistenceService;
+import org.openremote.container.timer.TimerService;
+import org.openremote.manager.asset.AssetProcessingException;
+import org.openremote.manager.asset.AssetStorageService;
+import org.openremote.manager.asset.AssetUpdateProcessor;
+import org.openremote.manager.security.ManagerIdentityService;
+import org.openremote.manager.web.ManagerWebService;
+import org.openremote.model.asset.Asset;
+import org.openremote.model.attribute.Attribute;
+import org.openremote.model.attribute.AttributeEvent.Source;
+import org.openremote.model.attribute.AttributeRef;
+import org.openremote.model.attribute.AttributeWriteFailure;
+import org.openremote.model.datapoint.AssetDatapoint;
+import org.openremote.model.datapoint.Datapoint;
+import org.openremote.model.datapoint.DatapointInterval;
+import org.openremote.model.datapoint.ValueDatapoint;
+import org.openremote.model.query.AssetQuery;
+import org.openremote.model.query.filter.AttributePredicate;
+import org.openremote.model.query.filter.NameValuePredicate;
+import org.openremote.model.util.Pair;
+import org.openremote.model.value.MetaItemType;
+import org.openremote.model.value.Values;
+import org.postgresql.util.PGInterval;
+import org.postgresql.util.PGobject;
+
+import javax.persistence.EntityManager;
+import javax.persistence.TypedQuery;
+import java.sql.*;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.time.temporal.ChronoUnit.DAYS;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
+import static org.openremote.container.util.MapAccess.getInteger;
+import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;
+
+/**
+ * Base class for all classes that store and retrieve {@link org.openremote.model.datapoint.Datapoint}.
+ */
+public abstract class AbstractDatapointService<T extends Datapoint> implements ContainerService {
+
+ public static final int PRIORITY = AssetStorageService.PRIORITY + 100;
+ protected PersistenceService persistenceService;
+ protected AssetStorageService assetStorageService;
+ protected TimerService timerService;
+ protected ScheduledExecutorService executorService;
+ protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
+
+ @Override
+ public int getPriority() {
+ return PRIORITY;
+ }
+
+ @Override
+ public void init(Container container) throws Exception {
+ persistenceService = container.getService(PersistenceService.class);
+ assetStorageService = container.getService(AssetStorageService.class);
+ timerService = container.getService(TimerService.class);
+ executorService = container.getExecutorService();
+ }
+
+ @Override
+ public void stop(Container container) throws Exception {
+ if (dataPointsPurgeScheduledFuture != null) {
+ dataPointsPurgeScheduledFuture.cancel(true);
+ }
+ }
+
+ public void upsertValue(String assetId, String attributeName, Object value, LocalDateTime timestamp) throws IllegalStateException {
+ persistenceService.doTransaction(em ->
+ em.unwrap(Session.class).doWork(connection -> {
+ PreparedStatement st;
+
+ try {
+ st = getUpsertPreparedStatement(connection);
+ setUpsertValues(st, assetId, attributeName, value, timestamp);
+ st.executeUpdate();
+ } catch (Exception e) {
+ String msg = "Failed to insert/update data point: ";
+ getLogger().log(Level.WARNING, msg, e);
+ throw new IllegalStateException(msg, e);
+ }
+ }));
+ }
+
+ public void upsertValues(String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) throws IllegalStateException {
+ persistenceService.doTransaction(em ->
+ em.unwrap(Session.class).doWork(connection -> {
+
+ getLogger().finest("Storing datapoints for: id=" + assetId + ", name=" + attributeName + ", count=" + valuesAndTimestamps.size());
+ PreparedStatement st;
+
+ try {
+ st = getUpsertPreparedStatement(connection);
+
+ for (Pair<?, LocalDateTime> valueAndTimestamp : valuesAndTimestamps) {
+ setUpsertValues(st, assetId, attributeName, valueAndTimestamp.key, valueAndTimestamp.value);
+ st.addBatch();
+ }
+ st.executeBatch();
+ } catch (Exception e) {
+ String msg = "Failed to insert/update data points: " + assetId + ", name=" + attributeName + ", count=" + valuesAndTimestamps.size();
+ getLogger().log(Level.WARNING, msg, e);
+ throw new IllegalStateException(msg, e);
+ }
+ }));
+ }
+
+ public List<T> getDatapoints(AttributeRef attributeRef) {
+ return persistenceService.doReturningTransaction(entityManager ->
+ entityManager.createQuery(
+ "select dp from " + getDatapointClass().getSimpleName() + " dp " +
+ "where dp.assetId = :assetId " +
+ "and dp.attributeName = :attributeName " +
+ "order by dp.timestamp desc",
+ getDatapointClass())
+ .setParameter("assetId", attributeRef.getId())
+ .setParameter("attributeName", attributeRef.getName())
+ .getResultList());
+ }
+
+ public long getDatapointsCount() {
+ return getDatapointsCount(null);
+ }
+
+ public long getDatapointsCount(AttributeRef attributeRef) {
+ return persistenceService.doReturningTransaction(entityManager -> {
+
+ String queryStr = attributeRef == null ?
+ "select count(dp) from " + getDatapointClass().getSimpleName() + " dp" :
+ "select count(dp) from " + getDatapointClass().getSimpleName() + " dp where dp.assetId = :assetId and dp.attributeName = :attributeName";
+
+ TypedQuery<Long> query = entityManager.createQuery(
+ queryStr,
+ Long.class);
+
+ if (attributeRef != null) {
+ query
+ .setParameter("assetId", attributeRef.getId())
+ .setParameter("attributeName", attributeRef.getName());
+ }
+
+ return query.getSingleResult();
+ });
+ }
+
+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
+ DatapointInterval datapointInterval,
+ Integer stepSize,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp) {
+
+ Asset<?> asset = assetStorageService.find(attributeRef.getId());
+ if (asset == null) {
+ throw new IllegalStateException("Asset not found: " + attributeRef.getId());
+ }
+
+ Attribute<?> assetAttribute = asset.getAttribute(attributeRef.getName())
+ .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeRef.getName()));
+
+ return getValueDatapoints(asset.getId(), assetAttribute, datapointInterval, stepSize, fromTimestamp, toTimestamp);
+ }
+
+ public ValueDatapoint<?>[] getValueDatapoints(String assetId,
+ Attribute<?> attribute,
+ DatapointInterval datapointInterval,
+ final Integer stepSize,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp) {
+
+ AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());
+
+ getLogger().finer("Getting datapoints for: " + attributeRef);
+
+ return persistenceService.doReturningTransaction(entityManager ->
+
+ entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {
+ @Override
+ public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {
+
+ Class<?> attributeType = attribute.getType().getType();
+ boolean isNumber = Number.class.isAssignableFrom(attributeType);
+ boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);
+ StringBuilder query = new StringBuilder();
+ boolean downsample = isNumber || isBoolean;
+
+ String truncate = null;
+ String part = null;
+ String interval = null;
+ String stepStr = null;
+ String partQuery = "date_part(?, ?)::int";
+ String partQuery2 = "date_part(?, TIMESTAMP)::int";
+ int step = 1;
+
+ if (downsample) {
+
+ switch (datapointInterval) {
+
+ case MINUTE:
+ // This works with minutes of the day so not constrained to step size < 60
+ step = stepSize == null ? 1 : Math.max(1, Math.min(1440, stepSize));
+ truncate = "day";
+ part = "min";
+ interval = "min";
+ partQuery = "(date_part('hour', ?)::int * 60 + date_part(?, ?)::int)";
+ partQuery2 = "(date_part('hour', TIMESTAMP)::int * 60 + date_part(?, TIMESTAMP)::int)";
+ break;
+ case HOUR:
+ step = stepSize == null ? 1 : Math.max(1, Math.min(24, stepSize));
+ truncate = "day";
+ part = "hour";
+ interval = "hour";
+ break;
+ case DAY:
+ step = stepSize == null ? 1 : Math.max(1, Math.min(365, stepSize));
+ truncate = "year";
+ part = "doy";
+ interval = "day";
+ break;
+ case WEEK:
+ step = stepSize == null ? 1 : Math.max(1, Math.min(53, stepSize));
+ truncate = "year";
+ part = "week";
+ interval = "week";
+ break;
+ case MONTH:
+ step = stepSize == null ? 1 : Math.max(1, Math.min(12, stepSize));
+ truncate = "year";
+ part = "month";
+ interval = "month";
+ break;
+ case YEAR:
+ step = stepSize == null ? 1 : Math.max(1, stepSize);
+ truncate = "decade";
+ part = "year";
+ interval = "year";
+ break;
+ default:
+ throw new UnsupportedOperationException("Can't handle interval: " + datapointInterval);
+ }
+ stepStr = step + " " + interval;
+
+ // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)
+ query.append("select PERIOD as X, AVG_VALUE as Y " +
+ "from generate_series(date_trunc(?, ?) + " + partQuery + " / ? * ?, date_trunc(?, ?) + " + partQuery + " / ? * ?, ?) PERIOD left join ( " +
+ "select (date_trunc(?, TIMESTAMP) + " + partQuery2 + " / ? * ?)::timestamp as TS, ");
+
+ if (isNumber) {
+ query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");
+ } else {
+ query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");
+ }
+
+ query.append("from " + getDatapointTableName() +
+ " where TIMESTAMP >= date_trunc(?, ?) and TIMESTAMP < (date_trunc(?, ?) + ?) and ENTITY_ID = ? and ATTRIBUTE_NAME = ? group by TS) DP on DP.TS = PERIOD order by PERIOD asc");
+
+ } else {
+ query.append("select distinct TIMESTAMP AS X, value AS Y from " + getDatapointTableName() +
+ " where " +
+ "TIMESTAMP >= ?" +
+ "and " +
+ "TIMESTAMP <= ? " +
+ "and " +
+ "ENTITY_ID = ? and ATTRIBUTE_NAME = ?"
+ );
+ }
+
+ try (PreparedStatement st = connection.prepareStatement(query.toString())) {
+
+ if (downsample) {
+ int counter = 1;
+ boolean isMinute = datapointInterval == DatapointInterval.MINUTE;
+
+ st.setString(counter++, truncate);
+ st.setObject(counter++, fromTimestamp);
+ if (isMinute) {
+ st.setObject(counter++, fromTimestamp);
+ }
+ st.setString(counter++, part);
+ st.setObject(counter++, fromTimestamp);
+ st.setInt(counter++, step);
+ st.setObject(counter++, new PGInterval(stepStr));
+ st.setString(counter++, truncate);
+ st.setObject(counter++, toTimestamp);
+ if (isMinute) {
+ st.setObject(counter++, toTimestamp);
+ }
+ st.setString(counter++, part);
+ st.setObject(counter++, toTimestamp);
+ st.setInt(counter++, step);
+ st.setObject(counter++, new PGInterval(stepStr));
+ st.setObject(counter++, new PGInterval(stepStr));
+ st.setString(counter++, truncate);
+ st.setString(counter++, part);
+ st.setInt(counter++, step);
+ st.setObject(counter++, new PGInterval(stepStr));
+ st.setString(counter++, interval);
+ st.setObject(counter++, fromTimestamp);
+ st.setString(counter++, interval);
+ st.setObject(counter++, toTimestamp);
+ st.setObject(counter++, new PGInterval(stepStr));
+ st.setString(counter++, attributeRef.getId());
+ st.setString(counter++, attributeRef.getName());
+ } else {
+ st.setObject(1, fromTimestamp);
+ st.setObject(2, toTimestamp);
+ st.setString(3, attributeRef.getId());
+ st.setString(4, attributeRef.getName());
+ }
+
+ try (ResultSet rs = st.executeQuery()) {
+ List<ValueDatapoint<?>> result = new ArrayList<>();
+ while (rs.next()) {
+ Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null) : null;
+ result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));
+ }
+ return result.toArray(new ValueDatapoint<?>[0]);
+ }
+ }
+ }
+ })
+ );
+ }
+
+ protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {
+ return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " +
+ "VALUES (?, ?, ?, ?) " +
+ "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE " +
+ "SET value = excluded.value");
+ }
+
+ protected void setUpsertValues(PreparedStatement st, String assetId, String attributeName, Object value, LocalDateTime timestamp) throws Exception {
+ PGobject pgJsonValue = new PGobject();
+ pgJsonValue.setType("jsonb");
+ pgJsonValue.setValue(Values.asJSON(value).orElse("null"));
+ st.setString(1, assetId);
+ st.setString(2, attributeName);
+ st.setObject(3, pgJsonValue);
+ st.setObject(4, timestamp);
+ }
+
+ protected abstract Class<T> getDatapointClass();
+
+ protected abstract String getDatapointTableName();
+
+ protected abstract Logger getLogger();
+
+ protected void doPurge(String whereClause, LocalDateTime dateTime) {
+ persistenceService.doTransaction(em -> em.createQuery(
+ "delete from " + getDatapointClass().getSimpleName() + " dp " + whereClause
+ ).setParameter("dt", dateTime).executeUpdate());
+ }
+
+ protected long getFirstPurgeMillis(Instant currentTime) {
+ // Schedule purge at approximately 3AM daily
+ return ChronoUnit.MILLIS.between(
+ currentTime,
+ currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "{" +
+ '}';
+ }
+}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
index 5204179db..8725aee9d 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
@@ -30,7 +30,9 @@ import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.http.RequestParams;
+import javax.ws.rs.BadRequestException;
import javax.ws.rs.BeanParam;
+import javax.ws.rs.NotSupportedException;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.time.Instant;
@@ -56,11 +58,12 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
@Override
public ValueDatapoint<?>[] getDatapoints(@BeanParam RequestParams requestParams,
- String assetId,
- String attributeName,
- DatapointInterval interval,
- long fromTimestamp,
- long toTimestamp) {
+ String assetId,
+ String attributeName,
+ DatapointInterval interval,
+ Integer stepSize,
+ long fromTimestamp,
+ long toTimestamp) {
try {
if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), assetId)) {
@@ -85,10 +88,13 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
return assetDatapointService.getValueDatapoints(assetId,
attribute,
interval,
+ stepSize,
LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault()),
LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault()));
} catch (IllegalStateException ex) {
- throw new WebApplicationException(ex, Response.Status.BAD_REQUEST);
+ throw new BadRequestException(ex);
+ } catch (UnsupportedOperationException ex) {
+ throw new NotSupportedException(ex);
}
}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index d110964ca..66f015fb3 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -1,45 +1,31 @@
package org.openremote.manager.datapoint;
-import org.hibernate.Session;
-import org.hibernate.jdbc.AbstractReturningWork;
-import org.openremote.model.Container;
-import org.openremote.model.ContainerService;
-import org.openremote.container.persistence.PersistenceService;
import org.openremote.container.timer.TimerService;
import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.asset.AssetUpdateProcessor;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebService;
+import org.openremote.model.Container;
import org.openremote.model.asset.Asset;
import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeEvent.Source;
-import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.attribute.AttributeWriteFailure;
import org.openremote.model.datapoint.AssetDatapoint;
-import org.openremote.model.datapoint.DatapointInterval;
-import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.query.AssetQuery;
import org.openremote.model.query.filter.AttributePredicate;
import org.openremote.model.query.filter.NameValuePredicate;
import org.openremote.model.util.Pair;
import org.openremote.model.value.MetaItemType;
-import org.openremote.model.value.Values;
-import org.postgresql.util.PGInterval;
-import org.postgresql.util.PGobject;
import javax.persistence.EntityManager;
-import javax.persistence.TypedQuery;
-import java.sql.*;
+import java.sql.Date;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
+import java.time.ZoneId;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -57,38 +43,24 @@ import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;
* and {@link #DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #DATA_POINTS_MAX_AGE_DAYS_DEFAULT}
* days.
*/
-public class AssetDatapointService implements ContainerService, AssetUpdateProcessor {
+public class AssetDatapointService extends AbstractDatapointService<AssetDatapoint> implements AssetUpdateProcessor {
public static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";
public static final int DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
- public static final int PRIORITY = AssetStorageService.PRIORITY + 100;
- protected PersistenceService persistenceService;
- protected AssetStorageService assetStorageService;
- protected TimerService timerService;
- protected ScheduledExecutorService executorService;
protected int maxDatapointAgeDays;
- protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
-
- @Override
- public int getPriority() {
- return PRIORITY;
- }
@Override
public void init(Container container) throws Exception {
- persistenceService = container.getService(PersistenceService.class);
- assetStorageService = container.getService(AssetStorageService.class);
- timerService = container.getService(TimerService.class);
- executorService = container.getExecutorService();
+ super.init(container);
container.getService(ManagerWebService.class).getApiSingletons().add(
- new AssetDatapointResourceImpl(
- container.getService(TimerService.class),
- container.getService(ManagerIdentityService.class),
- container.getService(AssetStorageService.class),
- this
- )
+ new AssetDatapointResourceImpl(
+ container.getService(TimerService.class),
+ container.getService(ManagerIdentityService.class),
+ container.getService(AssetStorageService.class),
+ this
+ )
);
maxDatapointAgeDays = getInteger(container.getConfig(), DATA_POINTS_MAX_AGE_DAYS, DATA_POINTS_MAX_AGE_DAYS_DEFAULT);
@@ -103,19 +75,12 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
if (maxDatapointAgeDays > 0) {
dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(
this::purgeDataPoints,
- getFirstRunMillis(timerService.getNow()),
+ getFirstPurgeMillis(timerService.getNow()),
Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS
);
}
}
- @Override
- public void stop(Container container) throws Exception {
- if (dataPointsPurgeScheduledFuture != null) {
- dataPointsPurgeScheduledFuture.cancel(true);
- }
- }
-
protected static boolean attributeIsStoreDatapoint(Attribute<?> attribute) {
return attribute.getMetaValue(STORE_DATA_POINTS).orElse(attribute.hasMeta(MetaItemType.AGENT_LINK));
}
@@ -127,213 +92,28 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
Source source) throws AssetProcessingException {
if (attributeIsStoreDatapoint(attribute) && attribute.getValue().isPresent()) { // Don't store datapoints with null value
-
- // Perform upsert on datapoint (datapoint isn't immutable then really and tied to postgresql but prevents entire attribute event from failing)
- LOG.finest("Storing datapoint for: " + attribute);
-
- PGobject pgJsonValue = new PGobject();
- pgJsonValue.setType("jsonb");
try {
- pgJsonValue.setValue(Values.asJSON(attribute.getValue().orElse(null)).orElse("null"));
- } catch (SQLException e) {
- throw new AssetProcessingException(AttributeWriteFailure.STATE_STORAGE_FAILED, "Failed to insert or update asset data point for attribute: " + attribute);
+ upsertValue(asset.getId(), attribute.getName(), attribute.getValue().orElse(null), LocalDateTime.ofInstant(Instant.ofEpochMilli(attribute.getTimestamp().orElseGet(timerService::getCurrentTimeMillis)), ZoneId.systemDefault()));
+ } catch (Exception e) {
+ throw new AssetProcessingException(AttributeWriteFailure.STATE_STORAGE_FAILED, "Failed to insert or update asset data point for attribute: " + attribute, e);
}
-
- em.unwrap(Session.class).doWork(connection -> {
- PreparedStatement st = connection.prepareStatement("INSERT INTO asset_datapoint (entity_id, attribute_name, value, timestamp) \n" +
- "VALUES (?, ?, ?, ?)\n" +
- "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE \n" +
- " SET value = excluded.value");
-
- st.setString(1, asset.getId());
- st.setString(2, attribute.getName());
- st.setObject(3, pgJsonValue);
- st.setTimestamp(4, new java.sql.Timestamp(attribute.getTimestamp().orElseGet(timerService::getCurrentTimeMillis)));
- st.executeUpdate();
- });
}
return false;
}
- public List<AssetDatapoint> getDatapoints(AttributeRef attributeRef) {
- return persistenceService.doReturningTransaction(entityManager ->
- entityManager.createQuery(
- "select dp from AssetDatapoint dp " +
- "where dp.assetId = :assetId " +
- "and dp.attributeName = :attributeName " +
- "order by dp.timestamp desc",
- AssetDatapoint.class)
- .setParameter("assetId", attributeRef.getId())
- .setParameter("attributeName", attributeRef.getName())
- .getResultList());
- }
-
-
- public long getDatapointsCount() {
- return getDatapointsCount(null);
- }
-
- public long getDatapointsCount(AttributeRef attributeRef) {
- return persistenceService.doReturningTransaction(entityManager -> {
-
- String queryStr = attributeRef == null ?
- "select count(dp) from AssetDatapoint dp" :
- "select count(dp) from AssetDatapoint dp where dp.assetId = :assetId and dp.attributeName = :attributeName";
-
- TypedQuery<Long> query = entityManager.createQuery(
- queryStr,
- Long.class);
-
- if (attributeRef != null) {
- query
- .setParameter("assetId", attributeRef.getId())
- .setParameter("attributeName", attributeRef.getName());
- }
-
- return query.getSingleResult();
- });
+ @Override
+ protected Class<AssetDatapoint> getDatapointClass() {
+ return AssetDatapoint.class;
}
- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
- DatapointInterval datapointInterval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp) {
-
- Asset<?> asset = assetStorageService.find(attributeRef.getId());
- if (asset == null) {
- throw new IllegalStateException("Asset not found: " + attributeRef.getId());
- }
-
- Attribute<?> assetAttribute = asset.getAttribute(attributeRef.getName())
- .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeRef.getName()));
-
- return getValueDatapoints(asset.getId(), assetAttribute, datapointInterval, fromTimestamp, toTimestamp);
+ @Override
+ protected String getDatapointTableName() {
+ return AssetDatapoint.TABLE_NAME;
}
- public ValueDatapoint<?>[] getValueDatapoints(String assetId, Attribute<?> attribute,
- DatapointInterval datapointInterval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp) {
-
- AttributeRef attributeRef = new AttributeRef(assetId, attribute.getName());
-
- LOG.finer("Getting datapoints for: " + attributeRef);
-
- return persistenceService.doReturningTransaction(entityManager ->
- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {
- @Override
- public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {
-
- String truncateX;
- String interval;
-
- switch (datapointInterval) {
- case MINUTE:
- truncateX = "minute";
- interval = "1 minute";
- break;
- case HOUR:
- truncateX = "hour";
- interval = "1 hour";
- break;
- case DAY:
- truncateX = "day";
- interval = "1 day";
- break;
- case WEEK:
- truncateX = "day";
- interval = "7 day";
- break;
- case MONTH:
- truncateX = "day";
- interval = "1 month";
- break;
- case YEAR:
- truncateX = "month";
- interval = "1 year";
- break;
- default:
- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);
- }
-
- Class<?> attributeType = attribute.getType().getType();
- boolean isNumber = Number.class.isAssignableFrom(attributeType);
- boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);
- StringBuilder query = new StringBuilder();
- boolean downsample = isNumber || isBoolean;
-
- if (downsample) {
-
- // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)
- query.append("select PERIOD.TS as X, coalesce(AVG_VALUE, null) as Y " +
- " from ( " +
- " select date_trunc(?, GS)::timestamp TS " +
- " from generate_series(?, ?, ?) GS " +
- " ) PERIOD " +
- " left join ( " +
- " select " +
- " date_trunc(?, TIMESTAMP)::timestamp as TS, ");
-
- if (isNumber) {
- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");
- } else {
- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");
- }
-
- query.append(" from ASSET_DATAPOINT " +
- " where " +
- " TIMESTAMP >= ? " +
- " and " +
- " TIMESTAMP <= ? " +
- " and " +
- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +
- " group by TS " +
- " ) DP on DP.TS >= PERIOD.TS and DP.TS < PERIOD.TS + ? " +
- " order by PERIOD.TS asc "
- );
- } else {
- query.append("select distinct TIMESTAMP AS X, value AS Y from ASSET_DATAPOINT " +
- "where " +
- "TIMESTAMP >= ?" +
- "and " +
- "TIMESTAMP <= ? " +
- "and " +
- "ENTITY_ID = ? and ATTRIBUTE_NAME = ? "
- );
- }
-
- try (PreparedStatement st = connection.prepareStatement(query.toString())) {
-
- if (downsample) {
- st.setString(1, truncateX);
- st.setObject(2, fromTimestamp);
- st.setObject(3, toTimestamp);
- st.setObject(4, new PGInterval(interval));
- st.setString(5, truncateX);
- st.setObject(6, fromTimestamp);
- st.setObject(7, toTimestamp);
- st.setString(8, attributeRef.getId());
- st.setString(9, attributeRef.getName());
- st.setObject(10, new PGInterval(interval));
- } else {
- st.setObject(1, fromTimestamp);
- st.setObject(2, toTimestamp);
- st.setString(3, attributeRef.getId());
- st.setString(4, attributeRef.getName());
- }
-
- try (ResultSet rs = st.executeQuery()) {
- List<ValueDatapoint<?>> result = new ArrayList<>();
- while (rs.next()) {
- Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null) : null;
- result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));
- }
- return result.toArray(new ValueDatapoint<?>[0]);
- }
- }
- }
- })
- );
+ @Override
+ protected Logger getLogger() {
+ return LOG;
}
protected void purgeDataPoints() {
@@ -401,22 +181,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
}
String whereStr = attributes.stream()
- .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")
- .collect(Collectors.joining(","));
+ .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")
+ .collect(Collectors.joining(","));
return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";
}
-
- protected long getFirstRunMillis(Instant currentTime) {
- // Schedule purge at approximately 3AM daily
- return ChronoUnit.MILLIS.between(
- currentTime,
- currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "{" +
- '}';
- }
}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
index 7ec39bfcf..11f8882eb 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
@@ -24,11 +24,11 @@ import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebResource;
import org.openremote.model.asset.Asset;
-import org.openremote.model.attribute.AttributeRef;
+import org.openremote.model.attribute.Attribute;
+import org.openremote.model.datapoint.AssetPredictedDatapointResource;
import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.http.RequestParams;
-import org.openremote.model.datapoint.AssetPredictedDatapointResource;
import javax.ws.rs.BeanParam;
import javax.ws.rs.WebApplicationException;
@@ -56,11 +56,12 @@ public class AssetPredictedDatapointResourceImpl extends ManagerWebResource impl
@Override
public ValueDatapoint<?>[] getPredictedDatapoints(@BeanParam RequestParams requestParams,
- String assetId,
- String attributeName,
- DatapointInterval datapointInterval,
- long fromTimestamp,
- long toTimestamp) {
+ String assetId,
+ String attributeName,
+ DatapointInterval interval,
+ Integer stepSize,
+ long fromTimestamp,
+ long toTimestamp) {
try {
if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), assetId)) {
@@ -78,9 +79,14 @@ public class AssetPredictedDatapointResourceImpl extends ManagerWebResource impl
throw new WebApplicationException(Response.Status.FORBIDDEN);
}
- return assetPredictedDatapointService.getValueDatapoints(
- new AttributeRef(assetId, attributeName),
- datapointInterval,
+ Attribute<?> attribute = asset.getAttribute(attributeName).orElseThrow(() ->
+ new WebApplicationException(Response.Status.NOT_FOUND)
+ );
+
+ return assetPredictedDatapointService.getValueDatapoints(assetId,
+ attribute,
+ interval,
+ stepSize,
LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault()),
LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault()));
} catch (IllegalStateException ex) {
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
index 24520475b..39c65799b 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
@@ -19,55 +19,32 @@
*/
package org.openremote.manager.datapoint;
-import org.hibernate.Session;
-import org.hibernate.jdbc.AbstractReturningWork;
import org.openremote.agent.protocol.ProtocolPredictedAssetService;
-import org.openremote.container.persistence.PersistenceService;
import org.openremote.container.timer.TimerService;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebService;
import org.openremote.model.Container;
-import org.openremote.model.ContainerService;
-import org.openremote.model.asset.Asset;
-import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeRef;
-import org.openremote.model.datapoint.DatapointInterval;
-import org.openremote.model.datapoint.ValueDatapoint;
+import org.openremote.model.datapoint.AssetPredictedDatapoint;
+import org.openremote.model.datapoint.Datapoint;
import org.openremote.model.util.Pair;
-import org.openremote.model.value.Values;
-import org.postgresql.util.PGInterval;
-import org.postgresql.util.PGobject;
-import javax.persistence.EntityManager;
-import javax.persistence.TypedQuery;
-import java.sql.*;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
-import java.time.temporal.ChronoUnit;
-import java.util.ArrayList;
-import java.util.Collections;
+import java.time.ZoneId;
import java.util.List;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
-import static java.time.temporal.ChronoUnit.DAYS;
+import static java.time.temporal.ChronoUnit.HOURS;
-public class AssetPredictedDatapointService implements ContainerService, ProtocolPredictedAssetService {
+public class AssetPredictedDatapointService extends AbstractDatapointService<AssetPredictedDatapoint> implements ProtocolPredictedAssetService {
- public static final int PRIORITY = AssetStorageService.PRIORITY + 300;
private static final Logger LOG = Logger.getLogger(AssetPredictedDatapointService.class.getName());
- protected PersistenceService persistenceService;
- protected AssetStorageService assetStorageService;
- protected TimerService timerService;
- protected ScheduledExecutorService executorService;
- protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
-
@Override
public int getPriority() {
return PRIORITY;
@@ -75,10 +52,7 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
@Override
public void init(Container container) throws Exception {
- persistenceService = container.getService(PersistenceService.class);
- assetStorageService = container.getService(AssetStorageService.class);
- timerService = container.getService(TimerService.class);
- executorService = container.getExecutorService();
+ super.init(container);
container.getService(ManagerWebService.class).getApiSingletons().add(
new AssetPredictedDatapointResourceImpl(
@@ -94,250 +68,50 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
public void start(Container container) throws Exception {
dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(
this::purgeDataPoints,
- getFirstRunMillis(timerService.getNow()),
+ getFirstPurgeMillis(timerService.getNow()),
Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS
);
}
- @Override
- public void stop(Container container) throws Exception {
- if (dataPointsPurgeScheduledFuture != null) {
- dataPointsPurgeScheduledFuture.cancel(true);
- }
- }
-
- public long getDatapointsCount() {
- return getDatapointsCount(null);
- }
-
- public long getDatapointsCount(AttributeRef attributeRef) {
- return persistenceService.doReturningTransaction(entityManager -> {
-
- String queryStr = attributeRef == null ?
- "select count(dp) from AssetPredictedDatapoint dp" :
- "select count(dp) from AssetPredictedDatapoint dp where dp.assetId = :assetId and dp.attributeName = :attributeName";
-
- TypedQuery<Long> query = entityManager.createQuery(
- queryStr,
- Long.class);
-
- if (attributeRef != null) {
- query
- .setParameter("assetId", attributeRef.getId())
- .setParameter("attributeName", attributeRef.getName());
- }
-
- return query.getSingleResult();
- });
- }
-
- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
- String truncate,
- String interval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp) {
- String assetId = attributeRef.getId();
- String attributeName = attributeRef.getName();
-
- Asset<?> asset = assetStorageService.find(assetId);
- if (asset == null) {
- throw new IllegalStateException("Asset not found: " + assetId);
- }
-
- Attribute<?> attribute = asset.getAttribute(attributeName)
- .orElseThrow(() -> new IllegalStateException("Attribute not found: " + attributeName));
-
- LOG.finer("Getting predicted datapoints for: " + attributeName);
-
- return persistenceService.doReturningTransaction(entityManager ->
- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<ValueDatapoint<?>[]>() {
- @Override
- public ValueDatapoint<?>[] execute(Connection connection) throws SQLException {
-
-
- Class<?> attributeType = attribute.getType().getType();
- boolean isNumber = Number.class.isAssignableFrom(attributeType);
- boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);
- StringBuilder query = new StringBuilder();
- boolean downsample = isNumber || isBoolean;
-
- if (downsample) {
- // TODO: Change this to use something like this max min decimation algorithm https://knowledge.ni.com/KnowledgeArticleDetails?id=kA00Z0000019YLKSA2&l=en-GB)
- query.append("select PERIOD.TS as X, coalesce(AVG_VALUE, null) as Y " +
- " from ( " +
- " select date_trunc(?, GS)::timestamp TS " +
- " from generate_series(?, ?, ?) GS " +
- " ) PERIOD " +
- " left join ( " +
- " select " +
- " date_trunc(?, TIMESTAMP)::timestamp as TS, ");
-
- if (isNumber) {
- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");
- } else {
- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");
- }
-
- query.append(" from ASSET_PREDICTED_DATAPOINT " +
- " where " +
- " TIMESTAMP >= ? " +
- " and " +
- " TIMESTAMP <= ? " +
- " and " +
- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +
- " group by TS " +
- " ) DP on DP.TS >= PERIOD.TS and DP.TS < PERIOD.TS + ? " +
- " order by PERIOD.TS asc "
- );
-
- } else {
- query.append("select distinct TIMESTAMP AS X, value AS Y from ASSET_PREDICTED_DATAPOINT " +
- "where " +
- "TIMESTAMP >= ? " +
- "and " +
- "TIMESTAMP < ? " +
- "and " +
- "ENTITY_ID = ? and ATTRIBUTE_NAME = ? "
- );
- }
-
- try (PreparedStatement st = connection.prepareStatement(query.toString())) {
-
- if (downsample) {
- st.setString(1, truncate);
- st.setObject(2, fromTimestamp);
- st.setObject(3, toTimestamp);
- st.setObject(4, new PGInterval(interval));
- st.setString(5, truncate);
- st.setObject(6, fromTimestamp);
- st.setObject(7, toTimestamp);
- st.setString(8, assetId);
- st.setString(9, attributeName);
- st.setObject(10, new PGInterval(interval));
- } else {
- st.setObject(1, fromTimestamp);
- st.setObject(2, toTimestamp);
- st.setString(3, assetId);
- st.setString(4, attributeName);
- }
-
- try (ResultSet rs = st.executeQuery()) {
- List<ValueDatapoint<?>> result = new ArrayList<>();
- while (rs.next()) {
- Object value = rs.getObject(2) != null ? Values.getValueCoerced(rs.getObject(2), attributeType).orElse(null) : null;
- result.add(new ValueDatapoint<>(rs.getTimestamp(1).getTime(), value));
- }
- return result.toArray(new ValueDatapoint<?>[0]);
- }
- }
- }
- })
- );
- }
-
- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
- DatapointInterval datapointInterval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp) {
- String truncateX;
- String interval;
-
- switch (datapointInterval) {
- case MINUTE:
- truncateX = "minute";
- interval = "1 minute";
- break;
- case HOUR:
- truncateX = "hour";
- interval = "1 hour";
- break;
- case DAY:
- truncateX = "day";
- interval = "1 day";
- break;
- case WEEK:
- truncateX = "day";
- interval = "7 day";
- break;
- case MONTH:
- truncateX = "day";
- interval = "1 month";
- break;
- case YEAR:
- truncateX = "month";
- interval = "1 year";
- break;
- default:
- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);
- }
-
- return getValueDatapoints(attributeRef, truncateX, interval, fromTimestamp, toTimestamp);
- }
-
public void updateValue(AttributeRef attributeRef, Object value, LocalDateTime timestamp) {
updateValue(attributeRef.getId(), attributeRef.getName(), value, timestamp);
}
public void updateValue(String assetId, String attributeName, Object value, LocalDateTime timestamp) {
- updateValues(assetId, attributeName, Collections.singletonList(new Pair<>(value, timestamp)));
+ upsertValue(assetId, attributeName, value, timestamp);
}
public void updateValues(String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) {
- persistenceService.doTransaction(em -> upsertValue(em, assetId, attributeName, valuesAndTimestamps));
+ persistenceService.doTransaction(em -> upsertValues(assetId, attributeName, valuesAndTimestamps));
}
- protected void upsertValue(EntityManager em, String assetId, String attributeName, List<Pair<?, LocalDateTime>> valuesAndTimestamps) {
- em.unwrap(Session.class).doWork(connection -> {
- PreparedStatement st = connection.prepareStatement("INSERT INTO asset_predicted_datapoint (entity_id, attribute_name, value, timestamp) \n" +
- "VALUES (?, ?, ?, ?)\n" +
- "ON CONFLICT (entity_id, attribute_name, timestamp) DO UPDATE \n" +
- " SET value = excluded.value");
-
- valuesAndTimestamps.forEach(valueAndTimestamp -> {
+ @Override
+ protected Class<AssetPredictedDatapoint> getDatapointClass() {
+ return AssetPredictedDatapoint.class;
+ }
- PGobject pgJsonValue = new PGobject();
- pgJsonValue.setType("jsonb");
- try {
- pgJsonValue.setValue(Values.asJSON(valueAndTimestamp.key).orElse("null"));
- st.setString(1, assetId);
- st.setString(2, attributeName);
- st.setObject(3, pgJsonValue);
- st.setObject(4, valueAndTimestamp.value);
- st.addBatch();
+ @Override
+ protected String getDatapointTableName() {
+ return AssetPredictedDatapoint.TABLE_NAME;
+ }
- } catch (Exception e) {
- String msg = "Failed to store predicted data point: " + new AttributeRef(assetId, attributeName);
- LOG.log(Level.WARNING, msg, e);
- throw new IllegalStateException(msg, e);
- }
- });
+ @Override
+ protected Logger getLogger() {
+ return LOG;
+ }
- st.executeBatch();
- });
+ @Override
+ protected long getFirstPurgeMillis(Instant currentTime) {
+ return super.getFirstPurgeMillis(currentTime) - 1800000; // Run half hour before default
}
protected void purgeDataPoints() {
- LOG.info("Starting data points purge daily task");
-
try {
// Purge data points not in the above list using default duration
- LOG.finer("Purging predicted data points older than 1 day");
-
- persistenceService.doTransaction(em -> em.createQuery(
- "delete from AssetPredictedDatapoint dp " +
- "where dp.timestamp < :dt"
- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(1, DAYS))).executeUpdate());
+ LOG.finer("Purging predicted data points older than now");
+ doPurge("where dp.timestamp < :dt", LocalDateTime.ofInstant(timerService.getNow().truncatedTo(HOURS), ZoneId.systemDefault()));
} catch (Exception e) {
LOG.log(Level.WARNING, "Failed to run data points purge", e);
}
-
- LOG.info("Finished data points purge daily task");
- }
-
- protected long getFirstRunMillis(Instant currentTime) {
- // Schedule purge at approximately 2AM daily
- return ChronoUnit.MILLIS.between(
- currentTime,
- currentTime.truncatedTo(DAYS).plus(26, ChronoUnit.HOURS));
}
}
diff --git a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java
index b66eeb950..1f1ccfbb0 100644
--- a/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java
+++ b/manager/src/main/java/org/openremote/manager/energy/EnergyOptimisationService.java
@@ -19,6 +19,7 @@
*/
package org.openremote.manager.energy;
+import com.google.common.util.concurrent.AtomicDouble;
import org.apache.camel.builder.RouteBuilder;
import org.openremote.container.message.MessageBrokerService;
import org.openremote.container.persistence.PersistenceEvent;
@@ -36,6 +37,7 @@ import org.openremote.model.asset.impl.*;
import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeEvent;
import org.openremote.model.attribute.AttributeRef;
+import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.query.AssetQuery;
import org.openremote.model.query.LogicGroup;
@@ -338,8 +340,8 @@ public class EnergyOptimisationService extends RouteBuilder implements Container
.types(ElectricityStorageAsset.class)
.attributes(
new AttributePredicate().name(new StringPredicate(ElectricityAsset.POWER.getName())),
- new AttributePredicate(ElectricityStorageAsset.SUPPORTS_IMPORT.getName(), new BooleanPredicate(false)),
- new AttributePredicate(ElectricityStorageAsset.SUPPORTS_EXPORT.getName(), new BooleanPredicate(false))
+ new AttributePredicate(ElectricityStorageAsset.SUPPORTS_IMPORT.getName(), new BooleanPredicate(true), true, null),
+ new AttributePredicate(ElectricityStorageAsset.SUPPORTS_EXPORT.getName(), new BooleanPredicate(true), true, null)
)
).forEach(asset -> {
@SuppressWarnings("OptionalGetWithoutIsPresent")
@@ -470,16 +472,44 @@ public class EnergyOptimisationService extends RouteBuilder implements Container
LocalDateTime timestamp = LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault());
ValueDatapoint<?>[] predictedData = assetPredictedDatapointService.getValueDatapoints(
ref,
- "minute",
- (long)(intervalSize * 60) + " minute",
+ DatapointInterval.MINUTE,
+ (int)(intervalSize * 60),
timestamp,
timestamp.plus(24, HOURS).minus((long)(intervalSize * 60), ChronoUnit.MINUTES)
);
if (predictedData.length != values.length) {
LOG.warning("Returned predicted data point count does not match interval count: Ref=" + ref + ", expected=" + values.length + ", actual=" + predictedData.length);
} else {
- IntStream.range(0, predictedData.length).forEach(i ->
- values[i] = predictedData[i].getValue() != null ? (double)(Object)predictedData[i].getValue() : 0d);
+
+ IntStream.range(0, predictedData.length).forEach(i -> {
+ if (predictedData[i].getValue() != null) {
+ values[i] = (double) (Object) predictedData[i].getValue();
+ } else {
+ // Average previous and next values to fill in gaps (goes up to 5 back and forward) - this fixes
+ // issues with resolution differences between stored predicted data and optimisation interval
+ Double previous = null;
+ Double next = null;
+ int j = i-1;
+ while (previous == null && j >= 0) {
+ previous = (Double) predictedData[j].getValue();
+ j--;
+ }
+ j = i+1;
+ while (next == null && j < predictedData.length) {
+ next = (Double) predictedData[j].getValue();
+ j++;
+ }
+ if (next == null) {
+ next = previous;
+ }
+ if (previous == null) {
+ previous = next;
+ }
+ if (next != null) {
+ values[i] = (previous + next) / 2;
+ }
+ }
+ });
}
} else {
LOG.finer("Electricity asset doesn't have any predicted data for attribute: Ref=" + ref);
@@ -575,19 +605,28 @@ public class EnergyOptimisationService extends RouteBuilder implements Container
}
LOG.finer(getLogPrefix(optimisationAssetId) + "Applying earning opportunities for storage asset: " + storageAsset.getId());
+
optimiser.applyEarningOpportunities(importCostAndPower, exportCostAndPower, energyLevelMins, energyLevelMaxs, powerSetpoints, energyLevelCalculator, powerImportMaxCalculator, powerExportMaxCalculator);
if (LOG.isLoggable(Level.FINER)) {
- LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity power set points = " + Arrays.toString(powerSetpoints));
+ LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity power set points for storage asset: " + storageAsset.getId() + " = " + Arrays.toString(powerSetpoints));
+ }
+
+ if (LOG.isLoggable(Level.FINEST)) {
+
+ double previousCost = 0d;
+ double optimisedCost = 0d;
+
+ for (int i = 0; i < powerSetpoints.length; i++) {
+ double power = powerNets[i];
+ previousCost += Math.abs(power) * (power > 0 ? costImports[i] : costExports[i]) * optimiser.intervalSize;
+ power += powerSetpoints[i];
+ optimisedCost += Math.abs(power) * (power > 0 ? costImports[i] : costExports[i]) * optimiser.intervalSize;
+ optimisedCost += Math.abs(power) * (power > 0 ? storageAsset.getTariffImport().orElse(0d) : storageAsset.getTariffExport().orElse(0d)) * optimiser.intervalSize;
+ }
- double[][] finalImportCostAndPower = importCostAndPower;
- double[][] finalExportCostAndPower = exportCostAndPower;
- double totalCost = IntStream.range(0, powerSetpoints.length).mapToDouble(i -> {
- double setpoint = powerSetpoints[i];
- double cost = setpoint < 0d ? finalExportCostAndPower[i][0] : finalImportCostAndPower[i][0];
- return cost * setpoint;
- }).reduce(0, Double::sum);
+ // TODO Include storage import/export tariff
- LOG.finer(getLogPrefix(optimisationAssetId) + "Calculated earning opportunity cost = " + totalCost);
+ LOG.finest(getLogPrefix(optimisationAssetId) + "Optimisation result for storage asset: " + storageAsset.getId() + " [Previous cost=" + previousCost + ", new cost=" + optimisedCost + "]");
}
return powerSetpoints;
diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
index 2334b4b04..a55c9d092 100644
--- a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
+++ b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
@@ -42,7 +42,12 @@ public class HistoricFacade<T extends Ruleset> extends HistoricDatapoints {
}
@Override
- public ValueDatapoint<?>[] getValueDataPoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
- return assetDatapointService.getValueDatapoints(attributeRef, interval, fromTimestamp, toTimestamp);
+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
+ return assetDatapointService.getValueDatapoints(attributeRef, interval, null, fromTimestamp, toTimestamp);
+ }
+
+ @Override
+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, int stepSize, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
+ return assetDatapointService.getValueDatapoints(attributeRef, interval, stepSize, fromTimestamp, toTimestamp);
}
}
diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
index 746b84486..78f2640cd 100644
--- a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
+++ b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
@@ -39,14 +39,15 @@ public class PredictedFacade<T extends Ruleset> extends PredictedDatapoints {
this.assetPredictedDatapointService = assetPredictedDatapointService;
}
+
@Override
- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, String truncate, String interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
- return assetPredictedDatapointService.getValueDatapoints(attributeRef, truncate, interval, fromTimestamp, toTimestamp);
+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
+ return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, null, fromTimestamp, toTimestamp);
}
@Override
- public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
- return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, fromTimestamp, toTimestamp);
+ public ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef, DatapointInterval interval, int stepSize, LocalDateTime fromTimestamp, LocalDateTime toTimestamp) {
+ return assetPredictedDatapointService.getValueDatapoints(attributeRef, interval, stepSize, fromTimestamp, toTimestamp);
}
@Override
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java
index e8f8546d3..3cc7d7a63 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapoint.java
@@ -30,6 +30,7 @@ import javax.persistence.Table;
@Table(name = "ASSET_DATAPOINT")
@org.hibernate.annotations.Immutable
public class AssetDatapoint extends Datapoint {
+ public static final String TABLE_NAME = "asset_datapoint";
public AssetDatapoint() {
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
index fbba0a575..6bb64228e 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
@@ -44,6 +44,7 @@ public interface AssetDatapointResource {
@PathParam("assetId") String assetId,
@PathParam("attributeName") String attributeName,
@QueryParam("interval") DatapointInterval datapointInterval,
+ @QueryParam("step") Integer stepSize,
@QueryParam("fromTimestamp") long fromTimestamp,
@QueryParam("toTimestamp") long toTimestamp);
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
index 69da8253d..6f905292b 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
@@ -28,9 +28,10 @@ import javax.persistence.Entity;
import javax.persistence.Table;
@Entity
-@Table(name = "ASSET_PREDICTED_DATAPOINT")
+@Table(name = AssetPredictedDatapoint.TABLE_NAME)
@org.hibernate.annotations.Immutable
public class AssetPredictedDatapoint extends Datapoint {
+ public static final String TABLE_NAME = "asset_predicted_datapoint";
public AssetPredictedDatapoint() {
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
index 177608b31..89cc9e57a 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
@@ -20,8 +20,6 @@
package org.openremote.model.datapoint;
import org.openremote.model.Constants;
-import org.openremote.model.datapoint.DatapointInterval;
-import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.http.RequestParams;
import javax.annotation.security.RolesAllowed;
import javax.ws.rs.*;
@@ -42,10 +40,11 @@ public interface AssetPredictedDatapointResource {
@Produces(APPLICATION_JSON)
@RolesAllowed({Constants.READ_ASSETS_ROLE})
ValueDatapoint<?>[] getPredictedDatapoints(@BeanParam RequestParams requestParams,
- @PathParam("assetId") String assetId,
- @PathParam("attributeName") String attributeName,
- @QueryParam("interval") DatapointInterval datapointInterval,
- @QueryParam("fromTimestamp") long fromTimestamp,
- @QueryParam("toTimestamp") long toTimestamp);
+ @PathParam("assetId") String assetId,
+ @PathParam("attributeName") String attributeName,
+ @QueryParam("interval") DatapointInterval datapointInterval,
+ @QueryParam("step") Integer stepSize,
+ @QueryParam("fromTimestamp") long fromTimestamp,
+ @QueryParam("toTimestamp") long toTimestamp);
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java b/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java
index d0517bf17..f7125eb41 100644
--- a/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java
+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointInterval.java
@@ -21,7 +21,6 @@ package org.openremote.model.datapoint;
public enum DatapointInterval {
- NONE,
MINUTE,
HOUR,
DAY,
diff --git a/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java b/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java
index 473c6897b..82f01bde1 100644
--- a/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java
+++ b/model/src/main/java/org/openremote/model/rules/HistoricDatapoints.java
@@ -32,8 +32,14 @@ import java.time.LocalDateTime;
@TsIgnore
public abstract class HistoricDatapoints {
- public abstract ValueDatapoint<?>[] getValueDataPoints(AttributeRef attributeRef,
- DatapointInterval interval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp);
+ public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
+ DatapointInterval interval,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp);
+
+ public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
+ DatapointInterval interval,
+ int stepSize,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp);
}
diff --git a/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java b/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java
index d31412a31..90a9c3169 100644
--- a/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java
+++ b/model/src/main/java/org/openremote/model/rules/PredictedDatapoints.java
@@ -33,18 +33,16 @@ import java.time.LocalDateTime;
@TsIgnore
public abstract class PredictedDatapoints {
-
public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
- String truncate,
- String interval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp);
-
+ DatapointInterval interval,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp);
public abstract ValueDatapoint<?>[] getValueDatapoints(AttributeRef attributeRef,
- DatapointInterval interval,
- LocalDateTime fromTimestamp,
- LocalDateTime toTimestamp);
+ DatapointInterval interval,
+ int stepSize,
+ LocalDateTime fromTimestamp,
+ LocalDateTime toTimestamp);
public abstract void updateValue(String assetId, String attributeName, Object value, LocalDateTime timestamp);
public abstract void updateValue(AttributeRef attributeRef, Object value, LocalDateTime timestamp);
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
index 340081fd5..f368b942f 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
@@ -21,9 +21,11 @@ import java.time.temporal.ChronoUnit
import java.util.concurrent.TimeUnit
import static java.util.concurrent.TimeUnit.DAYS
+import static java.util.concurrent.TimeUnit.MINUTES
import static java.util.concurrent.TimeUnit.SECONDS
import static org.openremote.manager.datapoint.AssetDatapointService.DATA_POINTS_MAX_AGE_DAYS_DEFAULT
import static org.openremote.test.setup.ManagerTestSetup.thingLightToggleAttributeName
+import static spock.util.matcher.HamcrestMatchers.closeTo
class AssetDatapointTest extends Specification implements ManagerContainerTrait {
@@ -41,8 +43,9 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
def assetProcessingService = container.getService(AssetProcessingService.class)
def assetDatapointService = container.getService(AssetDatapointService.class)
- and: "the clock is stopped for testing purposes"
+ and: "the clock is stopped for testing purposes and advanced to the next hour"
stopPseudoClock()
+ advancePseudoClock(Instant.ofEpochMilli(getClockTimeOf(container)).truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS).toEpochMilli() - getClockTimeOf(container), TimeUnit.MILLISECONDS, container)
then: "the simulator protocol instance should have been initialised and attributes linked"
conditions.eventually {
@@ -53,26 +56,32 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
when: "an attribute linked to the simulator agent receives some values"
def simulatorProtocol = ((SimulatorProtocol) agentService.protocolInstanceMap.get(managerTestSetup.agentId))
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), null)
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)
- and: "we move forward in time more than purge days and a simulated sensor receives a new value"
- advancePseudoClock(datapointPurgeDays, DAYS, container)
+ then: "the attribute should be updated"
+ conditions.eventually {
+ def thing = assetStorageService.find(managerTestSetup.thingId, true)
+ assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.3d
+ }
+
+ when: "a simulated sensor receives a new value"
+ advancePseudoClock(60, SECONDS, container)
def datapoint1ExpectedTimestamp = getClockTimeOf(container)
- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.3d)
+ simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 13.5d)
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerTestSetup.thingId, true)
- assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.3d
+ assert thing.getAttribute("light1PowerConsumption").flatMap { it.getValueAs(Double.class) }.orElse(null) == 13.5d
}
when: "a simulated sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
def datapoint2ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 14.4d)
@@ -83,7 +92,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
}
when: "a simulated sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
def datapoint3ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 15.5d)
@@ -94,7 +103,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
}
when: "a simulated sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(60, SECONDS, container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), null)
then: "the attribute should be updated"
@@ -117,7 +126,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
assert Values.getValue(datapoints.get(1).value, Double.class).orElse(null) == 14.4d
assert datapoints.get(1).timestamp == datapoint2ExpectedTimestamp
- assert Values.getValue(datapoints.get(2).value, Double.class).orElse(null) == 13.3d
+ assert Values.getValue(datapoints.get(2).value, Double.class).orElse(null) == 13.5d
assert datapoints.get(2).timestamp == datapoint1ExpectedTimestamp
assert Values.getValue(datapoints.get(3).value, Double.class).orElse(null) == 13.3d
@@ -132,18 +141,43 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
thing.getId(),
thing.getAttribute("light1PowerConsumption").orElseThrow({ new RuntimeException("Missing attribute") }),
DatapointInterval.MINUTE,
- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).plus(1, ChronoUnit.HOURS)
+ null,
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault())
)
assert aggregatedDatapoints.size() == 61
+ assert aggregatedDatapoints[54].value == 13.3
+ assert aggregatedDatapoints[55].value == null
+ assert aggregatedDatapoints[56].value == 13.3
+ assert aggregatedDatapoints[57].value == 13.5
+ assert aggregatedDatapoints[58].value == 14.4
+ assert aggregatedDatapoints[59].value == 15.5
+ assert aggregatedDatapoints[60].value == null
+ }
+
+ and: "when the step size is set on the datapoint retrieval then the datapoints should match"
+ conditions.eventually {
+ def thing = assetStorageService.find(managerTestSetup.thingId, true)
+ def aggregatedDatapoints = assetDatapointService.getValueDatapoints(
+ thing.getId(),
+ thing.getAttribute("light1PowerConsumption").orElseThrow({ new RuntimeException("Missing attribute") }),
+ DatapointInterval.MINUTE,
+ 5,
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault())
+ )
+ assert aggregatedDatapoints.size() == 13
+ assert aggregatedDatapoints[11].value, closeTo(13.36666, 0.0001)
+ assert aggregatedDatapoints[12].value == 14.95
}
+
// ------------------------------------
// Test boolean data point storage
// ------------------------------------
when: "a simulated boolean sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(1, MINUTES, container)
datapoint1ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), false)
@@ -154,7 +188,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
}
when: "a simulated sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(1, MINUTES, container)
datapoint2ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), true)
@@ -165,7 +199,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
}
when: "a simulated sensor receives a new value"
- advancePseudoClock(10, SECONDS, container)
+ advancePseudoClock(1, MINUTES, container)
datapoint3ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), false)
@@ -197,10 +231,14 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
thing.getId(),
thing.getAttribute(thingLightToggleAttributeName).orElseThrow({ new RuntimeException("Missing attribute") }),
DatapointInterval.MINUTE,
- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()),
- LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).plus(1, ChronoUnit.HOURS)
+ null,
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault()).minus(1, ChronoUnit.HOURS),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(getClockTimeOf(container)), ZoneId.systemDefault())
)
assert aggregatedDatapoints.size() == 61
+ assert aggregatedDatapoints[58].value == 0
+ assert aggregatedDatapoints[59].value == 1d
+ assert aggregatedDatapoints[60].value == 0
}
// ------------------------------------
@@ -232,7 +270,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
}
when: "the clock advances to the next days purge routine execution time"
- advancePseudoClock(assetDatapointService.getFirstRunMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)
+ advancePseudoClock(assetDatapointService.getFirstPurgeMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)
and: "the purge routine runs"
assetDatapointService.purgeDataPoints()
diff --git a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
index 053379f8e..52df4c4c5 100644
--- a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
@@ -15,6 +15,7 @@ import org.openremote.model.asset.impl.ElectricityStorageAsset
import org.openremote.model.asset.impl.ElectricitySupplierAsset
import org.openremote.model.attribute.AttributeEvent
import org.openremote.model.attribute.AttributeRef
+import org.openremote.model.datapoint.DatapointInterval
import org.openremote.test.ManagerContainerTrait
import org.openremote.test.setup.ManagerTestSetup
import spock.lang.Specification
@@ -124,8 +125,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai
assert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0d
def setpoints = assetPredictedDatapointService.getValueDatapoints(
new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),
- "minute",
- (long)(optimiser.intervalSize * 60) + " minute",
+ DatapointInterval.MINUTE,
+ (int)(optimiser.intervalSize * 60),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES)
)
@@ -152,8 +153,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai
assert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0d
def setpoints = assetPredictedDatapointService.getValueDatapoints(
new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),
- "minute",
- (long)(optimiser.intervalSize * 60) + " minute",
+ DatapointInterval.MINUTE,
+ (int)(optimiser.intervalSize * 60),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES)
)
@@ -180,8 +181,8 @@ class EnergyOptimisationAssetTest extends Specification implements ManagerContai
assert ((ElectricityStorageAsset)assetStorageService.find(managerTestSetup.electricityBatteryAssetId)).getPowerSetpoint().orElse(0d) == 0d
def setpoints = assetPredictedDatapointService.getValueDatapoints(
new AttributeRef(managerTestSetup.electricityBatteryAssetId, ElectricityAsset.POWER_SETPOINT.name),
- "minute",
- (long)(optimiser.intervalSize * 60) + " minute",
+ DatapointInterval.MINUTE,
+ (int)(optimiser.intervalSize * 60),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES),
LocalDateTime.ofInstant(optimisationTime, ZoneId.systemDefault()).plus(1, ChronoUnit.DAYS).minus((long)(optimiser.intervalSize * 60), ChronoUnit.MINUTES)
)