Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Added auto purging functionality to asset data point service with tests

Rich Turner5d3b04fab75c
diff --git a/container/src/main/java/org/openremote/container/timer/TimerService.java b/container/src/main/java/org/openremote/container/timer/TimerService.java
index 5aa3afc6e..f4d0b3824 100644
--- a/container/src/main/java/org/openremote/container/timer/TimerService.java
+++ b/container/src/main/java/org/openremote/container/timer/TimerService.java
@@ -24,7 +24,6 @@ import org.openremote.container.ContainerService;
import java.time.Instant;
import java.time.ZoneId;
-import java.time.ZonedDateTime;
import java.util.Date;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -113,8 +112,8 @@ public class TimerService implements ContainerService {
return getClock().getCurrentTimeMillis();
}
- public ZonedDateTime getNow() {
- return Instant.ofEpochMilli(getCurrentTimeMillis()).atZone(ZoneId.systemDefault());
+ public Instant getNow() {
+ return Instant.ofEpochMilli(getCurrentTimeMillis());
}
@Override
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 6ec010e25..3d0a6a431 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -10,56 +10,103 @@ import org.openremote.container.web.WebService;
import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.asset.AssetUpdateProcessor;
+import org.openremote.manager.concurrent.ManagerExecutorService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.model.asset.Asset;
import org.openremote.model.asset.AssetAttribute;
+import org.openremote.model.asset.AssetMeta;
import org.openremote.model.attribute.AttributeEvent.Source;
import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.datapoint.AssetDatapoint;
import org.openremote.model.datapoint.Datapoint;
import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.NumberDatapoint;
+import org.openremote.model.query.AssetQuery;
+import org.openremote.model.query.BaseAssetQuery;
+import org.openremote.model.query.filter.AttributeMetaPredicate;
+import org.openremote.model.value.Values;
import org.postgresql.util.PGInterval;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
import java.sql.*;
import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
import java.util.function.Function;
+import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.time.temporal.ChronoUnit.DAYS;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toList;
/**
- * Store and retrieve datapoints for asset attributes.
+ * Store and retrieve datapoints for asset attributes and periodically purge data points based on
+ * {@link org.openremote.model.asset.AssetMeta#DATA_POINTS_MAX_AGE_DAYS} {@link org.openremote.model.attribute.MetaItem}
+ * and {@link #DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #DATA_POINTS_MAX_AGE_DAYS_DEFAULT}
+ * days.
*/
public class AssetDatapointService implements ContainerService, AssetUpdateProcessor {
+ public static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";
+ public static final String DATA_POINTS_MAX_AGE_DAYS_DEFAULT = "30";
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
-
protected PersistenceService persistenceService;
+ protected AssetStorageService assetStorageService;
+ protected TimerService timerService;
+ protected ManagerExecutorService managerExecutorService;
+ protected int maxDatapointAgeDays;
+ protected ScheduledFuture dataPointsPurgeScheduledFuture;
@Override
public void init(Container container) throws Exception {
persistenceService = container.getService(PersistenceService.class);
+ assetStorageService = container.getService(AssetStorageService.class);
+ timerService = container.getService(TimerService.class);
+ managerExecutorService = container.getService(ManagerExecutorService.class);
container.getService(WebService.class).getApiSingletons().add(
- new AssetDatapointResourceImpl(
- container.getService(TimerService.class),
- container.getService(ManagerIdentityService.class),
- container.getService(AssetStorageService.class),
- this
- )
+ new AssetDatapointResourceImpl(
+ container.getService(TimerService.class),
+ container.getService(ManagerIdentityService.class),
+ container.getService(AssetStorageService.class),
+ this
+ )
+ );
+
+ maxDatapointAgeDays = Integer.parseInt(
+ container.getConfig().getOrDefault(DATA_POINTS_MAX_AGE_DAYS, DATA_POINTS_MAX_AGE_DAYS_DEFAULT)
);
+ if (maxDatapointAgeDays <= 0) {
+ LOG.warning(DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");
+ }
}
@Override
public void start(Container container) throws Exception {
+ if (maxDatapointAgeDays > 0) {
+ long period = 24L * 3600L * 1000L;
+
+ dataPointsPurgeScheduledFuture = managerExecutorService.scheduleAtFixedRate(
+ this::purgeDataPoints,
+ getFirstRunMillis(timerService.getNow()),
+ period);
+
+ }
}
@Override
public void stop(Container container) throws Exception {
+ if (dataPointsPurgeScheduledFuture != null) {
+ dataPointsPurgeScheduledFuture.cancel(true);
+ }
}
@Override
@@ -68,9 +115,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
AssetAttribute attribute,
Source source) throws AssetProcessingException {
if (Datapoint.isDatapointsCapable(attribute)
- && attribute.isStoreDatapoints()
- && attribute.getStateEvent().isPresent()
- && attribute.getStateEvent().get().getValue().isPresent()) { // Don't store datapoints with null value
+ && attribute.isStoreDatapoints()
+ && attribute.getStateEvent().isPresent()
+ && attribute.getStateEvent().get().getValue().isPresent()) { // Don't store datapoints with null value
LOG.finest("Storing datapoint for: " + attribute);
AssetDatapoint assetDatapoint = new AssetDatapoint(attribute.getStateEvent().get());
em.persist(assetDatapoint);
@@ -80,15 +127,15 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
public List<AssetDatapoint> getDatapoints(AttributeRef attributeRef) {
return persistenceService.doReturningTransaction(entityManager ->
- entityManager.createQuery(
- "select dp from AssetDatapoint dp " +
- "where dp.entityId = :assetId " +
- "and dp.attributeName = :attributeName " +
- "order by dp.timestamp desc",
- AssetDatapoint.class)
- .setParameter("assetId", attributeRef.getEntityId())
- .setParameter("attributeName", attributeRef.getAttributeName())
- .getResultList());
+ entityManager.createQuery(
+ "select dp from AssetDatapoint dp " +
+ "where dp.entityId = :assetId " +
+ "and dp.attributeName = :attributeName " +
+ "order by dp.timestamp desc",
+ AssetDatapoint.class)
+ .setParameter("assetId", attributeRef.getEntityId())
+ .setParameter("attributeName", attributeRef.getAttributeName())
+ .getResultList());
}
@@ -100,17 +147,17 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
return persistenceService.doReturningTransaction(entityManager -> {
String queryStr = attributeRef == null ?
- "select count(dp) from AssetDatapoint dp" :
- "select count(dp) from AssetDatapoint dp where dp.entityId = :assetId and dp.attributeName = :attributeName";
+ "select count(dp) from AssetDatapoint dp" :
+ "select count(dp) from AssetDatapoint dp where dp.entityId = :assetId and dp.attributeName = :attributeName";
TypedQuery<Long> query = entityManager.createQuery(
- queryStr,
- Long.class);
+ queryStr,
+ Long.class);
if (attributeRef != null) {
query
- .setParameter("assetId", attributeRef.getEntityId())
- .setParameter("attributeName", attributeRef.getAttributeName());
+ .setParameter("assetId", attributeRef.getEntityId())
+ .setParameter("attributeName", attributeRef.getAttributeName());
}
return query.getSingleResult();
@@ -125,119 +172,198 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
AttributeRef attributeRef = attribute.getReferenceOrThrow();
return persistenceService.doReturningTransaction(entityManager ->
- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<NumberDatapoint[]>() {
- @Override
- public NumberDatapoint[] execute(Connection connection) throws SQLException {
-
- String truncateX;
- String step;
- String interval;
- Function<Timestamp, String> labelFunction;
-
- SimpleDateFormat dayFormat = new SimpleDateFormat("dd. MMM yyyy");
- SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm");
- switch (datapointInterval) {
- case HOUR:
- truncateX = "minute";
- step = "1 minute";
- interval = "1 hour";
- labelFunction = timeFormat::format;
- break;
- case DAY:
- truncateX = "hour";
- step = "1 hour";
- interval = "1 day";
- labelFunction = timeFormat::format;
- break;
- case WEEK:
- truncateX = "day";
- step = "1 day";
- interval = "7 day";
- labelFunction = dayFormat::format;
- break;
- case MONTH:
- truncateX = "day";
- step = "1 day";
- interval = "1 month";
- labelFunction = dayFormat::format;
- break;
- case YEAR:
- truncateX = "month";
- step = "1 month";
- interval = "1 year";
- labelFunction = dayFormat::format;
- break;
- default:
- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);
- }
+ entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<NumberDatapoint[]>() {
+ @Override
+ public NumberDatapoint[] execute(Connection connection) throws SQLException {
+
+ String truncateX;
+ String step;
+ String interval;
+ Function<Timestamp, String> labelFunction;
+
+ SimpleDateFormat dayFormat = new SimpleDateFormat("dd. MMM yyyy");
+ SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm");
+ switch (datapointInterval) {
+ case HOUR:
+ truncateX = "minute";
+ step = "1 minute";
+ interval = "1 hour";
+ labelFunction = timeFormat::format;
+ break;
+ case DAY:
+ truncateX = "hour";
+ step = "1 hour";
+ interval = "1 day";
+ labelFunction = timeFormat::format;
+ break;
+ case WEEK:
+ truncateX = "day";
+ step = "1 day";
+ interval = "7 day";
+ labelFunction = dayFormat::format;
+ break;
+ case MONTH:
+ truncateX = "day";
+ step = "1 day";
+ interval = "1 month";
+ labelFunction = dayFormat::format;
+ break;
+ case YEAR:
+ truncateX = "month";
+ step = "1 month";
+ interval = "1 year";
+ labelFunction = dayFormat::format;
+ break;
+ default:
+ throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);
+ }
- StringBuilder query = new StringBuilder();
-
- query.append("select TS as X, coalesce(AVG_VALUE, null) as Y " +
- " from ( " +
- " select date_trunc(?, GS)::timestamp TS " +
- " from generate_series(to_timestamp(?) - ?, to_timestamp(?), ?) GS " +
- " ) TS " +
- " left join ( " +
- " select " +
- " date_trunc(?, to_timestamp(TIMESTAMP / 1000))::timestamp as TS, ");
-
- switch (attribute.getTypeOrThrow().getValueType()) {
- case NUMBER:
- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");
- break;
- case BOOLEAN:
- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");
- break;
- default:
- throw new IllegalArgumentException("Can't aggregate number datapoints for type of: " + attribute);
- }
+ StringBuilder query = new StringBuilder();
+
+ query.append("select TS as X, coalesce(AVG_VALUE, null) as Y " +
+ " from ( " +
+ " select date_trunc(?, GS)::timestamp TS " +
+ " from generate_series(to_timestamp(?) - ?, to_timestamp(?), ?) GS " +
+ " ) TS " +
+ " left join ( " +
+ " select " +
+ " date_trunc(?, to_timestamp(TIMESTAMP / 1000))::timestamp as TS, ");
+
+ switch (attribute.getTypeOrThrow().getValueType()) {
+ case NUMBER:
+ query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");
+ break;
+ case BOOLEAN:
+ query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");
+ break;
+ default:
+ throw new IllegalArgumentException("Can't aggregate number datapoints for type of: " + attribute);
+ }
- query.append(" from ASSET_DATAPOINT " +
- " where " +
- " to_timestamp(TIMESTAMP / 1000) >= to_timestamp(?) - ? " +
- " and " +
- " to_timestamp(TIMESTAMP / 1000) <= to_timestamp(?) " +
- " and " +
- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +
- " group by TS " +
- " ) DP using (TS) " +
- " order by TS asc "
- );
-
- try (PreparedStatement st = connection.prepareStatement(query.toString())) {
-
- long timestampSeconds = timestamp / 1000;
- st.setString(1, truncateX);
- st.setLong(2, timestampSeconds);
- st.setObject(3, new PGInterval(interval));
- st.setLong(4, timestampSeconds);
- st.setObject(5, new PGInterval(step));
- st.setString(6, truncateX);
- st.setLong(7, timestampSeconds);
- st.setObject(8, new PGInterval(interval));
- st.setLong(9, timestampSeconds);
- st.setString(10, attributeRef.getEntityId());
- st.setString(11, attributeRef.getAttributeName());
-
- try (ResultSet rs = st.executeQuery()) {
- List<NumberDatapoint> result = new ArrayList<>();
- while (rs.next()) {
- String label = labelFunction.apply(rs.getTimestamp(1));
- Number value = rs.getObject(2) != null ? rs.getDouble(2) : null;
- result.add(new NumberDatapoint(label, value));
+ query.append(" from ASSET_DATAPOINT " +
+ " where " +
+ " to_timestamp(TIMESTAMP / 1000) >= to_timestamp(?) - ? " +
+ " and " +
+ " to_timestamp(TIMESTAMP / 1000) <= to_timestamp(?) " +
+ " and " +
+ " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +
+ " group by TS " +
+ " ) DP using (TS) " +
+ " order by TS asc "
+ );
+
+ try (PreparedStatement st = connection.prepareStatement(query.toString())) {
+
+ long timestampSeconds = timestamp / 1000;
+ st.setString(1, truncateX);
+ st.setLong(2, timestampSeconds);
+ st.setObject(3, new PGInterval(interval));
+ st.setLong(4, timestampSeconds);
+ st.setObject(5, new PGInterval(step));
+ st.setString(6, truncateX);
+ st.setLong(7, timestampSeconds);
+ st.setObject(8, new PGInterval(interval));
+ st.setLong(9, timestampSeconds);
+ st.setString(10, attributeRef.getEntityId());
+ st.setString(11, attributeRef.getAttributeName());
+
+ try (ResultSet rs = st.executeQuery()) {
+ List<NumberDatapoint> result = new ArrayList<>();
+ while (rs.next()) {
+ String label = labelFunction.apply(rs.getTimestamp(1));
+ Number value = rs.getObject(2) != null ? rs.getDouble(2) : null;
+ result.add(new NumberDatapoint(label, value));
+ }
+ return result.toArray(new NumberDatapoint[result.size()]);
}
- return result.toArray(new NumberDatapoint[result.size()]);
}
}
- }
- })
+ })
);
}
+ protected void purgeDataPoints() {
+ LOG.info("Starting data points purge daily task");
+
+ // Get list of attributes that have custom durations
+ List<Asset> assets = assetStorageService.findAll(
+ new AssetQuery()
+ .attributeMeta(
+ new AttributeMetaPredicate(AssetMeta.DATA_POINTS_MAX_AGE_DAYS),
+ new AttributeMetaPredicate(AssetMeta.STORE_DATA_POINTS))
+ .select(new BaseAssetQuery.Select(BaseAssetQuery.Include.ONLY_ID_AND_NAME_AND_ATTRIBUTES)));
+
+ List<AssetAttribute> attributes = assets.stream()
+ .map(asset -> asset
+ .getAttributesStream()
+ .filter(assetAttribute ->
+ assetAttribute.isStoreDatapoints()
+ && assetAttribute.hasMetaItem(AssetMeta.DATA_POINTS_MAX_AGE_DAYS))
+ .collect(toList()))
+ .flatMap(List::stream)
+ .collect(toList());
+
+ // Purge data points not in the above list using default duration
+ LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);
+
+ persistenceService.doTransaction(em -> em.createQuery(
+ "delete from AssetDatapoint dp " +
+ "where dp.timestamp < :dt" + buildWhereClause(attributes, true)
+ ).setParameter("dt", 1000L * timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS).getEpochSecond()).executeUpdate());
+
+ if (!attributes.isEmpty()) {
+ // Purge data points that have specific age constraints
+ Map<Integer, List<AssetAttribute>> ageAttributeRefMap = attributes.stream()
+ .collect(groupingBy(attribute ->
+ attribute
+ .getMetaItem(AssetMeta.DATA_POINTS_MAX_AGE_DAYS)
+ .flatMap(metaItem ->
+ Values.getIntegerCoerced(metaItem.getValue().orElse(null)))
+ .orElse(maxDatapointAgeDays)));
+
+ ageAttributeRefMap.forEach((age, attrs) -> {
+ LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);
+
+ try {
+ persistenceService.doTransaction(em -> em.createQuery(
+ "delete from AssetDatapoint dp " +
+ "where dp.timestamp < :dt" + buildWhereClause(attrs, false)
+ ).setParameter("dt", 1000L * timerService.getNow().truncatedTo(DAYS).minus(age, DAYS).getEpochSecond()).executeUpdate());
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);
+ }
+ });
+ }
+
+ LOG.info("Finished data points purge daily task");
+ }
+
+ protected String buildWhereClause(List<AssetAttribute> attributes, boolean negate) {
+
+ if (attributes.isEmpty()) {
+ return "";
+ }
+
+ String whereStr = attributes.stream()
+ .map(assetAttribute -> {
+ AttributeRef attributeRef = assetAttribute.getReferenceOrThrow();
+ return "('" + attributeRef.getEntityId() + "','" + attributeRef.getAttributeName() + "')";
+ })
+ .collect(Collectors.joining(","));
+
+ return " and (dp.entityId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";
+ }
+
+ protected long getFirstRunMillis(Instant currentTime) {
+ // Schedule purge at approximately 3AM daily
+ return ChronoUnit.MILLIS.between(
+ currentTime,
+ currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));
+ }
+
@Override
public String toString() {
return getClass().getSimpleName() + "{" +
- '}';
+ '}';
}
}
diff --git a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java b/manager/src/main/java/org/openremote/manager/notification/NotificationService.java
index c9d0b3fe9..8c51c04fa 100644
--- a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java
+++ b/manager/src/main/java/org/openremote/manager/notification/NotificationService.java
@@ -44,8 +44,7 @@ import org.openremote.model.util.TimeUtil;
import javax.persistence.Query;
import javax.persistence.TypedQuery;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
+import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalField;
import java.time.temporal.WeekFields;
@@ -56,11 +55,10 @@ import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import static java.time.temporal.ChronoUnit.*;
import static org.openremote.manager.notification.NotificationProcessingException.Reason.*;
import static org.openremote.model.notification.Notification.HEADER_SOURCE;
-import static org.openremote.model.notification.Notification.Source.ASSET_RULESET;
-import static org.openremote.model.notification.Notification.Source.CLIENT;
-import static org.openremote.model.notification.Notification.Source.INTERNAL;
+import static org.openremote.model.notification.Notification.Source.*;
// TODO Implement notification purging - configurable MAX_AGE for notifications?
public class NotificationService extends RouteBuilder implements ContainerService {
@@ -294,7 +292,7 @@ public class NotificationService extends RouteBuilder implements ContainerServic
.setSourceId(sourceId.get())
.setTarget(targets.getType())
.setTargetId(targetId)
- .setSentOn(Date.from(timerService.getNow().toInstant()));
+ .setSentOn(Date.from(timerService.getNow()));
sentNotification = em.merge(sentNotification);
long id = sentNotification.getId();
@@ -499,8 +497,8 @@ public class NotificationService extends RouteBuilder implements ContainerServic
}
}
- protected ZonedDateTime getRepeatAfterTimestamp(Notification notification, ZonedDateTime lastSend) {
- ZonedDateTime timestamp = null;
+ protected Instant getRepeatAfterTimestamp(Notification notification, Instant lastSend) {
+ Instant timestamp = null;
if (TextUtil.isNullOrEmpty(notification.getName())) {
return null;
@@ -511,19 +509,19 @@ public class NotificationService extends RouteBuilder implements ContainerServic
switch (notification.getRepeatFrequency()) {
case HOURLY:
- timestamp = lastSend.truncatedTo(ChronoUnit.HOURS).plusHours(1);
+ timestamp = lastSend.truncatedTo(HOURS).plus(1, HOURS);
break;
case DAILY:
- timestamp = lastSend.truncatedTo(ChronoUnit.DAYS).plusDays(1);
+ timestamp = lastSend.truncatedTo(DAYS).plus(1, DAYS);
break;
case WEEKLY:
- timestamp = lastSend.truncatedTo(ChronoUnit.WEEKS).plusWeeks(1);
+ timestamp = lastSend.truncatedTo(WEEKS).plus(1, WEEKS);
break;
case MONTHLY:
- timestamp = lastSend.truncatedTo(ChronoUnit.MONTHS).plusMonths(1);
+ timestamp = lastSend.truncatedTo(MONTHS).plus(1, MONTHS);
break;
case ANNUALLY:
- timestamp = lastSend.truncatedTo(ChronoUnit.YEARS).plusYears(1);
+ timestamp = lastSend.truncatedTo(YEARS).plus(1, YEARS);
break;
}
} else if (!TextUtil.isNullOrEmpty(notification.getRepeatInterval())) {
@@ -610,7 +608,6 @@ public class NotificationService extends RouteBuilder implements ContainerServic
return lastSend == null ||
(notification.getRepeatFrequency() != RepeatFrequency.ONCE &&
- timerService.getNow().plusSeconds(1).isAfter(getRepeatAfterTimestamp(notification, ZonedDateTime.ofInstant(lastSend.toInstant(),
- ZoneId.systemDefault()))));
+ timerService.getNow().plusSeconds(1).isAfter(getRepeatAfterTimestamp(notification, lastSend.toInstant())));
}
}
\ No newline at end of file
diff --git a/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java b/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java
index e0ac7d27b..53cea7961 100644
--- a/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java
+++ b/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java
@@ -21,8 +21,6 @@ package org.openremote.manager.setup.builtin;
import org.openremote.agent.protocol.simulator.SimulatorProtocol;
import org.openremote.container.Container;
-import org.openremote.manager.asset.console.ConsoleResourceImpl;
-import org.openremote.manager.rules.geofence.ORConsoleGeofenceAssetAdapter;
import org.openremote.manager.security.UserConfiguration;
import org.openremote.manager.setup.AbstractManagerSetup;
import org.openremote.model.asset.Asset;
@@ -30,8 +28,6 @@ import org.openremote.model.asset.AssetAttribute;
import org.openremote.model.asset.AssetMeta;
import org.openremote.model.asset.UserAsset;
import org.openremote.model.attribute.*;
-import org.openremote.model.console.ConsoleConfiguration;
-import org.openremote.model.console.ConsoleProvider;
import org.openremote.model.geo.GeoJSONPoint;
import org.openremote.model.security.Tenant;
import org.openremote.model.simulator.element.ColorSimulatorElement;
@@ -44,7 +40,6 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import static org.openremote.model.asset.AssetMeta.*;
@@ -68,8 +63,8 @@ public class ManagerDemoSetup extends AbstractManagerSetup {
public static final int DEMO_RULE_STATES_GLOBAL_WITH_SCENES = DEMO_RULE_STATES_CUSTOMER_A_WITH_SCENES;
public static GeoJSONPoint SMART_OFFICE_LOCATION = new GeoJSONPoint(5.460315214821094, 51.44541688237109);
public static GeoJSONPoint SMART_HOME_LOCATION = new GeoJSONPoint(5.470945, 51.438000);
- public final String agentProtocolConfigName = "simulator123";
- public final String thingLightToggleAttributeName = "light1Toggle";
+ public static final String agentProtocolConfigName = "simulator123";
+ public static final String thingLightToggleAttributeName = "light1Toggle";
final protected boolean importDemoScenes;
public String smartOfficeId;
public String groundFloorId;
@@ -187,6 +182,10 @@ public class ManagerDemoSetup extends AbstractManagerSetup {
new MetaItem(
STORE_DATA_POINTS,
Values.create(true)),
+ new MetaItem(
+ DATA_POINTS_MAX_AGE_DAYS,
+ Values.create(7)
+ ),
new MetaItem(
AGENT_LINK,
new AttributeRef(agent.getId(), agentProtocolConfigName).toArrayValue()),
diff --git a/model/src/main/java/org/openremote/model/asset/AssetMeta.java b/model/src/main/java/org/openremote/model/asset/AssetMeta.java
index 5a8dd863d..5ad9e3c8e 100644
--- a/model/src/main/java/org/openremote/model/asset/AssetMeta.java
+++ b/model/src/main/java/org/openremote/model/asset/AssetMeta.java
@@ -34,6 +34,7 @@ import static org.openremote.model.Constants.ASSET_META_NAMESPACE;
import static org.openremote.model.attribute.MetaItem.MetaItemFailureReason.META_ITEM_VALUE_MISMATCH;
import static org.openremote.model.attribute.MetaItemDescriptor.Access.ACCESS_PRIVATE;
import static org.openremote.model.util.TextUtil.REGEXP_PATTERN_DOUBLE;
+import static org.openremote.model.util.TextUtil.REGEXP_PATTERN_INTEGER_POSITIVE_NON_ZERO;
/**
* Asset attribute meta item name is an arbitrary string. It should be URI. This enum contains the well-known URIs for
@@ -288,6 +289,20 @@ public enum AssetMeta implements MetaItemDescriptor {
Values.create(true),
true),
+ /**
+ * How long to store attribute values in time series database (data older than this will be automatically purged)
+ */
+ DATA_POINTS_MAX_AGE_DAYS(
+ ASSET_META_NAMESPACE + ":dataPointsMaxAgeDays",
+ new Access(true, false, true),
+ ValueType.NUMBER,
+ REGEXP_PATTERN_INTEGER_POSITIVE_NON_ZERO,
+ PatternFailure.INTEGER_POSITIVE_NON_ZERO.name(),
+ null,
+ false,
+ value -> value == null ? Optional.empty() : Values.getIntegerCoerced(value)
+ .isPresent() ? Optional.empty() : Optional.of(new ValidationFailure(META_ITEM_VALUE_MISMATCH, "Integer"))),
+
/**
* Should attribute writes be processed by the rules engines as {@link AssetState} facts, with a lifecycle that
* reflects the state of the asset attribute. Each attribute will have one fact at all times in rules memory. These
diff --git a/profile/deploy.yml b/profile/deploy.yml
index 6ef60b67b..28fd75285 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -220,6 +220,10 @@ services:
# unless you want to deploy a modified version of the Manager UI and JavaScript components.
# MANAGER_DOCROOT: 'client/src/main/webapp'
+ # Configure the default max age for data points stored in the database (individual attributes can use a custom
+ # value by using the DATA_POINTS_MAX_AGE_DAYS AssetMeta item).
+ # DATA_POINTS_MAX_AGE_DAYS = 30
+
# Custom JVM options (increase heap size if required)
JAVA_OPTS: '-Xms400m -Xmx400m
-XX:CompressedClassSpaceSize=25m -XX:MaxMetaspaceSize=150m
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
index d44f43810..8d1903768 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
@@ -12,18 +12,24 @@ import org.openremote.test.ManagerContainerTrait
import spock.lang.Specification
import spock.util.concurrent.PollingConditions
+import java.time.Instant
+import java.util.concurrent.TimeUnit
+
+import static java.util.concurrent.TimeUnit.HOURS
import static java.util.concurrent.TimeUnit.SECONDS
+import static org.openremote.manager.datapoint.AssetDatapointService.DATA_POINTS_MAX_AGE_DAYS
+import static org.openremote.manager.setup.builtin.ManagerDemoSetup.thingLightToggleAttributeName
class AssetDatapointTest extends Specification implements ManagerContainerTrait {
- def "Receive number sensor values and store asset datapoints"() {
+ def "Test number and toggle attribute storage, retrieval and purging"() {
given: "expected conditions"
def conditions = new PollingConditions(timeout: 10, initialDelay: 1, delay: 1)
- when: "the demo agent and thing have been deployed"
+ when: "the demo agent and thing have been deployed with a DATA_POINTS_MAX_AGE_DAYS value of 1"
def serverPort = findEphemeralPort()
- def container = startContainerWithPseudoClock(defaultConfig(serverPort), defaultServices())
+ def container = startContainerWithPseudoClock defaultConfig(serverPort) << [(DATA_POINTS_MAX_AGE_DAYS): "1"], defaultServices()
def managerDemoSetup = container.getService(SetupService.class).getTaskOfType(ManagerDemoSetup.class)
def simulatorProtocol = container.getService(SimulatorProtocol.class)
def assetStorageService = container.getService(AssetStorageService.class)
@@ -39,6 +45,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
advancePseudoClock(10, SECONDS, container)
def datapoint1ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(13.3d))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
@@ -49,6 +56,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
advancePseudoClock(10, SECONDS, container)
def datapoint2ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(14.4d))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
@@ -59,6 +67,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
advancePseudoClock(10, SECONDS, container)
def datapoint3ExpectedTimestamp = getClockTimeOf(container)
simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(15.5d))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
@@ -68,6 +77,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
when: "a simulated sensor receives a new value"
advancePseudoClock(10, SECONDS, container)
simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", null) // No value!
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
@@ -103,62 +113,46 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
assert aggregatedDatapoints.size() == 61
}
- cleanup: "the server should be stopped"
- stopContainer(container)
- }
-
- def "Receive boolean sensor values and store asset datapoints"() {
-
- given: "expected conditions"
- def conditions = new PollingConditions(timeout: 10, initialDelay: 1, delay: 1)
+ // ------------------------------------
+ // Test boolean data point storage
+ // ------------------------------------
- when: "the demo agent and thing have been deployed"
- def serverPort = findEphemeralPort()
- def container = startContainerWithPseudoClock(defaultConfig(serverPort), defaultServices())
- def managerDemoSetup = container.getService(SetupService.class).getTaskOfType(ManagerDemoSetup.class)
- def simulatorProtocol = container.getService(SimulatorProtocol.class)
- def assetStorageService = container.getService(AssetStorageService.class)
- def assetDatapointService = container.getService(AssetDatapointService.class)
-
- then: "the simulator elements should have the initial state"
- conditions.eventually {
- def state = simulatorProtocol.getValue(managerDemoSetup.thingId, "light1Toggle")
- assert Values.getBoolean(state.orElse(null)).orElse(null)
- }
-
- when: "a simulated sensor receives a new value"
+ when: "a simulated boolean sensor receives a new value"
advancePseudoClock(10, SECONDS, container)
- def datapoint1ExpectedTimestamp = getClockTimeOf(container)
- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(false))
+ datapoint1ExpectedTimestamp = getClockTimeOf(container)
+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(false))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
- assert !thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)
+ assert !thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)
}
when: "a simulated sensor receives a new value"
advancePseudoClock(10, SECONDS, container)
- def datapoint2ExpectedTimestamp = getClockTimeOf(container)
- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(true))
+ datapoint2ExpectedTimestamp = getClockTimeOf(container)
+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(true))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
- assert thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)
+ assert thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)
}
when: "a simulated sensor receives a new value"
advancePseudoClock(10, SECONDS, container)
- def datapoint3ExpectedTimestamp = getClockTimeOf(container)
- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(false))
+ datapoint3ExpectedTimestamp = getClockTimeOf(container)
+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(false))
+
then: "the attribute should be updated"
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
- assert !thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)
+ assert !thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)
}
expect: "the datapoints to be stored"
conditions.eventually {
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1Toggle"))
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
assert datapoints.size() == 3
assert !Values.getBoolean(datapoints.get(0).value).orElse(null)
@@ -175,13 +169,102 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
conditions.eventually {
def thing = assetStorageService.find(managerDemoSetup.thingId, true)
def aggregatedDatapoints = assetDatapointService.aggregateDatapoints(
- thing.getAttribute("light1Toggle").orElseThrow({ new RuntimeException("Missing attribute")}),
+ thing.getAttribute(thingLightToggleAttributeName).orElseThrow({ new RuntimeException("Missing attribute")}),
DatapointInterval.HOUR,
getClockTimeOf(container)
)
assert aggregatedDatapoints.size() == 61
}
+ // ------------------------------------
+ // Test purging of data points
+ // ------------------------------------
+
+ when: "time moves forward by more than a day"
+ advancePseudoClock(26, HOURS, container)
+
+ and: "the power sensor with default max age receives a new value"
+ def datapoint4ExpectedTimestamp = getClockTimeOf(container)
+ simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(17.5d))
+
+ and: "the toggle sensor with a custom max age of 7 days receives a new value"
+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(true))
+
+ then: "the datapoints should be stored"
+ conditions.eventually {
+ def powerDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))
+ def toggleDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
+
+ assert powerDatapoints.size() > 4
+ assert Values.getNumber(powerDatapoints.get(0).value).orElse(null) == 17.5d
+ assert powerDatapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+
+ assert toggleDatapoints.size() == 4
+ assert Values.getBoolean(toggleDatapoints.get(0).value).orElse(false)
+ assert toggleDatapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+ }
+
+ when: "the daily data point purge routine executes the next day"
+ // Move clock to next days purge runtime
+ advancePseudoClock(assetDatapointService.getFirstRunMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)
+ assetDatapointService.purgeDataPoints()
+
+ then: "data points older than 1 day should be purged for the power sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))
+ assert datapoints.size() == 1
+ assert Values.getNumber(datapoints.get(0).value).orElse(null) == 17.5d
+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+ }
+
+ and: "no data points should have been purged for the toggle sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
+ assert datapoints.size() == 4
+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)
+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+ }
+
+ when: "the daily data point purge routine executes 3 days later"
+ advancePseudoClock(3, TimeUnit.DAYS, container)
+ assetDatapointService.purgeDataPoints()
+
+ then: "all data points should be purged for power sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))
+ assert datapoints.isEmpty()
+ }
+
+ and: "no data points should have been purged for the toggle sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
+ assert datapoints.size() == 4
+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)
+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+ }
+
+ when: "the daily data point purge routine executes 3 days later"
+ advancePseudoClock(3, TimeUnit.DAYS, container)
+ assetDatapointService.purgeDataPoints()
+
+ then: "data points older than 7 days should have been purged for the toggle sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
+ assert datapoints.size() == 1
+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)
+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
+ }
+
+ when: "the daily data point purge routine executes 1 day later"
+ advancePseudoClock(1, TimeUnit.DAYS, container)
+ assetDatapointService.purgeDataPoints()
+
+ then: "all data points should have been purged for the toggle sensor"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))
+ assert datapoints.isEmpty()
+ }
+
cleanup: "the server should be stopped"
stopContainer(container)
}
diff --git a/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy b/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy
index 425890280..8a2d7562b 100644
--- a/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy
@@ -418,11 +418,11 @@ class NotificationTest extends Specification implements ManagerContainerTrait {
when: "a repeat frequency is set and a notification is sent"
// Move clock to beginning of next hour
- def advancement = Instant.ofEpochMilli(getClockTimeOf(container)).atZone(ZoneId.systemDefault())
+ def advancement = Instant.ofEpochMilli(getClockTimeOf(container))
.truncatedTo(ChronoUnit.HOURS)
- .plusMinutes(59)
+ .plus(59, ChronoUnit.MINUTES)
- advancePseudoClock(ChronoUnit.MILLIS.between(Instant.ofEpochMilli(getClockTimeOf(container)).atZone(ZoneId.systemDefault()), advancement), TimeUnit.MILLISECONDS, container)
+ advancePseudoClock(ChronoUnit.MILLIS.between(Instant.ofEpochMilli(getClockTimeOf(container)), advancement), TimeUnit.MILLISECONDS, container)
notification.setRepeatFrequency(RepeatFrequency.HOURLY)
testuser3NotificationResource.sendNotification(null, notification)
notifications = []