100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Added auto purging functionality to asset data point service with tests
Rich Turner5d3b04fab75c
diff --git a/container/src/main/java/org/openremote/container/timer/TimerService.java b/container/src/main/java/org/openremote/container/timer/TimerService.javaindex 5aa3afc6e..f4d0b3824 100644--- a/container/src/main/java/org/openremote/container/timer/TimerService.java+++ b/container/src/main/java/org/openremote/container/timer/TimerService.java@@ -24,7 +24,6 @@ import org.openremote.container.ContainerService;import java.time.Instant;import java.time.ZoneId;-import java.time.ZonedDateTime;import java.util.Date;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicLong;@@ -113,8 +112,8 @@ public class TimerService implements ContainerService {return getClock().getCurrentTimeMillis();}- public ZonedDateTime getNow() {- return Instant.ofEpochMilli(getCurrentTimeMillis()).atZone(ZoneId.systemDefault());+ public Instant getNow() {+ return Instant.ofEpochMilli(getCurrentTimeMillis());}@Overridediff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 6ec010e25..3d0a6a431 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -10,56 +10,103 @@ import org.openremote.container.web.WebService;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.asset.AssetUpdateProcessor;+import org.openremote.manager.concurrent.ManagerExecutorService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.model.asset.Asset;import org.openremote.model.asset.AssetAttribute;+import org.openremote.model.asset.AssetMeta;import org.openremote.model.attribute.AttributeEvent.Source;import org.openremote.model.attribute.AttributeRef;import org.openremote.model.datapoint.AssetDatapoint;import org.openremote.model.datapoint.Datapoint;import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.NumberDatapoint;+import org.openremote.model.query.AssetQuery;+import org.openremote.model.query.BaseAssetQuery;+import org.openremote.model.query.filter.AttributeMetaPredicate;+import org.openremote.model.value.Values;import org.postgresql.util.PGInterval;import javax.persistence.EntityManager;import javax.persistence.TypedQuery;import java.sql.*;import java.text.SimpleDateFormat;+import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.List;+import java.util.Map;+import java.util.concurrent.ScheduledFuture;import java.util.function.Function;+import java.util.logging.Level;import java.util.logging.Logger;+import java.util.stream.Collectors;++import static java.time.temporal.ChronoUnit.DAYS;+import static java.util.stream.Collectors.groupingBy;+import static java.util.stream.Collectors.toList;/**- * Store and retrieve datapoints for asset attributes.+ * Store and retrieve datapoints for asset attributes and periodically purge data points based on+ * {@link org.openremote.model.asset.AssetMeta#DATA_POINTS_MAX_AGE_DAYS} {@link org.openremote.model.attribute.MetaItem}+ * and {@link #DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #DATA_POINTS_MAX_AGE_DAYS_DEFAULT}+ * days.*/public class AssetDatapointService implements ContainerService, AssetUpdateProcessor {+ public static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";+ public static final String DATA_POINTS_MAX_AGE_DAYS_DEFAULT = "30";private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());-protected PersistenceService persistenceService;+ protected AssetStorageService assetStorageService;+ protected TimerService timerService;+ protected ManagerExecutorService managerExecutorService;+ protected int maxDatapointAgeDays;+ protected ScheduledFuture dataPointsPurgeScheduledFuture;@Overridepublic void init(Container container) throws Exception {persistenceService = container.getService(PersistenceService.class);+ assetStorageService = container.getService(AssetStorageService.class);+ timerService = container.getService(TimerService.class);+ managerExecutorService = container.getService(ManagerExecutorService.class);container.getService(WebService.class).getApiSingletons().add(- new AssetDatapointResourceImpl(- container.getService(TimerService.class),- container.getService(ManagerIdentityService.class),- container.getService(AssetStorageService.class),- this- )+ new AssetDatapointResourceImpl(+ container.getService(TimerService.class),+ container.getService(ManagerIdentityService.class),+ container.getService(AssetStorageService.class),+ this+ )+ );++ maxDatapointAgeDays = Integer.parseInt(+ container.getConfig().getOrDefault(DATA_POINTS_MAX_AGE_DAYS, DATA_POINTS_MAX_AGE_DAYS_DEFAULT));+ if (maxDatapointAgeDays <= 0) {+ LOG.warning(DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");+ }}@Overridepublic void start(Container container) throws Exception {+ if (maxDatapointAgeDays > 0) {+ long period = 24L * 3600L * 1000L;++ dataPointsPurgeScheduledFuture = managerExecutorService.scheduleAtFixedRate(+ this::purgeDataPoints,+ getFirstRunMillis(timerService.getNow()),+ period);++ }}@Overridepublic void stop(Container container) throws Exception {+ if (dataPointsPurgeScheduledFuture != null) {+ dataPointsPurgeScheduledFuture.cancel(true);+ }}@Override@@ -68,9 +115,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProceAssetAttribute attribute,Source source) throws AssetProcessingException {if (Datapoint.isDatapointsCapable(attribute)- && attribute.isStoreDatapoints()- && attribute.getStateEvent().isPresent()- && attribute.getStateEvent().get().getValue().isPresent()) { // Don't store datapoints with null value+ && attribute.isStoreDatapoints()+ && attribute.getStateEvent().isPresent()+ && attribute.getStateEvent().get().getValue().isPresent()) { // Don't store datapoints with null valueLOG.finest("Storing datapoint for: " + attribute);AssetDatapoint assetDatapoint = new AssetDatapoint(attribute.getStateEvent().get());em.persist(assetDatapoint);@@ -80,15 +127,15 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProcepublic List<AssetDatapoint> getDatapoints(AttributeRef attributeRef) {return persistenceService.doReturningTransaction(entityManager ->- entityManager.createQuery(- "select dp from AssetDatapoint dp " +- "where dp.entityId = :assetId " +- "and dp.attributeName = :attributeName " +- "order by dp.timestamp desc",- AssetDatapoint.class)- .setParameter("assetId", attributeRef.getEntityId())- .setParameter("attributeName", attributeRef.getAttributeName())- .getResultList());+ entityManager.createQuery(+ "select dp from AssetDatapoint dp " ++ "where dp.entityId = :assetId " ++ "and dp.attributeName = :attributeName " ++ "order by dp.timestamp desc",+ AssetDatapoint.class)+ .setParameter("assetId", attributeRef.getEntityId())+ .setParameter("attributeName", attributeRef.getAttributeName())+ .getResultList());}@@ -100,17 +147,17 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProcereturn persistenceService.doReturningTransaction(entityManager -> {String queryStr = attributeRef == null ?- "select count(dp) from AssetDatapoint dp" :- "select count(dp) from AssetDatapoint dp where dp.entityId = :assetId and dp.attributeName = :attributeName";+ "select count(dp) from AssetDatapoint dp" :+ "select count(dp) from AssetDatapoint dp where dp.entityId = :assetId and dp.attributeName = :attributeName";TypedQuery<Long> query = entityManager.createQuery(- queryStr,- Long.class);+ queryStr,+ Long.class);if (attributeRef != null) {query- .setParameter("assetId", attributeRef.getEntityId())- .setParameter("attributeName", attributeRef.getAttributeName());+ .setParameter("assetId", attributeRef.getEntityId())+ .setParameter("attributeName", attributeRef.getAttributeName());}return query.getSingleResult();@@ -125,119 +172,198 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProceAttributeRef attributeRef = attribute.getReferenceOrThrow();return persistenceService.doReturningTransaction(entityManager ->- entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<NumberDatapoint[]>() {- @Override- public NumberDatapoint[] execute(Connection connection) throws SQLException {-- String truncateX;- String step;- String interval;- Function<Timestamp, String> labelFunction;-- SimpleDateFormat dayFormat = new SimpleDateFormat("dd. MMM yyyy");- SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm");- switch (datapointInterval) {- case HOUR:- truncateX = "minute";- step = "1 minute";- interval = "1 hour";- labelFunction = timeFormat::format;- break;- case DAY:- truncateX = "hour";- step = "1 hour";- interval = "1 day";- labelFunction = timeFormat::format;- break;- case WEEK:- truncateX = "day";- step = "1 day";- interval = "7 day";- labelFunction = dayFormat::format;- break;- case MONTH:- truncateX = "day";- step = "1 day";- interval = "1 month";- labelFunction = dayFormat::format;- break;- case YEAR:- truncateX = "month";- step = "1 month";- interval = "1 year";- labelFunction = dayFormat::format;- break;- default:- throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);- }+ entityManager.unwrap(Session.class).doReturningWork(new AbstractReturningWork<NumberDatapoint[]>() {+ @Override+ public NumberDatapoint[] execute(Connection connection) throws SQLException {++ String truncateX;+ String step;+ String interval;+ Function<Timestamp, String> labelFunction;++ SimpleDateFormat dayFormat = new SimpleDateFormat("dd. MMM yyyy");+ SimpleDateFormat timeFormat = new SimpleDateFormat("HH:mm");+ switch (datapointInterval) {+ case HOUR:+ truncateX = "minute";+ step = "1 minute";+ interval = "1 hour";+ labelFunction = timeFormat::format;+ break;+ case DAY:+ truncateX = "hour";+ step = "1 hour";+ interval = "1 day";+ labelFunction = timeFormat::format;+ break;+ case WEEK:+ truncateX = "day";+ step = "1 day";+ interval = "7 day";+ labelFunction = dayFormat::format;+ break;+ case MONTH:+ truncateX = "day";+ step = "1 day";+ interval = "1 month";+ labelFunction = dayFormat::format;+ break;+ case YEAR:+ truncateX = "month";+ step = "1 month";+ interval = "1 year";+ labelFunction = dayFormat::format;+ break;+ default:+ throw new IllegalArgumentException("Can't handle interval: " + datapointInterval);+ }- StringBuilder query = new StringBuilder();-- query.append("select TS as X, coalesce(AVG_VALUE, null) as Y " +- " from ( " +- " select date_trunc(?, GS)::timestamp TS " +- " from generate_series(to_timestamp(?) - ?, to_timestamp(?), ?) GS " +- " ) TS " +- " left join ( " +- " select " +- " date_trunc(?, to_timestamp(TIMESTAMP / 1000))::timestamp as TS, ");-- switch (attribute.getTypeOrThrow().getValueType()) {- case NUMBER:- query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");- break;- case BOOLEAN:- query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");- break;- default:- throw new IllegalArgumentException("Can't aggregate number datapoints for type of: " + attribute);- }+ StringBuilder query = new StringBuilder();++ query.append("select TS as X, coalesce(AVG_VALUE, null) as Y " ++ " from ( " ++ " select date_trunc(?, GS)::timestamp TS " ++ " from generate_series(to_timestamp(?) - ?, to_timestamp(?), ?) GS " ++ " ) TS " ++ " left join ( " ++ " select " ++ " date_trunc(?, to_timestamp(TIMESTAMP / 1000))::timestamp as TS, ");++ switch (attribute.getTypeOrThrow().getValueType()) {+ case NUMBER:+ query.append(" AVG(VALUE::text::numeric) as AVG_VALUE ");+ break;+ case BOOLEAN:+ query.append(" AVG(case when VALUE::text::boolean is true then 1 else 0 end) as AVG_VALUE ");+ break;+ default:+ throw new IllegalArgumentException("Can't aggregate number datapoints for type of: " + attribute);+ }- query.append(" from ASSET_DATAPOINT " +- " where " +- " to_timestamp(TIMESTAMP / 1000) >= to_timestamp(?) - ? " +- " and " +- " to_timestamp(TIMESTAMP / 1000) <= to_timestamp(?) " +- " and " +- " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " +- " group by TS " +- " ) DP using (TS) " +- " order by TS asc "- );-- try (PreparedStatement st = connection.prepareStatement(query.toString())) {-- long timestampSeconds = timestamp / 1000;- st.setString(1, truncateX);- st.setLong(2, timestampSeconds);- st.setObject(3, new PGInterval(interval));- st.setLong(4, timestampSeconds);- st.setObject(5, new PGInterval(step));- st.setString(6, truncateX);- st.setLong(7, timestampSeconds);- st.setObject(8, new PGInterval(interval));- st.setLong(9, timestampSeconds);- st.setString(10, attributeRef.getEntityId());- st.setString(11, attributeRef.getAttributeName());-- try (ResultSet rs = st.executeQuery()) {- List<NumberDatapoint> result = new ArrayList<>();- while (rs.next()) {- String label = labelFunction.apply(rs.getTimestamp(1));- Number value = rs.getObject(2) != null ? rs.getDouble(2) : null;- result.add(new NumberDatapoint(label, value));+ query.append(" from ASSET_DATAPOINT " ++ " where " ++ " to_timestamp(TIMESTAMP / 1000) >= to_timestamp(?) - ? " ++ " and " ++ " to_timestamp(TIMESTAMP / 1000) <= to_timestamp(?) " ++ " and " ++ " ENTITY_ID = ? and ATTRIBUTE_NAME = ? " ++ " group by TS " ++ " ) DP using (TS) " ++ " order by TS asc "+ );++ try (PreparedStatement st = connection.prepareStatement(query.toString())) {++ long timestampSeconds = timestamp / 1000;+ st.setString(1, truncateX);+ st.setLong(2, timestampSeconds);+ st.setObject(3, new PGInterval(interval));+ st.setLong(4, timestampSeconds);+ st.setObject(5, new PGInterval(step));+ st.setString(6, truncateX);+ st.setLong(7, timestampSeconds);+ st.setObject(8, new PGInterval(interval));+ st.setLong(9, timestampSeconds);+ st.setString(10, attributeRef.getEntityId());+ st.setString(11, attributeRef.getAttributeName());++ try (ResultSet rs = st.executeQuery()) {+ List<NumberDatapoint> result = new ArrayList<>();+ while (rs.next()) {+ String label = labelFunction.apply(rs.getTimestamp(1));+ Number value = rs.getObject(2) != null ? rs.getDouble(2) : null;+ result.add(new NumberDatapoint(label, value));+ }+ return result.toArray(new NumberDatapoint[result.size()]);}- return result.toArray(new NumberDatapoint[result.size()]);}}- }- })+ }));}+ protected void purgeDataPoints() {+ LOG.info("Starting data points purge daily task");++ // Get list of attributes that have custom durations+ List<Asset> assets = assetStorageService.findAll(+ new AssetQuery()+ .attributeMeta(+ new AttributeMetaPredicate(AssetMeta.DATA_POINTS_MAX_AGE_DAYS),+ new AttributeMetaPredicate(AssetMeta.STORE_DATA_POINTS))+ .select(new BaseAssetQuery.Select(BaseAssetQuery.Include.ONLY_ID_AND_NAME_AND_ATTRIBUTES)));++ List<AssetAttribute> attributes = assets.stream()+ .map(asset -> asset+ .getAttributesStream()+ .filter(assetAttribute ->+ assetAttribute.isStoreDatapoints()+ && assetAttribute.hasMetaItem(AssetMeta.DATA_POINTS_MAX_AGE_DAYS))+ .collect(toList()))+ .flatMap(List::stream)+ .collect(toList());++ // Purge data points not in the above list using default duration+ LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);++ persistenceService.doTransaction(em -> em.createQuery(+ "delete from AssetDatapoint dp " ++ "where dp.timestamp < :dt" + buildWhereClause(attributes, true)+ ).setParameter("dt", 1000L * timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS).getEpochSecond()).executeUpdate());++ if (!attributes.isEmpty()) {+ // Purge data points that have specific age constraints+ Map<Integer, List<AssetAttribute>> ageAttributeRefMap = attributes.stream()+ .collect(groupingBy(attribute ->+ attribute+ .getMetaItem(AssetMeta.DATA_POINTS_MAX_AGE_DAYS)+ .flatMap(metaItem ->+ Values.getIntegerCoerced(metaItem.getValue().orElse(null)))+ .orElse(maxDatapointAgeDays)));++ ageAttributeRefMap.forEach((age, attrs) -> {+ LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);++ try {+ persistenceService.doTransaction(em -> em.createQuery(+ "delete from AssetDatapoint dp " ++ "where dp.timestamp < :dt" + buildWhereClause(attrs, false)+ ).setParameter("dt", 1000L * timerService.getNow().truncatedTo(DAYS).minus(age, DAYS).getEpochSecond()).executeUpdate());+ } catch (Exception e) {+ LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);+ }+ });+ }++ LOG.info("Finished data points purge daily task");+ }++ protected String buildWhereClause(List<AssetAttribute> attributes, boolean negate) {++ if (attributes.isEmpty()) {+ return "";+ }++ String whereStr = attributes.stream()+ .map(assetAttribute -> {+ AttributeRef attributeRef = assetAttribute.getReferenceOrThrow();+ return "('" + attributeRef.getEntityId() + "','" + attributeRef.getAttributeName() + "')";+ })+ .collect(Collectors.joining(","));++ return " and (dp.entityId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";+ }++ protected long getFirstRunMillis(Instant currentTime) {+ // Schedule purge at approximately 3AM daily+ return ChronoUnit.MILLIS.between(+ currentTime,+ currentTime.truncatedTo(DAYS).plus(27, ChronoUnit.HOURS));+ }+@Overridepublic String toString() {return getClass().getSimpleName() + "{" +- '}';+ '}';}}diff --git a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java b/manager/src/main/java/org/openremote/manager/notification/NotificationService.javaindex c9d0b3fe9..8c51c04fa 100644--- a/manager/src/main/java/org/openremote/manager/notification/NotificationService.java+++ b/manager/src/main/java/org/openremote/manager/notification/NotificationService.java@@ -44,8 +44,7 @@ import org.openremote.model.util.TimeUtil;import javax.persistence.Query;import javax.persistence.TypedQuery;-import java.time.ZoneId;-import java.time.ZonedDateTime;+import java.time.Instant;import java.time.temporal.ChronoUnit;import java.time.temporal.TemporalField;import java.time.temporal.WeekFields;@@ -56,11 +55,10 @@ import java.util.logging.Logger;import java.util.stream.Collectors;import java.util.stream.IntStream;+import static java.time.temporal.ChronoUnit.*;import static org.openremote.manager.notification.NotificationProcessingException.Reason.*;import static org.openremote.model.notification.Notification.HEADER_SOURCE;-import static org.openremote.model.notification.Notification.Source.ASSET_RULESET;-import static org.openremote.model.notification.Notification.Source.CLIENT;-import static org.openremote.model.notification.Notification.Source.INTERNAL;+import static org.openremote.model.notification.Notification.Source.*;// TODO Implement notification purging - configurable MAX_AGE for notifications?public class NotificationService extends RouteBuilder implements ContainerService {@@ -294,7 +292,7 @@ public class NotificationService extends RouteBuilder implements ContainerServic.setSourceId(sourceId.get()).setTarget(targets.getType()).setTargetId(targetId)- .setSentOn(Date.from(timerService.getNow().toInstant()));+ .setSentOn(Date.from(timerService.getNow()));sentNotification = em.merge(sentNotification);long id = sentNotification.getId();@@ -499,8 +497,8 @@ public class NotificationService extends RouteBuilder implements ContainerServic}}- protected ZonedDateTime getRepeatAfterTimestamp(Notification notification, ZonedDateTime lastSend) {- ZonedDateTime timestamp = null;+ protected Instant getRepeatAfterTimestamp(Notification notification, Instant lastSend) {+ Instant timestamp = null;if (TextUtil.isNullOrEmpty(notification.getName())) {return null;@@ -511,19 +509,19 @@ public class NotificationService extends RouteBuilder implements ContainerServicswitch (notification.getRepeatFrequency()) {case HOURLY:- timestamp = lastSend.truncatedTo(ChronoUnit.HOURS).plusHours(1);+ timestamp = lastSend.truncatedTo(HOURS).plus(1, HOURS);break;case DAILY:- timestamp = lastSend.truncatedTo(ChronoUnit.DAYS).plusDays(1);+ timestamp = lastSend.truncatedTo(DAYS).plus(1, DAYS);break;case WEEKLY:- timestamp = lastSend.truncatedTo(ChronoUnit.WEEKS).plusWeeks(1);+ timestamp = lastSend.truncatedTo(WEEKS).plus(1, WEEKS);break;case MONTHLY:- timestamp = lastSend.truncatedTo(ChronoUnit.MONTHS).plusMonths(1);+ timestamp = lastSend.truncatedTo(MONTHS).plus(1, MONTHS);break;case ANNUALLY:- timestamp = lastSend.truncatedTo(ChronoUnit.YEARS).plusYears(1);+ timestamp = lastSend.truncatedTo(YEARS).plus(1, YEARS);break;}} else if (!TextUtil.isNullOrEmpty(notification.getRepeatInterval())) {@@ -610,7 +608,6 @@ public class NotificationService extends RouteBuilder implements ContainerServicreturn lastSend == null ||(notification.getRepeatFrequency() != RepeatFrequency.ONCE &&- timerService.getNow().plusSeconds(1).isAfter(getRepeatAfterTimestamp(notification, ZonedDateTime.ofInstant(lastSend.toInstant(),- ZoneId.systemDefault()))));+ timerService.getNow().plusSeconds(1).isAfter(getRepeatAfterTimestamp(notification, lastSend.toInstant())));}}\ No newline at end of filediff --git a/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java b/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.javaindex e0ac7d27b..53cea7961 100644--- a/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java+++ b/manager/src/main/java/org/openremote/manager/setup/builtin/ManagerDemoSetup.java@@ -21,8 +21,6 @@ package org.openremote.manager.setup.builtin;import org.openremote.agent.protocol.simulator.SimulatorProtocol;import org.openremote.container.Container;-import org.openremote.manager.asset.console.ConsoleResourceImpl;-import org.openremote.manager.rules.geofence.ORConsoleGeofenceAssetAdapter;import org.openremote.manager.security.UserConfiguration;import org.openremote.manager.setup.AbstractManagerSetup;import org.openremote.model.asset.Asset;@@ -30,8 +28,6 @@ import org.openremote.model.asset.AssetAttribute;import org.openremote.model.asset.AssetMeta;import org.openremote.model.asset.UserAsset;import org.openremote.model.attribute.*;-import org.openremote.model.console.ConsoleConfiguration;-import org.openremote.model.console.ConsoleProvider;import org.openremote.model.geo.GeoJSONPoint;import org.openremote.model.security.Tenant;import org.openremote.model.simulator.element.ColorSimulatorElement;@@ -44,7 +40,6 @@ import java.time.LocalDateTime;import java.time.ZoneId;import java.time.ZonedDateTime;import java.util.Arrays;-import java.util.HashMap;import java.util.List;import static org.openremote.model.asset.AssetMeta.*;@@ -68,8 +63,8 @@ public class ManagerDemoSetup extends AbstractManagerSetup {public static final int DEMO_RULE_STATES_GLOBAL_WITH_SCENES = DEMO_RULE_STATES_CUSTOMER_A_WITH_SCENES;public static GeoJSONPoint SMART_OFFICE_LOCATION = new GeoJSONPoint(5.460315214821094, 51.44541688237109);public static GeoJSONPoint SMART_HOME_LOCATION = new GeoJSONPoint(5.470945, 51.438000);- public final String agentProtocolConfigName = "simulator123";- public final String thingLightToggleAttributeName = "light1Toggle";+ public static final String agentProtocolConfigName = "simulator123";+ public static final String thingLightToggleAttributeName = "light1Toggle";final protected boolean importDemoScenes;public String smartOfficeId;public String groundFloorId;@@ -187,6 +182,10 @@ public class ManagerDemoSetup extends AbstractManagerSetup {new MetaItem(STORE_DATA_POINTS,Values.create(true)),+ new MetaItem(+ DATA_POINTS_MAX_AGE_DAYS,+ Values.create(7)+ ),new MetaItem(AGENT_LINK,new AttributeRef(agent.getId(), agentProtocolConfigName).toArrayValue()),diff --git a/model/src/main/java/org/openremote/model/asset/AssetMeta.java b/model/src/main/java/org/openremote/model/asset/AssetMeta.javaindex 5a8dd863d..5ad9e3c8e 100644--- a/model/src/main/java/org/openremote/model/asset/AssetMeta.java+++ b/model/src/main/java/org/openremote/model/asset/AssetMeta.java@@ -34,6 +34,7 @@ import static org.openremote.model.Constants.ASSET_META_NAMESPACE;import static org.openremote.model.attribute.MetaItem.MetaItemFailureReason.META_ITEM_VALUE_MISMATCH;import static org.openremote.model.attribute.MetaItemDescriptor.Access.ACCESS_PRIVATE;import static org.openremote.model.util.TextUtil.REGEXP_PATTERN_DOUBLE;+import static org.openremote.model.util.TextUtil.REGEXP_PATTERN_INTEGER_POSITIVE_NON_ZERO;/*** Asset attribute meta item name is an arbitrary string. It should be URI. This enum contains the well-known URIs for@@ -288,6 +289,20 @@ public enum AssetMeta implements MetaItemDescriptor {Values.create(true),true),+ /**+ * How long to store attribute values in time series database (data older than this will be automatically purged)+ */+ DATA_POINTS_MAX_AGE_DAYS(+ ASSET_META_NAMESPACE + ":dataPointsMaxAgeDays",+ new Access(true, false, true),+ ValueType.NUMBER,+ REGEXP_PATTERN_INTEGER_POSITIVE_NON_ZERO,+ PatternFailure.INTEGER_POSITIVE_NON_ZERO.name(),+ null,+ false,+ value -> value == null ? Optional.empty() : Values.getIntegerCoerced(value)+ .isPresent() ? Optional.empty() : Optional.of(new ValidationFailure(META_ITEM_VALUE_MISMATCH, "Integer"))),+/*** Should attribute writes be processed by the rules engines as {@link AssetState} facts, with a lifecycle that* reflects the state of the asset attribute. Each attribute will have one fact at all times in rules memory. Thesediff --git a/profile/deploy.yml b/profile/deploy.ymlindex 6ef60b67b..28fd75285 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -220,6 +220,10 @@ services:# unless you want to deploy a modified version of the Manager UI and JavaScript components.# MANAGER_DOCROOT: 'client/src/main/webapp'+ # Configure the default max age for data points stored in the database (individual attributes can use a custom+ # value by using the DATA_POINTS_MAX_AGE_DAYS AssetMeta item).+ # DATA_POINTS_MAX_AGE_DAYS = 30+# Custom JVM options (increase heap size if required)JAVA_OPTS: '-Xms400m -Xmx400m-XX:CompressedClassSpaceSize=25m -XX:MaxMetaspaceSize=150mdiff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovyindex d44f43810..8d1903768 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy@@ -12,18 +12,24 @@ import org.openremote.test.ManagerContainerTraitimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions+import java.time.Instant+import java.util.concurrent.TimeUnit++import static java.util.concurrent.TimeUnit.HOURSimport static java.util.concurrent.TimeUnit.SECONDS+import static org.openremote.manager.datapoint.AssetDatapointService.DATA_POINTS_MAX_AGE_DAYS+import static org.openremote.manager.setup.builtin.ManagerDemoSetup.thingLightToggleAttributeNameclass AssetDatapointTest extends Specification implements ManagerContainerTrait {- def "Receive number sensor values and store asset datapoints"() {+ def "Test number and toggle attribute storage, retrieval and purging"() {given: "expected conditions"def conditions = new PollingConditions(timeout: 10, initialDelay: 1, delay: 1)- when: "the demo agent and thing have been deployed"+ when: "the demo agent and thing have been deployed with a DATA_POINTS_MAX_AGE_DAYS value of 1"def serverPort = findEphemeralPort()- def container = startContainerWithPseudoClock(defaultConfig(serverPort), defaultServices())+ def container = startContainerWithPseudoClock defaultConfig(serverPort) << [(DATA_POINTS_MAX_AGE_DAYS): "1"], defaultServices()def managerDemoSetup = container.getService(SetupService.class).getTaskOfType(ManagerDemoSetup.class)def simulatorProtocol = container.getService(SimulatorProtocol.class)def assetStorageService = container.getService(AssetStorageService.class)@@ -39,6 +45,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitadvancePseudoClock(10, SECONDS, container)def datapoint1ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(13.3d))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)@@ -49,6 +56,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitadvancePseudoClock(10, SECONDS, container)def datapoint2ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(14.4d))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)@@ -59,6 +67,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitadvancePseudoClock(10, SECONDS, container)def datapoint3ExpectedTimestamp = getClockTimeOf(container)simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(15.5d))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)@@ -68,6 +77,7 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitwhen: "a simulated sensor receives a new value"advancePseudoClock(10, SECONDS, container)simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", null) // No value!+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)@@ -103,62 +113,46 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitassert aggregatedDatapoints.size() == 61}- cleanup: "the server should be stopped"- stopContainer(container)- }-- def "Receive boolean sensor values and store asset datapoints"() {-- given: "expected conditions"- def conditions = new PollingConditions(timeout: 10, initialDelay: 1, delay: 1)+ // ------------------------------------+ // Test boolean data point storage+ // ------------------------------------- when: "the demo agent and thing have been deployed"- def serverPort = findEphemeralPort()- def container = startContainerWithPseudoClock(defaultConfig(serverPort), defaultServices())- def managerDemoSetup = container.getService(SetupService.class).getTaskOfType(ManagerDemoSetup.class)- def simulatorProtocol = container.getService(SimulatorProtocol.class)- def assetStorageService = container.getService(AssetStorageService.class)- def assetDatapointService = container.getService(AssetDatapointService.class)-- then: "the simulator elements should have the initial state"- conditions.eventually {- def state = simulatorProtocol.getValue(managerDemoSetup.thingId, "light1Toggle")- assert Values.getBoolean(state.orElse(null)).orElse(null)- }-- when: "a simulated sensor receives a new value"+ when: "a simulated boolean sensor receives a new value"advancePseudoClock(10, SECONDS, container)- def datapoint1ExpectedTimestamp = getClockTimeOf(container)- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(false))+ datapoint1ExpectedTimestamp = getClockTimeOf(container)+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(false))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)- assert !thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)+ assert !thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)}when: "a simulated sensor receives a new value"advancePseudoClock(10, SECONDS, container)- def datapoint2ExpectedTimestamp = getClockTimeOf(container)- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(true))+ datapoint2ExpectedTimestamp = getClockTimeOf(container)+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(true))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)- assert thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)+ assert thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)}when: "a simulated sensor receives a new value"advancePseudoClock(10, SECONDS, container)- def datapoint3ExpectedTimestamp = getClockTimeOf(container)- simulatorProtocol.putValue(managerDemoSetup.thingId, "light1Toggle", Values.create(false))+ datapoint3ExpectedTimestamp = getClockTimeOf(container)+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(false))+then: "the attribute should be updated"conditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)- assert !thing.getAttribute("light1Toggle").flatMap { it.getValueAsBoolean() }.orElse(null)+ assert !thing.getAttribute(thingLightToggleAttributeName).flatMap { it.getValueAsBoolean() }.orElse(null)}expect: "the datapoints to be stored"conditions.eventually {- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1Toggle"))+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))assert datapoints.size() == 3assert !Values.getBoolean(datapoints.get(0).value).orElse(null)@@ -175,13 +169,102 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitconditions.eventually {def thing = assetStorageService.find(managerDemoSetup.thingId, true)def aggregatedDatapoints = assetDatapointService.aggregateDatapoints(- thing.getAttribute("light1Toggle").orElseThrow({ new RuntimeException("Missing attribute")}),+ thing.getAttribute(thingLightToggleAttributeName).orElseThrow({ new RuntimeException("Missing attribute")}),DatapointInterval.HOUR,getClockTimeOf(container))assert aggregatedDatapoints.size() == 61}+ // ------------------------------------+ // Test purging of data points+ // ------------------------------------++ when: "time moves forward by more than a day"+ advancePseudoClock(26, HOURS, container)++ and: "the power sensor with default max age receives a new value"+ def datapoint4ExpectedTimestamp = getClockTimeOf(container)+ simulatorProtocol.putValue(managerDemoSetup.thingId, "light1PowerConsumption", Values.create(17.5d))++ and: "the toggle sensor with a custom max age of 7 days receives a new value"+ simulatorProtocol.putValue(managerDemoSetup.thingId, thingLightToggleAttributeName, Values.create(true))++ then: "the datapoints should be stored"+ conditions.eventually {+ def powerDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))+ def toggleDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))++ assert powerDatapoints.size() > 4+ assert Values.getNumber(powerDatapoints.get(0).value).orElse(null) == 17.5d+ assert powerDatapoints.get(0).timestamp == datapoint4ExpectedTimestamp++ assert toggleDatapoints.size() == 4+ assert Values.getBoolean(toggleDatapoints.get(0).value).orElse(false)+ assert toggleDatapoints.get(0).timestamp == datapoint4ExpectedTimestamp+ }++ when: "the daily data point purge routine executes the next day"+ // Move clock to next days purge runtime+ advancePseudoClock(assetDatapointService.getFirstRunMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)+ assetDatapointService.purgeDataPoints()++ then: "data points older than 1 day should be purged for the power sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))+ assert datapoints.size() == 1+ assert Values.getNumber(datapoints.get(0).value).orElse(null) == 17.5d+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp+ }++ and: "no data points should have been purged for the toggle sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))+ assert datapoints.size() == 4+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp+ }++ when: "the daily data point purge routine executes 3 days later"+ advancePseudoClock(3, TimeUnit.DAYS, container)+ assetDatapointService.purgeDataPoints()++ then: "all data points should be purged for power sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, "light1PowerConsumption"))+ assert datapoints.isEmpty()+ }++ and: "no data points should have been purged for the toggle sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))+ assert datapoints.size() == 4+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp+ }++ when: "the daily data point purge routine executes 3 days later"+ advancePseudoClock(3, TimeUnit.DAYS, container)+ assetDatapointService.purgeDataPoints()++ then: "data points older than 7 days should have been purged for the toggle sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))+ assert datapoints.size() == 1+ assert Values.getBoolean(datapoints.get(0).value).orElse(false)+ assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp+ }++ when: "the daily data point purge routine executes 1 day later"+ advancePseudoClock(1, TimeUnit.DAYS, container)+ assetDatapointService.purgeDataPoints()++ then: "all data points should have been purged for the toggle sensor"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerDemoSetup.thingId, thingLightToggleAttributeName))+ assert datapoints.isEmpty()+ }+cleanup: "the server should be stopped"stopContainer(container)}diff --git a/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy b/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovyindex 425890280..8a2d7562b 100644--- a/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy+++ b/test/src/test/groovy/org/openremote/test/notification/NotificationTest.groovy@@ -418,11 +418,11 @@ class NotificationTest extends Specification implements ManagerContainerTrait {when: "a repeat frequency is set and a notification is sent"// Move clock to beginning of next hour- def advancement = Instant.ofEpochMilli(getClockTimeOf(container)).atZone(ZoneId.systemDefault())+ def advancement = Instant.ofEpochMilli(getClockTimeOf(container)).truncatedTo(ChronoUnit.HOURS)- .plusMinutes(59)+ .plus(59, ChronoUnit.MINUTES)- advancePseudoClock(ChronoUnit.MILLIS.between(Instant.ofEpochMilli(getClockTimeOf(container)).atZone(ZoneId.systemDefault()), advancement), TimeUnit.MILLISECONDS, container)+ advancePseudoClock(ChronoUnit.MILLIS.between(Instant.ofEpochMilli(getClockTimeOf(container)), advancement), TimeUnit.MILLISECONDS, container)notification.setRepeatFrequency(RepeatFrequency.HOURLY)testuser3NotificationResource.sendNotification(null, notification)notifications = []