100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Added daily purge to predicted datapoints and also moved code to the datapoint package
Rich Turner5d503888ae14
diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.javaindex a13ed9dd4..e69208842 100644--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java@@ -37,7 +37,7 @@ import org.openremote.model.asset.UserAsset;import org.openremote.model.datapoint.AssetDatapoint;import org.openremote.model.gateway.GatewayConnection;import org.openremote.model.notification.SentNotification;-import org.openremote.model.predicted.AssetPredictedDatapoint;+import org.openremote.model.datapoint.AssetPredictedDatapoint;import org.openremote.model.rules.AssetRuleset;import org.openremote.model.rules.GlobalRuleset;import org.openremote.model.rules.TenantRuleset;diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 360c7709d..022c1f01e 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -100,10 +100,10 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProcepublic void start(Container container) throws Exception {if (maxDatapointAgeDays > 0) {dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(- this::purgeDataPoints,-+ this::purgeDataPoints,getFirstRunMillis(timerService.getNow()),- Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS);+ Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS+ );}}@@ -339,8 +339,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProceprotected void purgeDataPoints() {LOG.info("Starting data points purge daily task");- // Get list of attributes that have custom durations- List<Asset<?>> assets = assetStorageService.findAll(+ try {+ // Get list of attributes that have custom durations+ List<Asset<?>> assets = assetStorageService.findAll(new AssetQuery().attributes(new AttributePredicate().meta(@@ -348,7 +349,7 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce)).select(AssetQuery.Select.selectExcludePathAndParentInfo()));- List<Pair<String, Attribute<?>>> attributes = assets.stream()+ List<Pair<String, Attribute<?>>> attributes = assets.stream().map(asset -> asset.getAttributes().stream().filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))@@ -357,34 +358,37 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce.flatMap(List::stream).collect(toList());- // Purge data points not in the above list using default duration- LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);+ // Purge data points not in the above list using default duration+ LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);- persistenceService.doTransaction(em -> em.createQuery(+ persistenceService.doTransaction(em -> em.createQuery("delete from AssetDatapoint dp " +- "where dp.timestamp < :dt" + buildWhereClause(attributes, true)- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS))).executeUpdate());-- if (!attributes.isEmpty()) {- // Purge data points that have specific age constraints- Map<Integer, List<Pair<String, Attribute<?>>>> ageAttributeRefMap = attributes.stream()- .collect(groupingBy(attributeRef ->- attributeRef.value- .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)- .orElse(maxDatapointAgeDays)));-- ageAttributeRefMap.forEach((age, attrs) -> {- LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);-- try {- persistenceService.doTransaction(em -> em.createQuery(+ "where dp.timestamp < :dt" + buildWhereClause(attributes, true)+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS))).executeUpdate());++ if (!attributes.isEmpty()) {+ // Purge data points that have specific age constraints+ Map<Integer, List<Pair<String, Attribute<?>>>> ageAttributeRefMap = attributes.stream()+ .collect(groupingBy(attributeRef ->+ attributeRef.value+ .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)+ .orElse(maxDatapointAgeDays)));++ ageAttributeRefMap.forEach((age, attrs) -> {+ LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);++ try {+ persistenceService.doTransaction(em -> em.createQuery("delete from AssetDatapoint dp " +- "where dp.timestamp < :dt" + buildWhereClause(attrs, false)- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(age, DAYS))).executeUpdate());- } catch (Exception e) {- LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);- }- });+ "where dp.timestamp < :dt" + buildWhereClause(attrs, false)+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(age, DAYS))).executeUpdate());+ } catch (Exception e) {+ LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);+ }+ });+ }+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Failed to run data points purge", e);}LOG.info("Finished data points purge daily task");diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.javasimilarity index 98%rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.javarename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.javaindex 32a178042..6def5ac4a 100644--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java@@ -17,7 +17,7 @@* You should have received a copy of the GNU Affero General Public License* along with this program. If not, see <http://www.gnu.org/licenses/>.*/-package org.openremote.manager.predicted;+package org.openremote.manager.datapoint;import com.fasterxml.jackson.databind.node.ObjectNode;import org.openremote.model.Container;diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.javasimilarity index 97%rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.javarename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.javaindex d350951b3..2ac598b8d 100644--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java@@ -17,7 +17,7 @@* You should have received a copy of the GNU Affero General Public License* along with this program. If not, see <http://www.gnu.org/licenses/>.*/-package org.openremote.manager.predicted;+package org.openremote.manager.datapoint;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetStorageService;@@ -28,7 +28,7 @@ import org.openremote.model.attribute.AttributeRef;import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.http.RequestParams;-import org.openremote.model.predicted.AssetPredictedDatapointResource;+import org.openremote.model.datapoint.AssetPredictedDatapointResource;import javax.ws.rs.BeanParam;import javax.ws.rs.WebApplicationException;diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.javasimilarity index 85%rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.javarename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.javaindex d7dc8eae2..0ba6b1d7c 100644--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java@@ -17,18 +17,18 @@* You should have received a copy of the GNU Affero General Public License* along with this program. If not, see <http://www.gnu.org/licenses/>.*/-package org.openremote.manager.predicted;+package org.openremote.manager.datapoint;import org.hibernate.Session;import org.hibernate.jdbc.AbstractReturningWork;import org.openremote.agent.protocol.ProtocolPredictedAssetService;-import org.openremote.model.Container;-import org.openremote.model.ContainerService;import org.openremote.container.persistence.PersistenceService;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;+import org.openremote.model.Container;+import org.openremote.model.ContainerService;import org.openremote.model.asset.Asset;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeRef;@@ -39,14 +39,20 @@ import org.postgresql.util.PGInterval;import javax.persistence.EntityManager;import javax.persistence.TypedQuery;-import java.sql.Connection;-import java.sql.PreparedStatement;-import java.sql.ResultSet;-import java.sql.SQLException;+import java.sql.*;+import java.time.Duration;+import java.time.Instant;+import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.List;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;+import java.util.logging.Level;import java.util.logging.Logger;+import static java.time.temporal.ChronoUnit.DAYS;+public class AssetPredictedDatapointService implements ContainerService, ProtocolPredictedAssetService {public static final int PRIORITY = AssetStorageService.PRIORITY + 300;@@ -54,6 +60,9 @@ public class AssetPredictedDatapointService implements ContainerService, Protocoprotected PersistenceService persistenceService;protected AssetStorageService assetStorageService;+ protected TimerService timerService;+ protected ScheduledExecutorService executorService;+ protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;@Overridepublic int getPriority() {@@ -64,6 +73,8 @@ public class AssetPredictedDatapointService implements ContainerService, Protocopublic void init(Container container) throws Exception {persistenceService = container.getService(PersistenceService.class);assetStorageService = container.getService(AssetStorageService.class);+ timerService = container.getService(TimerService.class);+ executorService = container.getExecutorService();container.getService(ManagerWebService.class).getApiSingletons().add(new AssetPredictedDatapointResourceImpl(@@ -77,12 +88,18 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco@Overridepublic void start(Container container) throws Exception {-+ dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(+ this::purgeDataPoints,+ getFirstRunMillis(timerService.getNow()),+ Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS+ );}@Overridepublic void stop(Container container) throws Exception {-+ if (dataPointsPurgeScheduledFuture != null) {+ dataPointsPurgeScheduledFuture.cancel(true);+ }}public long getDatapointsCount() {@@ -271,4 +288,29 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco" SET value = excluded.value", assetId, attributeName, value, timestamp)).executeUpdate();}++ protected void purgeDataPoints() {+ LOG.info("Starting data points purge daily task");++ try {+ // Purge data points not in the above list using default duration+ LOG.fine("Purging predicted data points older than 1 day");++ persistenceService.doTransaction(em -> em.createQuery(+ "delete from AssetPredictedDatapoint dp " ++ "where dp.timestamp < :dt"+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(1, DAYS))).executeUpdate());+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Failed to run data points purge", e);+ }++ LOG.info("Finished data points purge daily task");+ }++ protected long getFirstRunMillis(Instant currentTime) {+ // Schedule purge at approximately 2AM daily+ return ChronoUnit.MILLIS.between(+ currentTime,+ currentTime.truncatedTo(DAYS).plus(26, ChronoUnit.HOURS));+ }}diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.javaindex 76980f397..a12b8e211 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java@@ -28,7 +28,7 @@ import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.datapoint.AssetDatapointService;import org.openremote.manager.event.ClientEventService;import org.openremote.manager.notification.NotificationService;-import org.openremote.manager.predicted.AssetPredictedDatapointService;+import org.openremote.manager.datapoint.AssetPredictedDatapointService;import org.openremote.manager.rules.facade.*;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.model.asset.Asset;diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.javaindex f30c132d0..7ea49b2de 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java@@ -33,7 +33,7 @@ import org.openremote.manager.datapoint.AssetDatapointService;import org.openremote.manager.event.ClientEventService;import org.openremote.manager.gateway.GatewayService;import org.openremote.manager.notification.NotificationService;-import org.openremote.manager.predicted.AssetPredictedDatapointService;+import org.openremote.manager.datapoint.AssetPredictedDatapointService;import org.openremote.manager.rules.flow.FlowResourceImpl;import org.openremote.manager.rules.geofence.GeofenceAssetAdapter;import org.openremote.manager.security.ManagerIdentityService;@@ -74,7 +74,6 @@ import static org.openremote.container.persistence.PersistenceEvent.isPersistencimport static org.openremote.container.util.MapAccess.getString;import static org.openremote.manager.gateway.GatewayService.isNotForGateway;import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttributes;-import static org.openremote.model.value.MetaItemType.RULE_STATE;/*** Manages {@link RulesEngine}s for stored {@link Ruleset}s and processes asset attribute updates.diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.javaindex 58e2af4ee..eba42431e 100644--- a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java+++ b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java@@ -20,14 +20,11 @@package org.openremote.manager.rules.facade;import org.openremote.manager.datapoint.AssetDatapointService;-import org.openremote.manager.predicted.AssetPredictedDatapointService;import org.openremote.manager.rules.RulesEngineId;import org.openremote.model.attribute.AttributeRef;-import org.openremote.model.datapoint.AssetDatapoint;import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.rules.HistoricDatapoints;-import org.openremote.model.rules.PredictedDatapoints;import org.openremote.model.rules.Ruleset;import java.util.logging.Logger;diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.javaindex 34c0a0fe8..37959f327 100644--- a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java+++ b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java@@ -19,7 +19,7 @@*/package org.openremote.manager.rules.facade;-import org.openremote.manager.predicted.AssetPredictedDatapointService;+import org.openremote.manager.datapoint.AssetPredictedDatapointService;import org.openremote.manager.rules.RulesEngineId;import org.openremote.model.attribute.AttributeRef;import org.openremote.model.datapoint.DatapointInterval;diff --git a/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java b/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.javaindex e91951bea..99be033dc 100644--- a/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java+++ b/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java@@ -29,7 +29,7 @@ import org.openremote.manager.asset.AssetProcessingService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.datapoint.AssetDatapointService;import org.openremote.manager.persistence.ManagerPersistenceService;-import org.openremote.manager.predicted.AssetPredictedDatapointService;+import org.openremote.manager.datapoint.AssetPredictedDatapointService;import org.openremote.manager.rules.RulesetStorageService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.model.Constants;diff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerServiceindex d45c384b1..155a96b1b 100644--- a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService@@ -20,7 +20,7 @@ org.openremote.manager.apps.ConsoleAppServiceorg.openremote.manager.web.ManagerWebServiceorg.openremote.manager.notification.PushNotificationHandlerorg.openremote.manager.notification.EmailNotificationHandler-org.openremote.manager.predicted.AssetPredictedDatapointService+org.openremote.manager.datapoint.AssetPredictedDatapointServiceorg.openremote.manager.gateway.GatewayServiceorg.openremote.manager.gateway.GatewayClientServiceorg.openremote.manager.mqtt.MqttBrokerServicediff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider b/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProviderindex 39292d156..88c14ee45 100644--- a/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider@@ -4,4 +4,4 @@ org.openremote.manager.system.SystemHealthStatusProviderorg.openremote.manager.rules.RulesHealthStatusProviderorg.openremote.manager.agent.AgentHealthStatusProviderorg.openremote.manager.datapoint.AssetDatapointHealthStatusProvider-org.openremote.manager.predicted.AssetPredictedDatapointHealthStatusProvider+org.openremote.manager.datapoint.AssetPredictedDatapointHealthStatusProviderdiff --git a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.javasimilarity index 97%rename from model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.javarename to model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.javaindex 5b5692cd8..69da8253d 100644--- a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java@@ -17,7 +17,7 @@* You should have received a copy of the GNU Affero General Public License* along with this program. If not, see <http://www.gnu.org/licenses/>.*/-package org.openremote.model.predicted;+package org.openremote.model.datapoint;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.attribute.AttributeRef;diff --git a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.javasimilarity index 98%rename from model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.javarename to model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.javaindex e980da330..177608b31 100644--- a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java@@ -17,7 +17,7 @@* You should have received a copy of the GNU Affero General Public License* along with this program. If not, see <http://www.gnu.org/licenses/>.*/-package org.openremote.model.predicted;+package org.openremote.model.datapoint;import org.openremote.model.Constants;import org.openremote.model.datapoint.DatapointInterval;