Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Added daily purge to predicted datapoints and also moved code to the datapoint package

Rich Turner5d503888ae14
diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
index a13ed9dd4..e69208842 100644
--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
@@ -37,7 +37,7 @@ import org.openremote.model.asset.UserAsset;
import org.openremote.model.datapoint.AssetDatapoint;
import org.openremote.model.gateway.GatewayConnection;
import org.openremote.model.notification.SentNotification;
-import org.openremote.model.predicted.AssetPredictedDatapoint;
+import org.openremote.model.datapoint.AssetPredictedDatapoint;
import org.openremote.model.rules.AssetRuleset;
import org.openremote.model.rules.GlobalRuleset;
import org.openremote.model.rules.TenantRuleset;
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 360c7709d..022c1f01e 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -100,10 +100,10 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
public void start(Container container) throws Exception {
if (maxDatapointAgeDays > 0) {
dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(
- this::purgeDataPoints,
-
+ this::purgeDataPoints,
getFirstRunMillis(timerService.getNow()),
- Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS);
+ Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS
+ );
}
}
@@ -339,8 +339,9 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
protected void purgeDataPoints() {
LOG.info("Starting data points purge daily task");
- // Get list of attributes that have custom durations
- List<Asset<?>> assets = assetStorageService.findAll(
+ try {
+ // Get list of attributes that have custom durations
+ List<Asset<?>> assets = assetStorageService.findAll(
new AssetQuery()
.attributes(
new AttributePredicate().meta(
@@ -348,7 +349,7 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
))
.select(AssetQuery.Select.selectExcludePathAndParentInfo()));
- List<Pair<String, Attribute<?>>> attributes = assets.stream()
+ List<Pair<String, Attribute<?>>> attributes = assets.stream()
.map(asset -> asset
.getAttributes().stream()
.filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))
@@ -357,34 +358,37 @@ public class AssetDatapointService implements ContainerService, AssetUpdateProce
.flatMap(List::stream)
.collect(toList());
- // Purge data points not in the above list using default duration
- LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);
+ // Purge data points not in the above list using default duration
+ LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);
- persistenceService.doTransaction(em -> em.createQuery(
+ persistenceService.doTransaction(em -> em.createQuery(
"delete from AssetDatapoint dp " +
- "where dp.timestamp < :dt" + buildWhereClause(attributes, true)
- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS))).executeUpdate());
-
- if (!attributes.isEmpty()) {
- // Purge data points that have specific age constraints
- Map<Integer, List<Pair<String, Attribute<?>>>> ageAttributeRefMap = attributes.stream()
- .collect(groupingBy(attributeRef ->
- attributeRef.value
- .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)
- .orElse(maxDatapointAgeDays)));
-
- ageAttributeRefMap.forEach((age, attrs) -> {
- LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);
-
- try {
- persistenceService.doTransaction(em -> em.createQuery(
+ "where dp.timestamp < :dt" + buildWhereClause(attributes, true)
+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS))).executeUpdate());
+
+ if (!attributes.isEmpty()) {
+ // Purge data points that have specific age constraints
+ Map<Integer, List<Pair<String, Attribute<?>>>> ageAttributeRefMap = attributes.stream()
+ .collect(groupingBy(attributeRef ->
+ attributeRef.value
+ .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)
+ .orElse(maxDatapointAgeDays)));
+
+ ageAttributeRefMap.forEach((age, attrs) -> {
+ LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);
+
+ try {
+ persistenceService.doTransaction(em -> em.createQuery(
"delete from AssetDatapoint dp " +
- "where dp.timestamp < :dt" + buildWhereClause(attrs, false)
- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(age, DAYS))).executeUpdate());
- } catch (Exception e) {
- LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);
- }
- });
+ "where dp.timestamp < :dt" + buildWhereClause(attrs, false)
+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(age, DAYS))).executeUpdate());
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);
+ }
+ });
+ }
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to run data points purge", e);
}
LOG.info("Finished data points purge daily task");
diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java
similarity index 98%
rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.java
rename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java
index 32a178042..6def5ac4a 100644
--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointHealthStatusProvider.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointHealthStatusProvider.java
@@ -17,7 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-package org.openremote.manager.predicted;
+package org.openremote.manager.datapoint;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.openremote.model.Container;
diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
similarity index 97%
rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.java
rename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
index d350951b3..2ac598b8d 100644
--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointResourceImpl.java
@@ -17,7 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-package org.openremote.manager.predicted;
+package org.openremote.manager.datapoint;
import org.openremote.container.timer.TimerService;
import org.openremote.manager.asset.AssetStorageService;
@@ -28,7 +28,7 @@ import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.http.RequestParams;
-import org.openremote.model.predicted.AssetPredictedDatapointResource;
+import org.openremote.model.datapoint.AssetPredictedDatapointResource;
import javax.ws.rs.BeanParam;
import javax.ws.rs.WebApplicationException;
diff --git a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
similarity index 85%
rename from manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.java
rename to manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
index d7dc8eae2..0ba6b1d7c 100644
--- a/manager/src/main/java/org/openremote/manager/predicted/AssetPredictedDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetPredictedDatapointService.java
@@ -17,18 +17,18 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-package org.openremote.manager.predicted;
+package org.openremote.manager.datapoint;
import org.hibernate.Session;
import org.hibernate.jdbc.AbstractReturningWork;
import org.openremote.agent.protocol.ProtocolPredictedAssetService;
-import org.openremote.model.Container;
-import org.openremote.model.ContainerService;
import org.openremote.container.persistence.PersistenceService;
import org.openremote.container.timer.TimerService;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebService;
+import org.openremote.model.Container;
+import org.openremote.model.ContainerService;
import org.openremote.model.asset.Asset;
import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeRef;
@@ -39,14 +39,20 @@ import org.postgresql.util.PGInterval;
import javax.persistence.EntityManager;
import javax.persistence.TypedQuery;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.sql.*;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
import java.util.logging.Logger;
+import static java.time.temporal.ChronoUnit.DAYS;
+
public class AssetPredictedDatapointService implements ContainerService, ProtocolPredictedAssetService {
public static final int PRIORITY = AssetStorageService.PRIORITY + 300;
@@ -54,6 +60,9 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
protected PersistenceService persistenceService;
protected AssetStorageService assetStorageService;
+ protected TimerService timerService;
+ protected ScheduledExecutorService executorService;
+ protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
@Override
public int getPriority() {
@@ -64,6 +73,8 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
public void init(Container container) throws Exception {
persistenceService = container.getService(PersistenceService.class);
assetStorageService = container.getService(AssetStorageService.class);
+ timerService = container.getService(TimerService.class);
+ executorService = container.getExecutorService();
container.getService(ManagerWebService.class).getApiSingletons().add(
new AssetPredictedDatapointResourceImpl(
@@ -77,12 +88,18 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
@Override
public void start(Container container) throws Exception {
-
+ dataPointsPurgeScheduledFuture = executorService.scheduleAtFixedRate(
+ this::purgeDataPoints,
+ getFirstRunMillis(timerService.getNow()),
+ Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS
+ );
}
@Override
public void stop(Container container) throws Exception {
-
+ if (dataPointsPurgeScheduledFuture != null) {
+ dataPointsPurgeScheduledFuture.cancel(true);
+ }
}
public long getDatapointsCount() {
@@ -271,4 +288,29 @@ public class AssetPredictedDatapointService implements ContainerService, Protoco
" SET value = excluded.value", assetId, attributeName, value, timestamp))
.executeUpdate();
}
+
+ protected void purgeDataPoints() {
+ LOG.info("Starting data points purge daily task");
+
+ try {
+ // Purge data points not in the above list using default duration
+ LOG.fine("Purging predicted data points older than 1 day");
+
+ persistenceService.doTransaction(em -> em.createQuery(
+ "delete from AssetPredictedDatapoint dp " +
+ "where dp.timestamp < :dt"
+ ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(1, DAYS))).executeUpdate());
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to run data points purge", e);
+ }
+
+ LOG.info("Finished data points purge daily task");
+ }
+
+ protected long getFirstRunMillis(Instant currentTime) {
+ // Schedule purge at approximately 2AM daily
+ return ChronoUnit.MILLIS.between(
+ currentTime,
+ currentTime.truncatedTo(DAYS).plus(26, ChronoUnit.HOURS));
+ }
}
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
index 76980f397..a12b8e211 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesEngine.java
@@ -28,7 +28,7 @@ import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.datapoint.AssetDatapointService;
import org.openremote.manager.event.ClientEventService;
import org.openremote.manager.notification.NotificationService;
-import org.openremote.manager.predicted.AssetPredictedDatapointService;
+import org.openremote.manager.datapoint.AssetPredictedDatapointService;
import org.openremote.manager.rules.facade.*;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.model.asset.Asset;
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
index f30c132d0..7ea49b2de 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
@@ -33,7 +33,7 @@ import org.openremote.manager.datapoint.AssetDatapointService;
import org.openremote.manager.event.ClientEventService;
import org.openremote.manager.gateway.GatewayService;
import org.openremote.manager.notification.NotificationService;
-import org.openremote.manager.predicted.AssetPredictedDatapointService;
+import org.openremote.manager.datapoint.AssetPredictedDatapointService;
import org.openremote.manager.rules.flow.FlowResourceImpl;
import org.openremote.manager.rules.geofence.GeofenceAssetAdapter;
import org.openremote.manager.security.ManagerIdentityService;
@@ -74,7 +74,6 @@ import static org.openremote.container.persistence.PersistenceEvent.isPersistenc
import static org.openremote.container.util.MapAccess.getString;
import static org.openremote.manager.gateway.GatewayService.isNotForGateway;
import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttributes;
-import static org.openremote.model.value.MetaItemType.RULE_STATE;
/**
* Manages {@link RulesEngine}s for stored {@link Ruleset}s and processes asset attribute updates.
diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
index 58e2af4ee..eba42431e 100644
--- a/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
+++ b/manager/src/main/java/org/openremote/manager/rules/facade/HistoricFacade.java
@@ -20,14 +20,11 @@
package org.openremote.manager.rules.facade;
import org.openremote.manager.datapoint.AssetDatapointService;
-import org.openremote.manager.predicted.AssetPredictedDatapointService;
import org.openremote.manager.rules.RulesEngineId;
import org.openremote.model.attribute.AttributeRef;
-import org.openremote.model.datapoint.AssetDatapoint;
import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.rules.HistoricDatapoints;
-import org.openremote.model.rules.PredictedDatapoints;
import org.openremote.model.rules.Ruleset;
import java.util.logging.Logger;
diff --git a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
index 34c0a0fe8..37959f327 100644
--- a/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
+++ b/manager/src/main/java/org/openremote/manager/rules/facade/PredictedFacade.java
@@ -19,7 +19,7 @@
*/
package org.openremote.manager.rules.facade;
-import org.openremote.manager.predicted.AssetPredictedDatapointService;
+import org.openremote.manager.datapoint.AssetPredictedDatapointService;
import org.openremote.manager.rules.RulesEngineId;
import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.datapoint.DatapointInterval;
diff --git a/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java b/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java
index e91951bea..99be033dc 100644
--- a/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java
+++ b/manager/src/main/java/org/openremote/manager/setup/AbstractManagerSetup.java
@@ -29,7 +29,7 @@ import org.openremote.manager.asset.AssetProcessingService;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.datapoint.AssetDatapointService;
import org.openremote.manager.persistence.ManagerPersistenceService;
-import org.openremote.manager.predicted.AssetPredictedDatapointService;
+import org.openremote.manager.datapoint.AssetPredictedDatapointService;
import org.openremote.manager.rules.RulesetStorageService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.model.Constants;
diff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService
index d45c384b1..155a96b1b 100644
--- a/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService
+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.ContainerService
@@ -20,7 +20,7 @@ org.openremote.manager.apps.ConsoleAppService
org.openremote.manager.web.ManagerWebService
org.openremote.manager.notification.PushNotificationHandler
org.openremote.manager.notification.EmailNotificationHandler
-org.openremote.manager.predicted.AssetPredictedDatapointService
+org.openremote.manager.datapoint.AssetPredictedDatapointService
org.openremote.manager.gateway.GatewayService
org.openremote.manager.gateway.GatewayClientService
org.openremote.manager.mqtt.MqttBrokerService
diff --git a/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider b/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider
index 39292d156..88c14ee45 100644
--- a/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider
+++ b/manager/src/main/resources/META-INF/services/org.openremote.model.system.HealthStatusProvider
@@ -4,4 +4,4 @@ org.openremote.manager.system.SystemHealthStatusProvider
org.openremote.manager.rules.RulesHealthStatusProvider
org.openremote.manager.agent.AgentHealthStatusProvider
org.openremote.manager.datapoint.AssetDatapointHealthStatusProvider
-org.openremote.manager.predicted.AssetPredictedDatapointHealthStatusProvider
+org.openremote.manager.datapoint.AssetPredictedDatapointHealthStatusProvider
diff --git a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
similarity index 97%
rename from model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.java
rename to model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
index 5b5692cd8..69da8253d 100644
--- a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapoint.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapoint.java
@@ -17,7 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-package org.openremote.model.predicted;
+package org.openremote.model.datapoint;
import org.openremote.model.attribute.AttributeEvent;
import org.openremote.model.attribute.AttributeRef;
diff --git a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
similarity index 98%
rename from model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.java
rename to model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
index e980da330..177608b31 100644
--- a/model/src/main/java/org/openremote/model/predicted/AssetPredictedDatapointResource.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetPredictedDatapointResource.java
@@ -17,7 +17,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-package org.openremote.model.predicted;
+package org.openremote.model.datapoint;
import org.openremote.model.Constants;
import org.openremote.model.datapoint.DatapointInterval;