100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fixes #2727 (#2728)
Rich Turner9d67acda1c73
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 6e003c7f3..8f97aef90 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -154,52 +154,88 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiLOG.info("Running data points purge daily task");try {- // Get list of attributes that have custom durations+ // 1. Get list of attributes that have custom durationsList<Asset<?>> assets = assetStorageService.findAll(- new AssetQuery()- .attributes(- new AttributePredicate().meta(- new NameValuePredicate(MetaItemType.DATA_POINTS_MAX_AGE_DAYS, null)- )));+ new AssetQuery()+ .attributes(+ new AttributePredicate().meta(+ new NameValuePredicate(MetaItemType.DATA_POINTS_MAX_AGE_DAYS, null)+ )));List<Pair<String, Attribute<?>>> attributes = assets.stream()- .map(asset -> asset- .getAttributes().stream()- .filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))- .map(assetAttribute -> new Pair<String, Attribute<?>>(asset.getId(), assetAttribute))- .collect(toList()))- .flatMap(List::stream)- .collect(toList());-- // Purge data points not in the above list using default duration- LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);-- persistenceService.doTransaction(em -> em.createQuery(- "delete from AssetDatapoint dp " +- "where dp.timestamp < :dt" + buildWhereClause(attributes, true)- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS))).executeUpdate());-- if (!attributes.isEmpty()) {- // Purge data points that have specific age constraints- Map<Integer, List<Pair<String, Attribute<?>>>> ageAttributeRefMap = attributes.stream()- .collect(groupingBy(attributeRef ->- attributeRef.value- .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)- .orElse(maxDatapointAgeDays)));-- ageAttributeRefMap.forEach((age, attrs) -> {- LOG.fine("Purging data points of " + attrs.size() + " attributes that use a max age of " + age);+ .map(asset -> asset+ .getAttributes().stream()+ .filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))+ .map(assetAttribute -> new Pair<String, Attribute<?>>(asset.getId(), assetAttribute))+ .collect(toList()))+ .flatMap(List::stream)+ .toList();++ // 2. Perform the purge using a temporary table and native SQL+ persistenceService.doTransaction(em -> {+ em.unwrap(Session.class).doWork(connection -> {+ String tempTableName = "tmp_purge_attributes";- try {- persistenceService.doTransaction(em -> em.createQuery(- "delete from AssetDatapoint dp " +- "where dp.timestamp < :dt" + buildWhereClause(attrs, false)- ).setParameter("dt", Date.from(timerService.getNow().truncatedTo(DAYS).minus(age, DAYS))).executeUpdate());- } catch (Exception e) {- LOG.log(Level.SEVERE, "An error occurred whilst deleting data points, this should not happen", e);+ try (Statement statement = connection.createStatement()) {+ // Create temp table holding the asset ID, attribute name, and its custom max age+ statement.execute(+ "create temp table " + tempTableName + " (entity_id text, attribute_name text, max_age_days int) on commit drop"+ );+ }++ // Batch insert the custom retention rules to avoid giant IN clauses+ if (!attributes.isEmpty()) {+ try (PreparedStatement insertStatement = connection.prepareStatement(+ "insert into " + tempTableName + " (entity_id, attribute_name, max_age_days) values (?, ?, ?)"+ )) {+ for (Pair<String, Attribute<?>> attributeRef : attributes) {+ insertStatement.setString(1, attributeRef.key);+ insertStatement.setString(2, attributeRef.value.getName());++ int customAge = attributeRef.value+ .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)+ .orElse(maxDatapointAgeDays);+ insertStatement.setInt(3, customAge);++ insertStatement.addBatch();+ }+ insertStatement.executeBatch();+ }+ }++ // Purge default data points (NOT in the temp table)+ LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);+ long defaultEpochMillis = timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS).toEpochMilli();++ try (PreparedStatement defaultPurge = connection.prepareStatement(+ "delete from asset_datapoint ad " ++ "where ad.timestamp < to_timestamp(?) " ++ "and not exists (select 1 from " + tempTableName + " t where t.entity_id = ad.entity_id and t.attribute_name = ad.attribute_name)"+ )) {+ defaultPurge.setLong(1, defaultEpochMillis / 1000);+ defaultPurge.executeUpdate();+ }++ // Purge custom data points by joining against the temp table+ if (!attributes.isEmpty()) {+ LOG.fine("Purging data points with custom max age constraints");+ long currentEpochMillis = timerService.getNow().truncatedTo(DAYS).toEpochMilli();++ try (PreparedStatement customPurge = connection.prepareStatement(+ "delete from asset_datapoint ad " ++ "using " + tempTableName + " t " ++ "where ad.entity_id = t.entity_id " ++ "and ad.attribute_name = t.attribute_name " ++ // Delete where timestamp is older than (current date - custom age)+ "and ad.timestamp < to_timestamp(? - (t.max_age_days * 86400))"+ )) {+ customPurge.setLong(1, currentEpochMillis / 1000);+ customPurge.executeUpdate();+ }}});- }+ });+} catch (Exception e) {LOG.log(Level.WARNING, "Failed to run data points purge", e);}