100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Enable TimescaleDB hypercore functionality on asset_datapoint table (#2352)
Michael Rademakera34c0df77167
diff --git a/deployment/query-exporter/config.yaml b/deployment/query-exporter/config.yamlindex 134627dd4..5a7e5228c 100644--- a/deployment/query-exporter/config.yaml+++ b/deployment/query-exporter/config.yaml@@ -1,10 +1,5 @@# Query Exporter Configuration for OpenRemote PostgreSQL Monitoring# Documentation: https://github.com/albertodonato/query-exporter/blob/main/docs/configuration.rst-#-# Environment Variables:-# - TABLE_BLOAT_THRESHOLD: Table bloat ratio threshold for alerting (default: 1.2 = 20% bloat)-# - INDEX_BLOAT_THRESHOLD: Index bloat ratio threshold for alerting (default: 1.5 = 50% bloat)-# Set these in docker-compose or deployment environment to adjust sensitivitydatabases:openremote:@@ -19,6 +14,10 @@ databases:instance: openremotemetrics:+ pg_collation_mismatch_count:+ type: gauge+ description: Number of text indexes with collation version mismatches requiring a REINDEX+pg_cache_hit_percentage:type: gaugedescription: What percentage of data is being served instantly from RAM versus being slowly read from disk. You want this as high as possible@@ -42,23 +41,27 @@ metrics:pg_hot_update_percent:type: gaugelabels: [table_name]- description: Asset table percentage of updates that are HOT updates indicates good fillfactor+ description: Table percentage of updates that are HOT updates indicates good fillfactorpg_dead_tuple_percent:type: gaugelabels: [table_name]- description: Asset table ratio of dead tuples to live ones a ratio > 10-20% indicates not aggressive enough autovacuum+ description: Table ratio of dead tuples to live ones a ratio > 10-20% indicates not aggressive enough autovacuumpg_last_autovacuum_hours:type: gaugelabels: [table_name]- description: Asset table hours since last auto vacuum run successfully+ description: Table hours since last auto vacuum run successfullypg_last_autoanalyze_hours:type: gaugelabels: [table_name]description: Asset table hours since last auto analyze run successfully+ pg_db_disk_size:+ type: gauge+ description: DB size in MB+pg_datapoint_raw_data_size:type: gaugedescription: Asset datapoint table raw uncompressed size in MB@@ -99,10 +102,75 @@ metrics:type: gaugedescription: Asset datapoint table hypertable chunks not yet analyzed+ pg_datapoint_largest_uncompressed_chunk:+ type: gauge+ description: Asset datapoint table largest uncompressed hypertable chunk in MB++ pg_datapoint_uncompressed_cache_hit_ratio:+ type: gauge+ description: Asset datapoint table cache hit ratio for uncompressed chunks (Aim for 99%+)++ pg_datapoint_uncompressed_blks_read_total:+ type: counter+ description: Asset datapoint total physical disk blocks read for uncompressed chunks (Monitor rate with spikes indicate RAM spillover)++ pg_datapoint_compression_ratio:+ type: gauge+ description: Asset datapoint table compression ratio++ pg_datapoint_query:+ type: gauge+ description: Dummy metric to get typical query time metric+pg_background_errors:type: counterdescription: Count of errors in background worker processes+ pg_timescale_job_total_runs:+ type: counter+ labels: [job_id, proc_name]+ description: TimescaleDB job total runs by job++ pg_timescale_job_total_failures:+ type: counter+ labels: [job_id, proc_name]+ description: TimescaleDB job total failures by job++ pg_timescale_job_last_run_duration_seconds:+ type: gauge+ labels: [job_id, proc_name]+ description: TimescaleDB job last run duration in seconds++ pg_timescale_job_next_start_seconds:+ type: gauge+ labels: [job_id, proc_name]+ description: Seconds until next scheduled run for each TimescaleDB job++ pg_timescale_job_last_run_status:+ type: gauge+ labels: [job_id, proc_name, last_run_status]+ description: TimescaleDB job last run status marker (1)++ pg_wal_total:+ type: counter+ description: Total WAL written since statistics reset in MB++ pg_bgwriter_checkpoints_timed_total:+ type: counter+ description: Scheduled checkpoints executed++ pg_bgwriter_checkpoints_req_total:+ type: counter+ description: Requested checkpoints executed++ pg_bgwriter_checkpoint_write_time_seconds_total:+ type: counter+ description: Total time spent writing checkpoints in seconds++ pg_bgwriter_checkpoint_sync_time_seconds_total:+ type: counter+ description: Total time spent syncing checkpoints in seconds+pg_table_bloat_count:type: gaugedescription: Number of tables where dead tuples > 30% of live rows@@ -113,6 +181,30 @@ metrics:queries:+ collation_mismatch:+ interval: 86400 # Daily check+ databases: [openremote]+ metrics: [pg_collation_mismatch_count]+ sql: |+ WITH mismatched_collations AS (+ SELECT oid+ FROM pg_catalog.pg_collation+ WHERE collversion IS NOT NULL+ AND collversion <> pg_collation_actual_version(oid)+ ),+ mismatched_indexes AS (+ SELECT DISTINCT d.objid AS index_oid+ FROM pg_catalog.pg_depend d+ JOIN mismatched_collations mc ON d.refobjid = mc.oid+ WHERE d.classid = 'pg_class'::regclass+ AND d.refclassid = 'pg_collation'::regclass+ )+ SELECT count(i.oid) AS pg_collation_mismatch_count+ FROM pg_catalog.pg_class i+ JOIN pg_catalog.pg_index idx ON i.oid = idx.indexrelid+ JOIN mismatched_indexes mi ON i.oid = mi.index_oid+ WHERE i.relkind = 'i';+cache_hit_ratio:interval: 3600 # Hourlydatabases: [openremote]@@ -158,6 +250,13 @@ queries:FROM pg_stat_user_tablesWHERE schemaname = 'openremote';+ db_size:+ interval: 3600 # Hourly+ databases: [openremote]+ metrics: [pg_db_disk_size]+ sql: |+ SELECT ROUND(pg_database_size(current_database()) / 1048576.0, 0) as pg_db_disk_size;+datapoint_table_size:interval: 3600 # Hourlydatabases: [openremote]@@ -189,7 +288,22 @@ queries:FROM timescaledb_information.chunksWHERE hypertable_name = 'asset_datapoint'GROUP BY hypertable_name;-++ datapoint_largest_uncompressed_chunk:+ interval: 86400 # Daily+ databases: [openremote]+ metrics: [pg_datapoint_largest_uncompressed_chunk]+ sql: |+ SELECT+ ROUND(s.total_bytes / 1048576.0, 0) AS pg_datapoint_largest_uncompressed_chunk+ FROM timescaledb_information.chunks c+ JOIN chunks_detailed_size('asset_datapoint') s+ ON c.chunk_schema = s.chunk_schema+ AND c.chunk_name = s.chunk_name+ WHERE c.is_compressed = false+ ORDER BY s.total_bytes DESC+ LIMIT 1;+datapoint_autoanalyze:interval: 86400 # Dailydatabases: [openremote]@@ -203,6 +317,53 @@ queries:AND last_analyze IS NULLAND n_mod_since_analyze > current_setting('autovacuum_analyze_threshold')::numeric;+ datapoint_historical_aggregation:+ interval: 3600 # Hourly+ databases: [openremote]+ metrics: [pg_datapoint_query]+ sql: |+ SELECT datapoint_metric_query() as pg_datapoint_query;++ uncompressed_memory_pressure:+ interval: 3600 # Hourly+ databases: [openremote]+ metrics: [pg_datapoint_uncompressed_cache_hit_ratio, pg_datapoint_uncompressed_blks_read_total]+ sql: |+ WITH uncompressed_chunks AS (+ SELECT chunk_name+ FROM timescaledb_information.chunks+ WHERE hypertable_name = 'asset_datapoint'+ AND is_compressed = false+ )+ SELECT+ -- Calculate the percentage of data served directly from RAM+ ROUND(+ COALESCE(+ (sum(s.heap_blks_hit) + sum(s.idx_blks_hit) + sum(s.toast_blks_hit))::numeric /+ NULLIF(+ sum(s.heap_blks_hit) + sum(s.idx_blks_hit) + sum(s.toast_blks_hit) ++ sum(s.heap_blks_read) + sum(s.idx_blks_read) + sum(s.toast_blks_read),+ 0) * 100,+ 0),+ 2) AS pg_datapoint_uncompressed_cache_hit_ratio,++ -- Track the raw number of times Postgres had to go to the physical disk+ COALESCE(sum(s.heap_blks_read) + sum(s.idx_blks_read) + sum(s.toast_blks_read), 0) AS pg_datapoint_uncompressed_blks_read_total+ FROM pg_statio_user_tables s+ JOIN uncompressed_chunks c ON s.relname = c.chunk_name;++ compression_stats:+ interval: 86400 # Daily+ databases: [ openremote ]+ metrics: [pg_datapoint_compression_ratio]+ sql: |+ SELECT+ COALESCE(+ (before_compression_total_bytes::numeric / NULLIF(after_compression_total_bytes, 0)),+ 1.0+ ) AS pg_datapoint_compression_ratio+ FROM hypertable_compression_stats('asset_datapoint');+background_errors:interval: 3600 # Hourlydatabases: [openremote]@@ -211,6 +372,52 @@ queries:SELECTSUM(total_failures) AS pg_background_errorsFROM timescaledb_information.job_stats;++ timescale_job_stats:+ interval: 3600 # Hourly+ databases: [openremote]+ metrics:+ [+ pg_timescale_job_total_runs,+ pg_timescale_job_total_failures,+ pg_timescale_job_last_run_duration_seconds,+ pg_timescale_job_next_start_seconds,+ pg_timescale_job_last_run_status+ ]+ sql: |+ SELECT+ s.job_id::text AS job_id,+ CONCAT(j.proc_schema, '.', j.proc_name) AS proc_name,+ COALESCE(s.last_run_status, 'unknown') AS last_run_status,+ COALESCE(s.total_runs, 0) AS pg_timescale_job_total_runs,+ COALESCE(s.total_failures, 0) AS pg_timescale_job_total_failures,+ EXTRACT(EPOCH FROM COALESCE(s.last_run_duration, INTERVAL '0 seconds')) AS pg_timescale_job_last_run_duration_seconds,+ EXTRACT(EPOCH FROM (COALESCE(s.next_start, now()) - now())) AS pg_timescale_job_next_start_seconds,+ 1 AS pg_timescale_job_last_run_status+ FROM timescaledb_information.job_stats s+ JOIN timescaledb_information.jobs j ON j.job_id = s.job_id+ WHERE j.hypertable_name = 'asset_datapoint';++ write_path_pressure:+ interval: 300 # 5 minutes+ databases: [openremote]+ metrics:+ [+ pg_wal_total,+ pg_bgwriter_checkpoints_timed_total,+ pg_bgwriter_checkpoints_req_total,+ pg_bgwriter_checkpoint_write_time_seconds_total,+ pg_bgwriter_checkpoint_sync_time_seconds_total+ ]+ sql: |+ SELECT+ (w.wal_bytes / 1048576.0) AS pg_wal_total,+ c.num_timed AS pg_bgwriter_checkpoints_timed_total,+ c.num_requested AS pg_bgwriter_checkpoints_req_total,+ ROUND((c.write_time / 1000.0)::numeric, 0) AS pg_bgwriter_checkpoint_write_time_seconds_total,+ ROUND((c.sync_time / 1000.0)::numeric, 0) AS pg_bgwriter_checkpoint_sync_time_seconds_total+ FROM pg_stat_wal w+ CROSS JOIN pg_stat_checkpointer c;table_and_index_bloat:interval: 300 # 5 minutes@@ -221,10 +428,7 @@ queries:SELECT count(*) AS tbl_countFROM pg_stat_user_tablesWHERE schemaname IN ('openremote', 'public')- -- 1. NOISE FILTER: Only look at tables larger than 5MB.AND pg_relation_size(relid) > 5 * 1024 * 1024- -- 2. THRESHOLD: Dead tuples are > 30% of live rows.- -- (Autovacuum runs at 20%, so 30% means autovacuum is failing to keep up)AND (n_dead_tup::numeric / NULLIF(n_live_tup, 0)) > 0.30),bloated_indexes AS (@@ -233,8 +437,6 @@ queries:WHERE schemaname IN ('openremote', 'public')-- 1. NOISE FILTER: Only look at indexes larger than 10MB.AND pg_relation_size(indexrelid) > 10 * 1024 * 1024- -- 2. ALARM THRESHOLD: Index size is > 150% of the raw table size.- -- (Catches severe page-splitting and fragmentation)AND (pg_relation_size(indexrelid)::numeric / GREATEST(pg_relation_size(relid), 1)) > 1.50)SELECTdiff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 8f97aef90..8dec9776e 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -3,72 +3,61 @@ package org.openremote.manager.datapoint;import org.hibernate.Session;import org.openremote.agent.protocol.ProtocolDatapointService;import org.openremote.container.timer.TimerService;-import org.openremote.manager.asset.OutdatedAttributeEvent;-import org.openremote.model.datapoint.DatapointExportFormat;-import org.openremote.model.datapoint.DatapointQueryTooLargeException;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetStorageService;+import org.openremote.manager.asset.OutdatedAttributeEvent;import org.openremote.manager.event.ClientEventService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebService;-import org.openremote.model.Constants;import org.openremote.model.Container;import org.openremote.model.asset.Asset;-import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.AttributeWriteFailure;import org.openremote.model.datapoint.AssetDatapoint;-import org.openremote.model.query.AssetQuery;-import org.openremote.model.query.filter.AttributePredicate;-import org.openremote.model.query.filter.NameValuePredicate;+import org.openremote.model.datapoint.DatapointExportFormat;+import org.openremote.model.datapoint.DatapointQueryTooLargeException;import org.openremote.model.syslog.SyslogCategory;-import org.openremote.model.util.Pair;import org.openremote.model.value.MetaHolder;import org.openremote.model.value.MetaItemType;import org.postgresql.PGConnection;import org.postgresql.copy.CopyManager;-import java.io.*;-import java.sql.Date;+import java.io.IOException;+import java.io.PipedInputStream;+import java.io.PipedOutputStream;import java.sql.PreparedStatement;import java.sql.Statement;-import java.time.*;-import java.util.Arrays;-import java.util.HashMap;-import java.util.List;-import java.util.Map;+import java.sql.Timestamp;+import java.time.Duration;+import java.time.Instant;+import java.time.ZoneId;+import java.util.*;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;-import java.util.regex.Pattern;import java.util.stream.Collectors;-import java.util.Set;-import java.util.TreeSet;-import static java.time.temporal.ChronoUnit.DAYS;-import static java.util.stream.Collectors.groupingBy;-import static java.util.stream.Collectors.toList;import static org.openremote.model.syslog.SyslogCategory.DATA;import static org.openremote.model.util.MapAccess.getInteger;import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;/**- * Store and retrieve datapoints for asset attributes and periodically purge data points based on- * {@link MetaItemType#DATA_POINTS_MAX_AGE_DAYS} {@link org.openremote.model.attribute.MetaItem}- * and {@link #OR_DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT}- * days.+ * Store and retrieve datapoints for asset attributes and periodically purge data points using+ * TimescaleDB's {@code drop_chunks()} with week-based retention aligned to 7-day chunk intervals.+ * Retention defaults to inifinite.*/public class AssetDatapointService extends AbstractDatapointService<AssetDatapoint> implements ProtocolDatapointService {+ public static final String OR_DATA_POINTS_MAX_AGE_WEEKS = "OR_DATA_POINTS_MAX_AGE_WEEKS";+ /** @deprecated Use {@link #OR_DATA_POINTS_MAX_AGE_WEEKS} instead. If set, value is converted to weeks (rounded up). */+ @Deprecatedpublic static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";- public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;public static final String OR_DATA_POINTS_EXPORT_LIMIT = "OR_DATA_POINTS_EXPORT_LIMIT";public static final int OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT = 1000000;private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());private static final Logger DATA_EXPORT_LOG = SyslogCategory.getLogger(DATA, AssetDatapointResourceImpl.class);-- protected int maxDatapointAgeDays;+ protected int maxDatapointAgeWeeks = -1;protected int datapointExportLimit;@Override@@ -84,12 +73,20 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi));- maxDatapointAgeDays = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_DAYS, OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT);+ if (container.getConfig().containsKey(OR_DATA_POINTS_MAX_AGE_WEEKS)) {+ maxDatapointAgeWeeks = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_WEEKS, -1);+ } else if (container.getConfig().containsKey(OR_DATA_POINTS_MAX_AGE_DAYS)) {+ int days = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_DAYS, -1);+ maxDatapointAgeWeeks = days > 0 ? (int) Math.ceil(days / 7.0) : days;+ LOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " is deprecated, use " + OR_DATA_POINTS_MAX_AGE_WEEKS + " instead. Converted " + days + " days to " + maxDatapointAgeWeeks + " weeks.");+ } else {+ maxDatapointAgeWeeks = -1;+ }- if (maxDatapointAgeDays <= 0) {- LOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");+ if (maxDatapointAgeWeeks <= 0) {+ LOG.warning("Data point purge disabled");} else {- LOG.log(Level.INFO, "Data point purge interval days = " + maxDatapointAgeDays);+ LOG.log(Level.INFO, "Data point purge retention = " + maxDatapointAgeWeeks + " weeks");}datapointExportLimit = getInteger(container.getConfig(), OR_DATA_POINTS_EXPORT_LIMIT, OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT);@@ -103,11 +100,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi@Overridepublic void start(Container container) throws Exception {- if (maxDatapointAgeDays > 0) {+ if (maxDatapointAgeWeeks > 0) {dataPointsPurgeScheduledFuture = scheduledExecutorService.scheduleAtFixedRate(this::purgeDataPoints,getFirstPurgeMillis(timerService.getNow()),- Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS+ Duration.ofDays(7).toMillis(), TimeUnit.MILLISECONDS);}@@ -151,109 +148,62 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi}protected void purgeDataPoints() {- LOG.info("Running data points purge daily task");+ LOG.info("Running data points purge task");try {- // 1. Get list of attributes that have custom durations- List<Asset<?>> assets = assetStorageService.findAll(- new AssetQuery()- .attributes(- new AttributePredicate().meta(- new NameValuePredicate(MetaItemType.DATA_POINTS_MAX_AGE_DAYS, null)- )));-- List<Pair<String, Attribute<?>>> attributes = assets.stream()- .map(asset -> asset- .getAttributes().stream()- .filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))- .map(assetAttribute -> new Pair<String, Attribute<?>>(asset.getId(), assetAttribute))- .collect(toList()))- .flatMap(List::stream)- .toList();-- // 2. Perform the purge using a temporary table and native SQLpersistenceService.doTransaction(em -> {- em.unwrap(Session.class).doWork(connection -> {- String tempTableName = "tmp_purge_attributes";+ // Get the schema-qualified hypertable name+ @SuppressWarnings("unchecked")+ List<String> hypertables = em.createNativeQuery(+ "SELECT hypertable_schema || '.' || hypertable_name " ++ "FROM timescaledb_information.hypertables " ++ "WHERE hypertable_name = 'asset_datapoint'", String.class+ ).getResultList();++ if (hypertables.isEmpty()) {+ LOG.warning("asset_datapoint is not a TimescaleDB hypertable, skipping purge. " ++ "TimescaleDB with hypercore is required for data point storage.");+ return;+ }- try (Statement statement = connection.createStatement()) {- // Create temp table holding the asset ID, attribute name, and its custom max age- statement.execute(- "create temp table " + tempTableName + " (entity_id text, attribute_name text, max_age_days int) on commit drop"- );+ String qualifiedTableName = hypertables.getFirst();++ // Find the maximum custom retention across all attributes with DATA_POINTS_MAX_AGE_DAYS meta+ int effectiveRetentionWeeks = maxDatapointAgeWeeks;+ Number maxCustomDays = (Number) em.createNativeQuery(+ "SELECT MAX((attr_val->'meta'->>'dataPointsMaxAgeDays')::integer) " ++ "FROM asset, jsonb_each(asset.attributes) AS a(attr_key, attr_val) " ++ "WHERE jsonb_exists(attr_val->'meta', 'dataPointsMaxAgeDays')"+ ).getSingleResult();++ if (maxCustomDays != null) {+ int customRetentionWeeks = (int) Math.ceil(maxCustomDays.intValue() / 7.0);+ effectiveRetentionWeeks = Math.max(effectiveRetentionWeeks, customRetentionWeeks);+ if (customRetentionWeeks > maxDatapointAgeWeeks) {+ LOG.info("Custom attribute retention (" + maxCustomDays.intValue() + " days / " ++ customRetentionWeeks + " weeks) exceeds system default (" + maxDatapointAgeWeeks ++ " weeks), using " + effectiveRetentionWeeks + " weeks");}+ }- // Batch insert the custom retention rules to avoid giant IN clauses- if (!attributes.isEmpty()) {- try (PreparedStatement insertStatement = connection.prepareStatement(- "insert into " + tempTableName + " (entity_id, attribute_name, max_age_days) values (?, ?, ?)"- )) {- for (Pair<String, Attribute<?>> attributeRef : attributes) {- insertStatement.setString(1, attributeRef.key);- insertStatement.setString(2, attributeRef.value.getName());-- int customAge = attributeRef.value- .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)- .orElse(maxDatapointAgeDays);- insertStatement.setInt(3, customAge);-- insertStatement.addBatch();- }- insertStatement.executeBatch();- }- }+ // Compute cutoff from effective retention+ Instant cutoff = timerService.getNow().minus(Duration.ofDays((long) effectiveRetentionWeeks * 7));+ Timestamp cutoffTimestamp = Timestamp.from(cutoff);- // Purge default data points (NOT in the temp table)- LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);- long defaultEpochMillis = timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS).toEpochMilli();+ LOG.info("Dropping chunks older than " + cutoffTimestamp + " (" + effectiveRetentionWeeks + " weeks retention)");- try (PreparedStatement defaultPurge = connection.prepareStatement(- "delete from asset_datapoint ad " +- "where ad.timestamp < to_timestamp(?) " +- "and not exists (select 1 from " + tempTableName + " t where t.entity_id = ad.entity_id and t.attribute_name = ad.attribute_name)"- )) {- defaultPurge.setLong(1, defaultEpochMillis / 1000);- defaultPurge.executeUpdate();- }+ Number dropped = (Number) em.createNativeQuery("SELECT count(*) FROM public.drop_chunks(CAST(:hypertable AS regclass), older_than => CAST(:cutoff AS timestamp))")+ .setParameter("hypertable", qualifiedTableName)+ .setParameter("cutoff", cutoffTimestamp)+ .getSingleResult();- // Purge custom data points by joining against the temp table- if (!attributes.isEmpty()) {- LOG.fine("Purging data points with custom max age constraints");- long currentEpochMillis = timerService.getNow().truncatedTo(DAYS).toEpochMilli();-- try (PreparedStatement customPurge = connection.prepareStatement(- "delete from asset_datapoint ad " +- "using " + tempTableName + " t " +- "where ad.entity_id = t.entity_id " +- "and ad.attribute_name = t.attribute_name " +- // Delete where timestamp is older than (current date - custom age)- "and ad.timestamp < to_timestamp(? - (t.max_age_days * 86400))"- )) {- customPurge.setLong(1, currentEpochMillis / 1000);- customPurge.executeUpdate();- }- }- });+ LOG.info("Successfully purged data points using drop_chunks, drop count = " + dropped);});-} catch (Exception e) {- LOG.log(Level.WARNING, "Failed to run data points purge", e);+ LOG.log(Level.SEVERE, "Failed to run data points purge", e);}}- protected String buildWhereClause(List<Pair<String, Attribute<?>>> attributes, boolean negate) {-- if (attributes.isEmpty()) {- return "";- }-- String whereStr = attributes.stream()- .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")- .collect(Collectors.joining(","));-- return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";- }-/*** Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped into the manager* container so it can be accessed by this process.diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sqlnew file mode 100644index 000000000..8f7dec3df--- /dev/null+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sql@@ -0,0 +1,38 @@+-- This is a function to ensure indexes are used for the dynamically retrieved entity and attribute.++CREATE OR REPLACE FUNCTION datapoint_metric_query()+RETURNS integer AS $$+DECLARE+target_entity text;+ target_attr text;+ result_count integer;+BEGIN+ -- 1. Find the heaviest entity and attribute safely+ -- REMOVED the jsonb_typeof check to guarantee an Index Only Scan+SELECT entity_id, attribute_name+INTO target_entity, target_attr+FROM asset_datapoint+WHERE timestamp > now() - INTERVAL '1 day'+GROUP BY entity_id, attribute_name+ORDER BY count(*) DESC+ LIMIT 1;++-- 2. Run the heavy aggregation using dynamic SQL.+EXECUTE format('+ WITH aggregated_points AS (+ SELECT+ time_bucket(''1 hour'', timestamp) AS bucket,+ avg((value::text)::numeric) AS avg_value+ FROM asset_datapoint+ WHERE entity_id = %L+ AND attribute_name = %L+ AND timestamp BETWEEN now() - INTERVAL ''14 days'' AND now() - INTERVAL ''7 days''+ GROUP BY bucket+ )+ SELECT count(bucket) FROM aggregated_points;+ ', target_entity, target_attr) INTO result_count;++-- 3. Return the final row count to the exporter+RETURN result_count;+END;+$$ LANGUAGE plpgsql;diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sqlnew file mode 100644index 000000000..7b1bb0c88--- /dev/null+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sql@@ -0,0 +1,42 @@+-- FAIL FAST IF THE DB IMAGE DOES NOT SUPPORT HYPERCORE+DO $$+ DECLARE+ ts_version text;+ has_hypercore_policy_proc boolean;+ BEGIN+ -- 1. EXISTING VERSION CHECKS+ SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';++ IF ts_version IS NULL THEN+ RAISE EXCEPTION 'TimescaleDB extension is missing.';+ END IF;++ -- 2. SAFE REMOVAL (Sub-blocks for error handling)+ -- This "nests" the logic within your existing block+ BEGIN+ -- remove_compression_policy is a FUNCTION+ -- Use PERFORM to call a function inside PL/pgSQL+ PERFORM public.remove_compression_policy('${schemaName}.asset_datapoint');+ EXCEPTION WHEN OTHERS THEN+ RAISE NOTICE 'Could not remove legacy policy: %', SQLERRM;+ END;++ BEGIN+ -- remove_columnstore_policy is a PROCEDURE+ -- Use EXECUTE 'CALL...' for maximum compatibility+ EXECUTE 'CALL public.remove_columnstore_policy(''${schemaName}.asset_datapoint'', if_exists => true)';+ EXCEPTION WHEN OTHERS THEN+ RAISE NOTICE 'Could not remove columnstore policy: %', SQLERRM;+ END;++ END $$;++-- 3. ENABLE COLUMNSTORE (Must be outside DO block if it requires its own transaction)+ALTER TABLE ${schemaName}.asset_datapoint SET (+ timescaledb.enable_columnstore = true,+ timescaledb.orderby = 'timestamp DESC',+ timescaledb.segmentby = 'entity_id,attribute_name',+ timescaledb.chunk_interval = '7 days');++-- ACTIVATE THE AUTOMATED POLICY+CALL public.add_columnstore_policy('${schemaName}.asset_datapoint', after => INTERVAL '7 days');diff --git a/model/src/main/java/org/openremote/model/value/MetaItemType.java b/model/src/main/java/org/openremote/model/value/MetaItemType.javaindex a7ab01295..777a5ea40 100644--- a/model/src/main/java/org/openremote/model/value/MetaItemType.java+++ b/model/src/main/java/org/openremote/model/value/MetaItemType.java@@ -92,8 +92,12 @@ public final class MetaItemType {public static final MetaItemDescriptor<Boolean> STORE_DATA_POINTS = new MetaItemDescriptor<>("storeDataPoints", ValueType.BOOLEAN);/**- * How long to store attribute values in time series database (data older than this will be automatically purged)+ * How long to store attribute values in time series database (data older than this will be automatically purged).+ * @deprecated Per-attribute retention is no longer supported. Purging now uses TimescaleDB drop_chunks with+ * a global week-based retention period configured via OR_DATA_POINTS_MAX_AGE_WEEKS. This meta item+ * is retained for backward compatibility but has no effect.*/+ @Deprecatedpublic static final MetaItemDescriptor<Integer> DATA_POINTS_MAX_AGE_DAYS = new MetaItemDescriptor<>("dataPointsMaxAgeDays", ValueType.POSITIVE_INTEGER);/**diff --git a/profile/deploy.yml b/profile/deploy.ymlindex 0a1dd7aa5..e12b8f0b4 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -486,7 +486,7 @@ services:- deployment-data:/deploymentports:- "127.0.0.1:8406:9560" # Localhost metrics access- - "${PRIVATE_IP:-127.0.0.1}:8406:8404" # Allows to also expose metrics on a given IP address,+ - "${PRIVATE_IP:-127.0.0.1}:8406:9560" # Allows to also expose metrics on a given IP address,# intended to be IP of the interface of the private subnet of the EC2 VMenvironment:TZ: ${TZ:-Europe/Amsterdam}diff --git a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.javaindex 85cadb4b3..b5d87839c 100644--- a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java+++ b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java@@ -42,7 +42,6 @@ import java.util.Arrays;import java.util.List;import java.util.Map;-import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT;import static org.openremote.model.Constants.*;import static org.openremote.model.value.MetaItemType.*;import static org.openremote.model.value.ValueType.*;@@ -165,10 +164,6 @@ public class ManagerTestSetup extends ManagerSetup {new MetaItem<>(STORE_DATA_POINTS,true),- new MetaItem<>(- DATA_POINTS_MAX_AGE_DAYS,- OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT *7- ),new MetaItem<>(AGENT_LINK,new SimulatorAgentLink(agent.getId()))diff --git a/test/load3/README.md b/test/load3/README.mdnew file mode 100644index 000000000..37a307ba3--- /dev/null+++ b/test/load3/README.md@@ -0,0 +1,45 @@+# TimescaleDB Benchmark++This script is an automated load-generator designed to benchmark query performance on a TimescaleDB instance; it can be used to compare performance before and after a DB configuration change.++## What it does+Instead of testing generic, hardcoded queries, this script dynamically adapts to your actual data:+1. **Dynamic Target Acquisition:** It queries the `asset_datapoint` table to find the 10 entities and attributes with the highest volume of numeric data over the last 3 days.+2. **Heavy Load Generation:** For each of those 10 targets, it executes three realistic, heavy time-series queries:+ * A 7-day hourly average.+ * A 14-day Largest Triangle Three Buckets (LTTB) visual downsample (500 points).+ * A 30-day daily Min/Max/Avg calculation.+3. **Pure Execution Timing:** It wraps every query in an `EXPLAIN ANALYZE` command to extract the exact millisecond execution time directly from the PostgreSQL engine, completely eliminating Docker or network latency from the benchmark.+4. **Excel-Ready Output:** It formats the results into a self-describing CSV file (`results.csv`), making it trivial to generate charts in Excel or Grafana.++## Prerequisites+* The script must be run on the host machine running the PostgreSQL Docker container.+* The TimescaleDB instance must have the `timescaledb_toolkit` extension installed (required for the `lttb()` downsampling function).+* The default Docker container name is expected to be `or-postgresql-1`. (This can be changed at the top of the script).+* The DB must have the `datapoint_metric_query` function as added in this FlywayDB migration script (`V20260506_01__DatapointMetricQuery.sql`)++## How to Run++1. Make the script executable:+ `chmod +x benchmark.sh`++2. Execute the script:+ `./benchmark.sh`++## Customization+You can easily adjust the behavior of the script by editing the configuration block at the top of `benchmark.sh`:+* `ITERATIONS`: Controls how many times the full suite of queries runs (Default is `5`).+* `CONTAINER_NAME`: Update this if your OpenRemote database container has a different name.++## Understanding the Output (`results.csv`)+The script generates a CSV file where the first column is the exact ISO8601 timestamp of when the iteration began.++The subsequent columns contain the query execution times in milliseconds. The headers are dynamically generated using a combination of the Entity ID (first 4 characters), Attribute Name, and Query Type.++**Example Header:** `4hOF-kwMax_Hourly_Avg_7d_ms`+* `4hOF`: First 4 characters of the `entity_id`+* `kwMax`: The `attribute_name`+* `Hourly_Avg_7d`: The query template executed+* `ms`: Value is measured in milliseconds++To visualize your benchmark, simply open `results.csv` in Excel, select all data, and insert a Line Chart. The headers will automatically generate a perfectly formatted legend.diff --git a/test/load3/benchmark.sh b/test/load3/benchmark.shnew file mode 100644index 000000000..b192c52ae--- /dev/null+++ b/test/load3/benchmark.sh@@ -0,0 +1,111 @@+#!/bin/bash++# Configuration+CONTAINER_NAME="or-postgresql-1"+DB_USER="postgres"+DB_NAME="openremote"+CSV_FILE="results.csv"+ITERATIONS=5++function run_psql() {+ docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -t -A -c "$1"+}++echo "Starting Benchmark Initialization..."++# 1. Find the top 10 entity_id & attribute_name values+TOP_ENTITIES_QUERY="+SELECT entity_id, attribute_name+FROM asset_datapoint+WHERE timestamp > now() - INTERVAL '3 days'+ AND jsonb_typeof(value) = 'number'+GROUP BY entity_id, attribute_name+ORDER BY count(*) DESC+LIMIT 10;+"++echo "Fetching top 10 entities..."+ENTITIES=$(run_psql "$TOP_ENTITIES_QUERY")++if [ -z "$ENTITIES" ]; then+ echo "Error: No numeric entities found in the last 3 days. Exiting."+ exit 1+fi++# 2. Define the Query Templates+QUERY_TEMPLATES=(+ "SELECT time_bucket('1 hour', timestamp) as bucket, avg((value::text)::numeric) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '7 days' GROUP BY bucket;"+ "SELECT time, value FROM unnest((SELECT lttb(timestamp, (value::text)::numeric, 500) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '14 days'));"+ "SELECT time_bucket('1 day', timestamp) as bucket, min((value::text)::numeric), max((value::text)::numeric), avg((value::text)::numeric) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '30 days' GROUP BY bucket;"+)+QUERY_NAMES=("Hourly_Avg_7d" "LTTB_500_14d" "Daily_MinMaxAvg_30d")++# 3. Build the descriptive CSV headers+echo "Generating headers..."+CSV_HEADER="Timestamp"++declare -a ENTITY_IDS+declare -a ATTRIBUTE_NAMES++while IFS='|' read -r entity_id attribute_name; do+ ENTITY_IDS+=("$entity_id")+ ATTRIBUTE_NAMES+=("$attribute_name")+done <<< "$ENTITIES"++# Create descriptive column headers+for i in "${!ENTITY_IDS[@]}"; do+ E_ID="${ENTITY_IDS[$i]}"+ A_NAME="${ATTRIBUTE_NAMES[$i]}"++ # SAFER PARSING: Grab first 4 chars of ID using 'cut' instead of bash string slicing+ SHORT_ID="$(echo "$E_ID" | cut -c 1-4)-${A_NAME}"++ for q in "${!QUERY_NAMES[@]}"; do+ Q_NAME="${QUERY_NAMES[$q]}"+ CSV_HEADER="${CSV_HEADER},${SHORT_ID}_${Q_NAME}_ms"+ done+done++# Initialize the CSV file+echo "$CSV_HEADER" > "$CSV_FILE"+echo "Headers created successfully."++# 4. Execute the load loop+echo "Starting load generation ($ITERATIONS iterations)..."++for (( iter=1; iter<=ITERATIONS; iter++ )); do+ START_TIME=$(date -Iseconds)+ CSV_ROW="$START_TIME"++ echo "Iteration $iter / $ITERATIONS starting at $START_TIME"++ for i in "${!ENTITY_IDS[@]}"; do+ E_ID="${ENTITY_IDS[$i]}"+ A_NAME="${ATTRIBUTE_NAMES[$i]}"++ for q in "${!QUERY_TEMPLATES[@]}"; do+ TEMPLATE="${QUERY_TEMPLATES[$q]}"++ RAW_QUERY=$(printf "$TEMPLATE" "$E_ID" "$A_NAME")+ ANALYZE_QUERY="EXPLAIN ANALYZE $RAW_QUERY"++ # SAFER PARSING: Execute and extract execution time.+ # Replaced 'awk' with 'tr' and 'cut' to avoid curly braces inside the subshell!+ OUT=$(run_psql "$ANALYZE_QUERY" 2>&1)+ EXEC_TIME=$(echo "$OUT" | grep "Execution Time" | tr -s ' ' | cut -d' ' -f3)++ if [ -z "$EXEC_TIME" ]; then+ EXEC_TIME="ERROR"+ echo " [!] Error running $A_NAME (Iter $iter)"+ fi++ CSV_ROW="${CSV_ROW},${EXEC_TIME}"+ done+ done++ # Write the row to the CSV+ echo "$CSV_ROW" >> "$CSV_FILE"+ echo " Completed iteration $iter."+done++echo "Benchmark complete! Results saved to $CSV_FILE"diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovynew file mode 100644index 000000000..1e3ae5a23--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy@@ -0,0 +1,246 @@+package org.openremote.test.assets++import org.hibernate.cfg.AvailableSettings+import org.openremote.container.persistence.PersistenceService+import org.openremote.manager.asset.AssetStorageService+import org.openremote.manager.datapoint.AssetDatapointService+import org.openremote.manager.datapoint.AssetPredictedDatapointService+import org.openremote.manager.setup.SetupService+import org.openremote.model.asset.impl.ThingAsset+import org.openremote.model.attribute.Attribute+import org.openremote.model.attribute.AttributeRef+import org.openremote.model.datapoint.ValueDatapoint+import org.openremote.model.value.ValueType+import org.openremote.setup.integration.KeycloakTestSetup+import org.openremote.test.ManagerContainerTrait+import spock.lang.Specification+import spock.util.concurrent.PollingConditions++import java.time.Instant+import java.time.temporal.ChronoUnit+import java.util.concurrent.TimeUnit++import static java.util.concurrent.TimeUnit.DAYS+import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_WEEKS++class AssetDatapointPurgeTest extends Specification implements ManagerContainerTrait {++ def "Test TimescaleDB hypercore compression and purge"() {++ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 30, delay: 1)++ and: "the container is started with a 4-week retention period"+ def config = defaultConfig()+ config.put(OR_DATA_POINTS_MAX_AGE_WEEKS, "4")+ def container = startContainer(config, defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)+ def persistenceService = container.getService(PersistenceService.class)++ and: "the schema name is retrieved"+ def schemaName = persistenceService.persistenceUnitProperties.getProperty(AvailableSettings.DEFAULT_SCHEMA)+ getLOG().info("Using schema: ${schemaName}")++ and: "the clock is stopped and advanced to a known time"+ stopPseudoClock()+ advancePseudoClock(Instant.ofEpochMilli(getClockTimeOf(container)).truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS).toEpochMilli() - getClockTimeOf(container), TimeUnit.MILLISECONDS, container)++ when: "a test asset is created with multiple attributes"+ def testAsset = new ThingAsset("Hypercore Test Asset")+ .setRealm(keycloakTestSetup.realmMaster.name)+ testAsset.addOrReplaceAttributes(+ new Attribute<>("temperature", ValueType.NUMBER),+ new Attribute<>("humidity", ValueType.NUMBER),+ new Attribute<>("pressure", ValueType.NUMBER),+ new Attribute<>("co2", ValueType.NUMBER)+ )+ testAsset = assetStorageService.merge(testAsset)++ then: "the asset should be created"+ testAsset.id != null+ getLOG().info("Created test asset with ID: ${testAsset.id}")++ when: "25.000 datapoints are inserted across multiple attributes with some within the 4-week retention window"+ def totalDatapoints = 25_000+ def attributeNames = ["temperature", "humidity", "pressure", "co2"]+ def datapointsPerAttribute = (totalDatapoints / attributeNames.size()) as int+ def recentDatapointsPerAttribute = (datapointsPerAttribute * 0.2) as int+ def oldDatapointsPerAttribute = datapointsPerAttribute - recentDatapointsPerAttribute+ def expectedRecentDatapoints = recentDatapointsPerAttribute * attributeNames.size()+ def currentTime = getClockTimeOf(container)+ def oldBaseTimestamp = Instant.ofEpochMilli(currentTime).minus(365, ChronoUnit.DAYS)+ def recentBaseTimestamp = Instant.ofEpochMilli(currentTime).minus(7, ChronoUnit.DAYS)++ getLOG().info("Starting to insert ${totalDatapoints} datapoints...")+ def insertStartTime = System.currentTimeMillis()++ // Insert datapoints in batches for better performance+ def batchSize = 10000+ attributeNames.each { attributeName ->+ getLOG().info("Inserting ${datapointsPerAttribute} datapoints for attribute: ${attributeName}")++ for (int batchStart = 0; batchStart < datapointsPerAttribute; batchStart += batchSize) {+ def batchEnd = Math.min(batchStart + batchSize, datapointsPerAttribute)+ def datapoints = []+ for (int index = batchStart; index < batchEnd; index++) {+ def timestamp = index < oldDatapointsPerAttribute+ ? oldBaseTimestamp.plus(index * 30, ChronoUnit.SECONDS).toEpochMilli()+ : recentBaseTimestamp.plus((index - oldDatapointsPerAttribute) * 30, ChronoUnit.SECONDS).toEpochMilli()+ def value = 20.0 + (Math.sin(index / 100.0) * 10.0) + (Math.random() * 2.0)+ datapoints.add(new ValueDatapoint<>(timestamp, value))+ }+ assetDatapointService.upsertValues(testAsset.id, attributeName, datapoints)++ if (batchEnd % (batchSize * 10) == 0 || batchEnd == datapointsPerAttribute) {+ getLOG().info(" Progress: ${batchEnd} / ${datapointsPerAttribute} datapoints inserted for ${attributeName}")+ }+ }+ }++ def insertEndTime = System.currentTimeMillis()+ def insertDuration = (insertEndTime - insertStartTime) / 1000.0+ getLOG().info("Finished inserting ${totalDatapoints} datapoints in ${insertDuration} seconds")++ then: "datapoints should be stored"+ conditions.eventually {+ def count = 0+ attributeNames.each { attributeName ->+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))+ count += datapoints.size()+ }+ assert count >= totalDatapoints * 0.99 // Allow for 1% margin+ getLOG().info("Verified ${count} datapoints stored")+ }++ when: "storage usage is measured before enabling hypercore"+ def orDatabaseSizeBefore = persistenceService.doReturningTransaction { em ->+ def query = em.createNativeQuery("""+ SELECT pg_database_size(pg_database.datname)+ FROM pg_database+ WHERE pg_database.datname = 'openremote';+ """)+ return query.getSingleResult() as Long+ }++ getLOG().info("\n=== Storage BEFORE Hypercore ===")+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeBefore / (1024.0 * 1024.0))}")++ and: "compression job is called manually"+ def job_id = persistenceService.doReturningTransaction { em ->+ def query = em.createNativeQuery("""+ SELECT job_id+ FROM timescaledb_information.jobs+ WHERE proc_name = 'policy_compression' AND hypertable_name = 'asset_datapoint';+ """)+ return query.getSingleResult()+ }++ // run_job contains internal COMMIT statements, so it must be executed outside of a Hibernate transaction+ def dataSource = persistenceService.persistenceUnitProperties.get(AvailableSettings.DATASOURCE) as javax.sql.DataSource+ def connection = dataSource.getConnection()+ try {+ connection.setAutoCommit(true)+ def stmt = connection.prepareCall("CALL public.run_job(?)")+ stmt.setInt(1, job_id as int)+ stmt.execute()+ stmt.close()+ } finally {+ connection.close()+ }++ def orDatabaseSizeAfter = persistenceService.doReturningTransaction { em ->+ def query = em.createNativeQuery("""+ SELECT pg_database_size(pg_database.datname)+ FROM pg_database+ WHERE pg_database.datname = 'openremote';+ """)+ return query.getSingleResult() as Long+ }++ getLOG().info("\n=== Storage AFTER Hypercore ===")+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeAfter / (1024.0 * 1024.0))}")++ then: "storage should be smaller after hypercore"+ orDatabaseSizeBefore > orDatabaseSizeAfter++ when: "Purging will be called, count datapoints before purging"+ def countBeforePurge = 0+ attributeNames.each { attributeName ->+ def dataPoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))+ countBeforePurge += dataPoints.size()+ }+ getLOG().info("Datapoints before purge: ${countBeforePurge}")++ and: "the purge routine is executed"+ def deleteStartTime = System.currentTimeMillis()+ assetDatapointService.purgeDataPoints()+ def deleteEndTime = System.currentTimeMillis()+ def deleteDuration = (deleteEndTime - deleteStartTime) / 1000.0++ getLOG().info("Purge completed in ${deleteDuration} seconds")++ then: "data points beyond the 4-week retention window should be purged via drop_chunks"+ conditions.eventually {+ def countAfterPurge = 0+ attributeNames.each { attributeName ->+ def dataPoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))+ countAfterPurge += dataPoints.size()+ }+ def deletedCount = countBeforePurge - countAfterPurge++ getLOG().info("Datapoints after purge: ${countAfterPurge}")+ getLOG().info("Deleted ${deletedCount} datapoints")+ getLOG().info("Deletion rate: ${String.format('%.0f', deletedCount / deleteDuration)} datapoints/second")++ // With 365 days of data and 4-week (28-day) retention, most data should be purged+ assert countAfterPurge < countBeforePurge+ assert countAfterPurge >= expectedRecentDatapoints+ assert deletedCount > 0+ }++ when: "storage usage is measured after deletion"+ def orDatabaseSizeAfterDeletion = persistenceService.doReturningTransaction { em ->+ def query = em.createNativeQuery("""+ SELECT pg_database_size(pg_database.datname)+ FROM pg_database+ WHERE pg_database.datname = 'openremote';+ """)+ return query.getSingleResult() as Long+ }++ getLOG().info("\n=== Storage AFTER Deletion ===")+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeAfterDeletion / (1024.0 * 1024.0))}")++ then: "final storage should be measured"+ true++ when: "final verification of remaining datapoints"+ def finalCount = 0+ attributeNames.each { attributeName ->+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))+ finalCount += datapoints.size()+ }+ def actualDeleted = countBeforePurge - finalCount+ def compressionRatio = ((orDatabaseSizeBefore - orDatabaseSizeAfter) / (double) orDatabaseSizeBefore) * 100++ getLOG().info("\n=== Final Summary ===")+ getLOG().info("Total datapoints inserted: ${totalDatapoints}")+ getLOG().info("Datapoints before purge: ${countBeforePurge}")+ getLOG().info("Datapoints deleted by purge: ${actualDeleted}")+ getLOG().info("Datapoints remaining: ${finalCount}")+ getLOG().info("Deletion percentage: ${String.format('%.2f', (actualDeleted / totalDatapoints * 100.0))}%")+ getLOG().info("\nStorage Summary:")+ getLOG().info(" Before hypercore: ${String.format('%.2f MB', orDatabaseSizeBefore / (1024.0 * 1024.0))}")+ getLOG().info(" After hypercore: ${String.format('%.2f MB', orDatabaseSizeAfter / (1024.0 * 1024.0))}")+ getLOG().info(" After deletion: ${String.format('%.2f MB', orDatabaseSizeAfterDeletion / (1024.0 * 1024.0))}")+ getLOG().info("\nPerformance Summary:")+ getLOG().info(" Insert time: ${insertDuration} seconds")+ getLOG().info(" Purge time: ${deleteDuration} seconds")+ getLOG().info(" Compression: ${String.format('%.2f', compressionRatio)}%")++ then: "No exception should have occurred"+ true+ }+}diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovyindex 279e06c32..b740dae59 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy@@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnitimport java.util.function.Consumerimport static java.util.concurrent.TimeUnit.*-import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULTimport static org.openremote.model.Constants.KEYCLOAK_CLIENT_IDimport static org.openremote.model.Constants.MASTER_REALMimport static org.openremote.setup.integration.ManagerTestSetup.thingLightToggleAttributeName@@ -109,7 +108,6 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitdef conditions = new PollingConditions(timeout: 10, delay: 0.2)when: "the demo agent and thing have been deployed"- def datapointPurgeDays = OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULTdef container = startContainer(defaultConfig(), defaultServices())def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)def agentService = container.getService(AgentService.class)@@ -353,107 +351,6 @@ class AssetDatapointTest extends Specification implements ManagerContainerTraitassert thing.getAttribute(thingLightToggleAttributeName).flatMap {it.getTimestamp()}.orElse(0) == getClockTimeOf(container)}- // ------------------------------------- // Test purging of data points- // -------------------------------------- when: "time moves forward by more than purge days"- advancePseudoClock(datapointPurgeDays + 1, DAYS, container)-- and: "the power sensor with default max age receives a new value"- def datapoint4ExpectedTimestamp = getClockTimeOf(container)- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 17.5d)-- and: "the toggle sensor with a custom max age of 7 days receives a new value"- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), true)-- then: "the datapoints should be stored"- conditions.eventually {- //def powerDatapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List- //def toggleDatapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List- def powerDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))- def toggleDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))-- assert powerDatapoints.size() >= 6- assert powerDatapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value == 17.5 }-- assert toggleDatapoints.size() >= 4- assert toggleDatapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}- }-- when: "the clock advances to the next days purge routine execution time"- advancePseudoClock(assetDatapointService.getFirstPurgeMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)-- and: "the purge routine runs"- assetDatapointService.purgeDataPoints()-- then: "data points older than purge days should be purged for the power sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))- assert datapoints.size() == 1- assert ValueUtil.getValue(datapoints.get(0).value, Double.class).orElse(null) == 17.5d- assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp- }-- and: "no data points should have been purged for the toggle sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))- assert datapoints.size() >= 4- assert ValueUtil.getBoolean(datapoints.get(0).value).orElse(false)- assert datapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}- }-- when: "the clock advances 3 times the purge duration"- advancePseudoClock(3 * datapointPurgeDays, DAYS, container)-- and: "the purge routine runs"- assetDatapointService.purgeDataPoints()-- then: "all data points should be purged for power sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))- assert datapoints.isEmpty()- }-- and: "no data points should have been purged for the toggle sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))- assert datapoints.size() >= 4- assert datapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}- }-- when: "the clock advances 3 times the purge duration"- advancePseudoClock(3 * datapointPurgeDays, DAYS, container)-- and: "the purge routine runs"- assetDatapointService.purgeDataPoints()-- then: "data points older than 7 days should have been purged for the toggle sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))- assert datapoints.size() == 1- assert ValueUtil.getBoolean(datapoints.get(0).value).orElse(false)- assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp- }-- when: "the clock advances by the purge duration"- advancePseudoClock(datapointPurgeDays, DAYS, container)-- and: "the purge routine runs"- assetDatapointService.purgeDataPoints()-- then: "all data points should have been purged for the toggle sensor"- conditions.eventually {- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))- assert datapoints.isEmpty()- }-when: "we subscribe to predicted data points events"List<AssetPredictedDatapointEvent> predictedEvents = new CopyOnWriteArrayList<>()Consumer<AssetPredictedDatapointEvent> predictedEventConsumer = { event ->