Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Enable TimescaleDB hypercore functionality on asset_datapoint table (#2352)

Michael Rademakera34c0df77167
diff --git a/deployment/query-exporter/config.yaml b/deployment/query-exporter/config.yaml
index 134627dd4..5a7e5228c 100644
--- a/deployment/query-exporter/config.yaml
+++ b/deployment/query-exporter/config.yaml
@@ -1,10 +1,5 @@
# Query Exporter Configuration for OpenRemote PostgreSQL Monitoring
# Documentation: https://github.com/albertodonato/query-exporter/blob/main/docs/configuration.rst
-#
-# Environment Variables:
-# - TABLE_BLOAT_THRESHOLD: Table bloat ratio threshold for alerting (default: 1.2 = 20% bloat)
-# - INDEX_BLOAT_THRESHOLD: Index bloat ratio threshold for alerting (default: 1.5 = 50% bloat)
-# Set these in docker-compose or deployment environment to adjust sensitivity
databases:
openremote:
@@ -19,6 +14,10 @@ databases:
instance: openremote
metrics:
+ pg_collation_mismatch_count:
+ type: gauge
+ description: Number of text indexes with collation version mismatches requiring a REINDEX
+
pg_cache_hit_percentage:
type: gauge
description: What percentage of data is being served instantly from RAM versus being slowly read from disk. You want this as high as possible
@@ -42,23 +41,27 @@ metrics:
pg_hot_update_percent:
type: gauge
labels: [table_name]
- description: Asset table percentage of updates that are HOT updates indicates good fillfactor
+ description: Table percentage of updates that are HOT updates indicates good fillfactor
pg_dead_tuple_percent:
type: gauge
labels: [table_name]
- description: Asset table ratio of dead tuples to live ones a ratio > 10-20% indicates not aggressive enough autovacuum
+ description: Table ratio of dead tuples to live ones a ratio > 10-20% indicates not aggressive enough autovacuum
pg_last_autovacuum_hours:
type: gauge
labels: [table_name]
- description: Asset table hours since last auto vacuum run successfully
+ description: Table hours since last auto vacuum run successfully
pg_last_autoanalyze_hours:
type: gauge
labels: [table_name]
description: Asset table hours since last auto analyze run successfully
+ pg_db_disk_size:
+ type: gauge
+ description: DB size in MB
+
pg_datapoint_raw_data_size:
type: gauge
description: Asset datapoint table raw uncompressed size in MB
@@ -99,10 +102,75 @@ metrics:
type: gauge
description: Asset datapoint table hypertable chunks not yet analyzed
+ pg_datapoint_largest_uncompressed_chunk:
+ type: gauge
+ description: Asset datapoint table largest uncompressed hypertable chunk in MB
+
+ pg_datapoint_uncompressed_cache_hit_ratio:
+ type: gauge
+ description: Asset datapoint table cache hit ratio for uncompressed chunks (Aim for 99%+)
+
+ pg_datapoint_uncompressed_blks_read_total:
+ type: counter
+ description: Asset datapoint total physical disk blocks read for uncompressed chunks (Monitor rate with spikes indicate RAM spillover)
+
+ pg_datapoint_compression_ratio:
+ type: gauge
+ description: Asset datapoint table compression ratio
+
+ pg_datapoint_query:
+ type: gauge
+ description: Dummy metric to get typical query time metric
+
pg_background_errors:
type: counter
description: Count of errors in background worker processes
+ pg_timescale_job_total_runs:
+ type: counter
+ labels: [job_id, proc_name]
+ description: TimescaleDB job total runs by job
+
+ pg_timescale_job_total_failures:
+ type: counter
+ labels: [job_id, proc_name]
+ description: TimescaleDB job total failures by job
+
+ pg_timescale_job_last_run_duration_seconds:
+ type: gauge
+ labels: [job_id, proc_name]
+ description: TimescaleDB job last run duration in seconds
+
+ pg_timescale_job_next_start_seconds:
+ type: gauge
+ labels: [job_id, proc_name]
+ description: Seconds until next scheduled run for each TimescaleDB job
+
+ pg_timescale_job_last_run_status:
+ type: gauge
+ labels: [job_id, proc_name, last_run_status]
+ description: TimescaleDB job last run status marker (1)
+
+ pg_wal_total:
+ type: counter
+ description: Total WAL written since statistics reset in MB
+
+ pg_bgwriter_checkpoints_timed_total:
+ type: counter
+ description: Scheduled checkpoints executed
+
+ pg_bgwriter_checkpoints_req_total:
+ type: counter
+ description: Requested checkpoints executed
+
+ pg_bgwriter_checkpoint_write_time_seconds_total:
+ type: counter
+ description: Total time spent writing checkpoints in seconds
+
+ pg_bgwriter_checkpoint_sync_time_seconds_total:
+ type: counter
+ description: Total time spent syncing checkpoints in seconds
+
pg_table_bloat_count:
type: gauge
description: Number of tables where dead tuples > 30% of live rows
@@ -113,6 +181,30 @@ metrics:
queries:
+ collation_mismatch:
+ interval: 86400 # Daily check
+ databases: [openremote]
+ metrics: [pg_collation_mismatch_count]
+ sql: |
+ WITH mismatched_collations AS (
+ SELECT oid
+ FROM pg_catalog.pg_collation
+ WHERE collversion IS NOT NULL
+ AND collversion <> pg_collation_actual_version(oid)
+ ),
+ mismatched_indexes AS (
+ SELECT DISTINCT d.objid AS index_oid
+ FROM pg_catalog.pg_depend d
+ JOIN mismatched_collations mc ON d.refobjid = mc.oid
+ WHERE d.classid = 'pg_class'::regclass
+ AND d.refclassid = 'pg_collation'::regclass
+ )
+ SELECT count(i.oid) AS pg_collation_mismatch_count
+ FROM pg_catalog.pg_class i
+ JOIN pg_catalog.pg_index idx ON i.oid = idx.indexrelid
+ JOIN mismatched_indexes mi ON i.oid = mi.index_oid
+ WHERE i.relkind = 'i';
+
cache_hit_ratio:
interval: 3600 # Hourly
databases: [openremote]
@@ -158,6 +250,13 @@ queries:
FROM pg_stat_user_tables
WHERE schemaname = 'openremote';
+ db_size:
+ interval: 3600 # Hourly
+ databases: [openremote]
+ metrics: [pg_db_disk_size]
+ sql: |
+ SELECT ROUND(pg_database_size(current_database()) / 1048576.0, 0) as pg_db_disk_size;
+
datapoint_table_size:
interval: 3600 # Hourly
databases: [openremote]
@@ -189,7 +288,22 @@ queries:
FROM timescaledb_information.chunks
WHERE hypertable_name = 'asset_datapoint'
GROUP BY hypertable_name;
-
+
+ datapoint_largest_uncompressed_chunk:
+ interval: 86400 # Daily
+ databases: [openremote]
+ metrics: [pg_datapoint_largest_uncompressed_chunk]
+ sql: |
+ SELECT
+ ROUND(s.total_bytes / 1048576.0, 0) AS pg_datapoint_largest_uncompressed_chunk
+ FROM timescaledb_information.chunks c
+ JOIN chunks_detailed_size('asset_datapoint') s
+ ON c.chunk_schema = s.chunk_schema
+ AND c.chunk_name = s.chunk_name
+ WHERE c.is_compressed = false
+ ORDER BY s.total_bytes DESC
+ LIMIT 1;
+
datapoint_autoanalyze:
interval: 86400 # Daily
databases: [openremote]
@@ -203,6 +317,53 @@ queries:
AND last_analyze IS NULL
AND n_mod_since_analyze > current_setting('autovacuum_analyze_threshold')::numeric;
+ datapoint_historical_aggregation:
+ interval: 3600 # Hourly
+ databases: [openremote]
+ metrics: [pg_datapoint_query]
+ sql: |
+ SELECT datapoint_metric_query() as pg_datapoint_query;
+
+ uncompressed_memory_pressure:
+ interval: 3600 # Hourly
+ databases: [openremote]
+ metrics: [pg_datapoint_uncompressed_cache_hit_ratio, pg_datapoint_uncompressed_blks_read_total]
+ sql: |
+ WITH uncompressed_chunks AS (
+ SELECT chunk_name
+ FROM timescaledb_information.chunks
+ WHERE hypertable_name = 'asset_datapoint'
+ AND is_compressed = false
+ )
+ SELECT
+ -- Calculate the percentage of data served directly from RAM
+ ROUND(
+ COALESCE(
+ (sum(s.heap_blks_hit) + sum(s.idx_blks_hit) + sum(s.toast_blks_hit))::numeric /
+ NULLIF(
+ sum(s.heap_blks_hit) + sum(s.idx_blks_hit) + sum(s.toast_blks_hit) +
+ sum(s.heap_blks_read) + sum(s.idx_blks_read) + sum(s.toast_blks_read),
+ 0) * 100,
+ 0),
+ 2) AS pg_datapoint_uncompressed_cache_hit_ratio,
+
+ -- Track the raw number of times Postgres had to go to the physical disk
+ COALESCE(sum(s.heap_blks_read) + sum(s.idx_blks_read) + sum(s.toast_blks_read), 0) AS pg_datapoint_uncompressed_blks_read_total
+ FROM pg_statio_user_tables s
+ JOIN uncompressed_chunks c ON s.relname = c.chunk_name;
+
+ compression_stats:
+ interval: 86400 # Daily
+ databases: [ openremote ]
+ metrics: [pg_datapoint_compression_ratio]
+ sql: |
+ SELECT
+ COALESCE(
+ (before_compression_total_bytes::numeric / NULLIF(after_compression_total_bytes, 0)),
+ 1.0
+ ) AS pg_datapoint_compression_ratio
+ FROM hypertable_compression_stats('asset_datapoint');
+
background_errors:
interval: 3600 # Hourly
databases: [openremote]
@@ -211,6 +372,52 @@ queries:
SELECT
SUM(total_failures) AS pg_background_errors
FROM timescaledb_information.job_stats;
+
+ timescale_job_stats:
+ interval: 3600 # Hourly
+ databases: [openremote]
+ metrics:
+ [
+ pg_timescale_job_total_runs,
+ pg_timescale_job_total_failures,
+ pg_timescale_job_last_run_duration_seconds,
+ pg_timescale_job_next_start_seconds,
+ pg_timescale_job_last_run_status
+ ]
+ sql: |
+ SELECT
+ s.job_id::text AS job_id,
+ CONCAT(j.proc_schema, '.', j.proc_name) AS proc_name,
+ COALESCE(s.last_run_status, 'unknown') AS last_run_status,
+ COALESCE(s.total_runs, 0) AS pg_timescale_job_total_runs,
+ COALESCE(s.total_failures, 0) AS pg_timescale_job_total_failures,
+ EXTRACT(EPOCH FROM COALESCE(s.last_run_duration, INTERVAL '0 seconds')) AS pg_timescale_job_last_run_duration_seconds,
+ EXTRACT(EPOCH FROM (COALESCE(s.next_start, now()) - now())) AS pg_timescale_job_next_start_seconds,
+ 1 AS pg_timescale_job_last_run_status
+ FROM timescaledb_information.job_stats s
+ JOIN timescaledb_information.jobs j ON j.job_id = s.job_id
+ WHERE j.hypertable_name = 'asset_datapoint';
+
+ write_path_pressure:
+ interval: 300 # 5 minutes
+ databases: [openremote]
+ metrics:
+ [
+ pg_wal_total,
+ pg_bgwriter_checkpoints_timed_total,
+ pg_bgwriter_checkpoints_req_total,
+ pg_bgwriter_checkpoint_write_time_seconds_total,
+ pg_bgwriter_checkpoint_sync_time_seconds_total
+ ]
+ sql: |
+ SELECT
+ (w.wal_bytes / 1048576.0) AS pg_wal_total,
+ c.num_timed AS pg_bgwriter_checkpoints_timed_total,
+ c.num_requested AS pg_bgwriter_checkpoints_req_total,
+ ROUND((c.write_time / 1000.0)::numeric, 0) AS pg_bgwriter_checkpoint_write_time_seconds_total,
+ ROUND((c.sync_time / 1000.0)::numeric, 0) AS pg_bgwriter_checkpoint_sync_time_seconds_total
+ FROM pg_stat_wal w
+ CROSS JOIN pg_stat_checkpointer c;
table_and_index_bloat:
interval: 300 # 5 minutes
@@ -221,10 +428,7 @@ queries:
SELECT count(*) AS tbl_count
FROM pg_stat_user_tables
WHERE schemaname IN ('openremote', 'public')
- -- 1. NOISE FILTER: Only look at tables larger than 5MB.
AND pg_relation_size(relid) > 5 * 1024 * 1024
- -- 2. THRESHOLD: Dead tuples are > 30% of live rows.
- -- (Autovacuum runs at 20%, so 30% means autovacuum is failing to keep up)
AND (n_dead_tup::numeric / NULLIF(n_live_tup, 0)) > 0.30
),
bloated_indexes AS (
@@ -233,8 +437,6 @@ queries:
WHERE schemaname IN ('openremote', 'public')
-- 1. NOISE FILTER: Only look at indexes larger than 10MB.
AND pg_relation_size(indexrelid) > 10 * 1024 * 1024
- -- 2. ALARM THRESHOLD: Index size is > 150% of the raw table size.
- -- (Catches severe page-splitting and fragmentation)
AND (pg_relation_size(indexrelid)::numeric / GREATEST(pg_relation_size(relid), 1)) > 1.50
)
SELECT
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 8f97aef90..8dec9776e 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -3,72 +3,61 @@ package org.openremote.manager.datapoint;
import org.hibernate.Session;
import org.openremote.agent.protocol.ProtocolDatapointService;
import org.openremote.container.timer.TimerService;
-import org.openremote.manager.asset.OutdatedAttributeEvent;
-import org.openremote.model.datapoint.DatapointExportFormat;
-import org.openremote.model.datapoint.DatapointQueryTooLargeException;
import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetStorageService;
+import org.openremote.manager.asset.OutdatedAttributeEvent;
import org.openremote.manager.event.ClientEventService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebService;
-import org.openremote.model.Constants;
import org.openremote.model.Container;
import org.openremote.model.asset.Asset;
-import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeEvent;
import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.attribute.AttributeWriteFailure;
import org.openremote.model.datapoint.AssetDatapoint;
-import org.openremote.model.query.AssetQuery;
-import org.openremote.model.query.filter.AttributePredicate;
-import org.openremote.model.query.filter.NameValuePredicate;
+import org.openremote.model.datapoint.DatapointExportFormat;
+import org.openremote.model.datapoint.DatapointQueryTooLargeException;
import org.openremote.model.syslog.SyslogCategory;
-import org.openremote.model.util.Pair;
import org.openremote.model.value.MetaHolder;
import org.openremote.model.value.MetaItemType;
import org.postgresql.PGConnection;
import org.postgresql.copy.CopyManager;
-import java.io.*;
-import java.sql.Date;
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
import java.sql.PreparedStatement;
import java.sql.Statement;
-import java.time.*;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.sql.Timestamp;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
-import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import java.util.Set;
-import java.util.TreeSet;
-import static java.time.temporal.ChronoUnit.DAYS;
-import static java.util.stream.Collectors.groupingBy;
-import static java.util.stream.Collectors.toList;
import static org.openremote.model.syslog.SyslogCategory.DATA;
import static org.openremote.model.util.MapAccess.getInteger;
import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;
/**
- * Store and retrieve datapoints for asset attributes and periodically purge data points based on
- * {@link MetaItemType#DATA_POINTS_MAX_AGE_DAYS} {@link org.openremote.model.attribute.MetaItem}
- * and {@link #OR_DATA_POINTS_MAX_AGE_DAYS} setting; storage duration defaults to {@value #OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT}
- * days.
+ * Store and retrieve datapoints for asset attributes and periodically purge data points using
+ * TimescaleDB's {@code drop_chunks()} with week-based retention aligned to 7-day chunk intervals.
+ * Retention defaults to inifinite.
*/
public class AssetDatapointService extends AbstractDatapointService<AssetDatapoint> implements ProtocolDatapointService {
+ public static final String OR_DATA_POINTS_MAX_AGE_WEEKS = "OR_DATA_POINTS_MAX_AGE_WEEKS";
+ /** @deprecated Use {@link #OR_DATA_POINTS_MAX_AGE_WEEKS} instead. If set, value is converted to weeks (rounded up). */
+ @Deprecated
public static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";
- public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;
public static final String OR_DATA_POINTS_EXPORT_LIMIT = "OR_DATA_POINTS_EXPORT_LIMIT";
public static final int OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT = 1000000;
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
private static final Logger DATA_EXPORT_LOG = SyslogCategory.getLogger(DATA, AssetDatapointResourceImpl.class);
-
- protected int maxDatapointAgeDays;
+ protected int maxDatapointAgeWeeks = -1;
protected int datapointExportLimit;
@Override
@@ -84,12 +73,20 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
)
);
- maxDatapointAgeDays = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_DAYS, OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT);
+ if (container.getConfig().containsKey(OR_DATA_POINTS_MAX_AGE_WEEKS)) {
+ maxDatapointAgeWeeks = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_WEEKS, -1);
+ } else if (container.getConfig().containsKey(OR_DATA_POINTS_MAX_AGE_DAYS)) {
+ int days = getInteger(container.getConfig(), OR_DATA_POINTS_MAX_AGE_DAYS, -1);
+ maxDatapointAgeWeeks = days > 0 ? (int) Math.ceil(days / 7.0) : days;
+ LOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " is deprecated, use " + OR_DATA_POINTS_MAX_AGE_WEEKS + " instead. Converted " + days + " days to " + maxDatapointAgeWeeks + " weeks.");
+ } else {
+ maxDatapointAgeWeeks = -1;
+ }
- if (maxDatapointAgeDays <= 0) {
- LOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");
+ if (maxDatapointAgeWeeks <= 0) {
+ LOG.warning("Data point purge disabled");
} else {
- LOG.log(Level.INFO, "Data point purge interval days = " + maxDatapointAgeDays);
+ LOG.log(Level.INFO, "Data point purge retention = " + maxDatapointAgeWeeks + " weeks");
}
datapointExportLimit = getInteger(container.getConfig(), OR_DATA_POINTS_EXPORT_LIMIT, OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT);
@@ -103,11 +100,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
@Override
public void start(Container container) throws Exception {
- if (maxDatapointAgeDays > 0) {
+ if (maxDatapointAgeWeeks > 0) {
dataPointsPurgeScheduledFuture = scheduledExecutorService.scheduleAtFixedRate(
this::purgeDataPoints,
getFirstPurgeMillis(timerService.getNow()),
- Duration.ofDays(1).toMillis(), TimeUnit.MILLISECONDS
+ Duration.ofDays(7).toMillis(), TimeUnit.MILLISECONDS
);
}
@@ -151,109 +148,62 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
}
protected void purgeDataPoints() {
- LOG.info("Running data points purge daily task");
+ LOG.info("Running data points purge task");
try {
- // 1. Get list of attributes that have custom durations
- List<Asset<?>> assets = assetStorageService.findAll(
- new AssetQuery()
- .attributes(
- new AttributePredicate().meta(
- new NameValuePredicate(MetaItemType.DATA_POINTS_MAX_AGE_DAYS, null)
- )));
-
- List<Pair<String, Attribute<?>>> attributes = assets.stream()
- .map(asset -> asset
- .getAttributes().stream()
- .filter(assetAttribute -> assetAttribute.hasMeta(MetaItemType.DATA_POINTS_MAX_AGE_DAYS))
- .map(assetAttribute -> new Pair<String, Attribute<?>>(asset.getId(), assetAttribute))
- .collect(toList()))
- .flatMap(List::stream)
- .toList();
-
- // 2. Perform the purge using a temporary table and native SQL
persistenceService.doTransaction(em -> {
- em.unwrap(Session.class).doWork(connection -> {
- String tempTableName = "tmp_purge_attributes";
+ // Get the schema-qualified hypertable name
+ @SuppressWarnings("unchecked")
+ List<String> hypertables = em.createNativeQuery(
+ "SELECT hypertable_schema || '.' || hypertable_name " +
+ "FROM timescaledb_information.hypertables " +
+ "WHERE hypertable_name = 'asset_datapoint'", String.class
+ ).getResultList();
+
+ if (hypertables.isEmpty()) {
+ LOG.warning("asset_datapoint is not a TimescaleDB hypertable, skipping purge. " +
+ "TimescaleDB with hypercore is required for data point storage.");
+ return;
+ }
- try (Statement statement = connection.createStatement()) {
- // Create temp table holding the asset ID, attribute name, and its custom max age
- statement.execute(
- "create temp table " + tempTableName + " (entity_id text, attribute_name text, max_age_days int) on commit drop"
- );
+ String qualifiedTableName = hypertables.getFirst();
+
+ // Find the maximum custom retention across all attributes with DATA_POINTS_MAX_AGE_DAYS meta
+ int effectiveRetentionWeeks = maxDatapointAgeWeeks;
+ Number maxCustomDays = (Number) em.createNativeQuery(
+ "SELECT MAX((attr_val->'meta'->>'dataPointsMaxAgeDays')::integer) " +
+ "FROM asset, jsonb_each(asset.attributes) AS a(attr_key, attr_val) " +
+ "WHERE jsonb_exists(attr_val->'meta', 'dataPointsMaxAgeDays')"
+ ).getSingleResult();
+
+ if (maxCustomDays != null) {
+ int customRetentionWeeks = (int) Math.ceil(maxCustomDays.intValue() / 7.0);
+ effectiveRetentionWeeks = Math.max(effectiveRetentionWeeks, customRetentionWeeks);
+ if (customRetentionWeeks > maxDatapointAgeWeeks) {
+ LOG.info("Custom attribute retention (" + maxCustomDays.intValue() + " days / " +
+ customRetentionWeeks + " weeks) exceeds system default (" + maxDatapointAgeWeeks +
+ " weeks), using " + effectiveRetentionWeeks + " weeks");
}
+ }
- // Batch insert the custom retention rules to avoid giant IN clauses
- if (!attributes.isEmpty()) {
- try (PreparedStatement insertStatement = connection.prepareStatement(
- "insert into " + tempTableName + " (entity_id, attribute_name, max_age_days) values (?, ?, ?)"
- )) {
- for (Pair<String, Attribute<?>> attributeRef : attributes) {
- insertStatement.setString(1, attributeRef.key);
- insertStatement.setString(2, attributeRef.value.getName());
-
- int customAge = attributeRef.value
- .getMetaValue(MetaItemType.DATA_POINTS_MAX_AGE_DAYS)
- .orElse(maxDatapointAgeDays);
- insertStatement.setInt(3, customAge);
-
- insertStatement.addBatch();
- }
- insertStatement.executeBatch();
- }
- }
+ // Compute cutoff from effective retention
+ Instant cutoff = timerService.getNow().minus(Duration.ofDays((long) effectiveRetentionWeeks * 7));
+ Timestamp cutoffTimestamp = Timestamp.from(cutoff);
- // Purge default data points (NOT in the temp table)
- LOG.fine("Purging data points of attributes that use default max age days of " + maxDatapointAgeDays);
- long defaultEpochMillis = timerService.getNow().truncatedTo(DAYS).minus(maxDatapointAgeDays, DAYS).toEpochMilli();
+ LOG.info("Dropping chunks older than " + cutoffTimestamp + " (" + effectiveRetentionWeeks + " weeks retention)");
- try (PreparedStatement defaultPurge = connection.prepareStatement(
- "delete from asset_datapoint ad " +
- "where ad.timestamp < to_timestamp(?) " +
- "and not exists (select 1 from " + tempTableName + " t where t.entity_id = ad.entity_id and t.attribute_name = ad.attribute_name)"
- )) {
- defaultPurge.setLong(1, defaultEpochMillis / 1000);
- defaultPurge.executeUpdate();
- }
+ Number dropped = (Number) em.createNativeQuery("SELECT count(*) FROM public.drop_chunks(CAST(:hypertable AS regclass), older_than => CAST(:cutoff AS timestamp))")
+ .setParameter("hypertable", qualifiedTableName)
+ .setParameter("cutoff", cutoffTimestamp)
+ .getSingleResult();
- // Purge custom data points by joining against the temp table
- if (!attributes.isEmpty()) {
- LOG.fine("Purging data points with custom max age constraints");
- long currentEpochMillis = timerService.getNow().truncatedTo(DAYS).toEpochMilli();
-
- try (PreparedStatement customPurge = connection.prepareStatement(
- "delete from asset_datapoint ad " +
- "using " + tempTableName + " t " +
- "where ad.entity_id = t.entity_id " +
- "and ad.attribute_name = t.attribute_name " +
- // Delete where timestamp is older than (current date - custom age)
- "and ad.timestamp < to_timestamp(? - (t.max_age_days * 86400))"
- )) {
- customPurge.setLong(1, currentEpochMillis / 1000);
- customPurge.executeUpdate();
- }
- }
- });
+ LOG.info("Successfully purged data points using drop_chunks, drop count = " + dropped);
});
-
} catch (Exception e) {
- LOG.log(Level.WARNING, "Failed to run data points purge", e);
+ LOG.log(Level.SEVERE, "Failed to run data points purge", e);
}
}
- protected String buildWhereClause(List<Pair<String, Attribute<?>>> attributes, boolean negate) {
-
- if (attributes.isEmpty()) {
- return "";
- }
-
- String whereStr = attributes.stream()
- .map(attributeRef -> "('" + attributeRef.key + "','" + attributeRef.value.getName() + "')")
- .collect(Collectors.joining(","));
-
- return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";
- }
-
/**
* Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped into the manager
* container so it can be accessed by this process.
diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sql
new file mode 100644
index 000000000..8f7dec3df
--- /dev/null
+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_01__DatapointMetricQuery.sql
@@ -0,0 +1,38 @@
+-- This is a function to ensure indexes are used for the dynamically retrieved entity and attribute.
+
+CREATE OR REPLACE FUNCTION datapoint_metric_query()
+RETURNS integer AS $$
+DECLARE
+target_entity text;
+ target_attr text;
+ result_count integer;
+BEGIN
+ -- 1. Find the heaviest entity and attribute safely
+ -- REMOVED the jsonb_typeof check to guarantee an Index Only Scan
+SELECT entity_id, attribute_name
+INTO target_entity, target_attr
+FROM asset_datapoint
+WHERE timestamp > now() - INTERVAL '1 day'
+GROUP BY entity_id, attribute_name
+ORDER BY count(*) DESC
+ LIMIT 1;
+
+-- 2. Run the heavy aggregation using dynamic SQL.
+EXECUTE format('
+ WITH aggregated_points AS (
+ SELECT
+ time_bucket(''1 hour'', timestamp) AS bucket,
+ avg((value::text)::numeric) AS avg_value
+ FROM asset_datapoint
+ WHERE entity_id = %L
+ AND attribute_name = %L
+ AND timestamp BETWEEN now() - INTERVAL ''14 days'' AND now() - INTERVAL ''7 days''
+ GROUP BY bucket
+ )
+ SELECT count(bucket) FROM aggregated_points;
+ ', target_entity, target_attr) INTO result_count;
+
+-- 3. Return the final row count to the exporter
+RETURN result_count;
+END;
+$$ LANGUAGE plpgsql;
diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sql
new file mode 100644
index 000000000..7b1bb0c88
--- /dev/null
+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260506_02__EnableHyperCore.sql
@@ -0,0 +1,42 @@
+-- FAIL FAST IF THE DB IMAGE DOES NOT SUPPORT HYPERCORE
+DO $$
+ DECLARE
+ ts_version text;
+ has_hypercore_policy_proc boolean;
+ BEGIN
+ -- 1. EXISTING VERSION CHECKS
+ SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
+
+ IF ts_version IS NULL THEN
+ RAISE EXCEPTION 'TimescaleDB extension is missing.';
+ END IF;
+
+ -- 2. SAFE REMOVAL (Sub-blocks for error handling)
+ -- This "nests" the logic within your existing block
+ BEGIN
+ -- remove_compression_policy is a FUNCTION
+ -- Use PERFORM to call a function inside PL/pgSQL
+ PERFORM public.remove_compression_policy('${schemaName}.asset_datapoint');
+ EXCEPTION WHEN OTHERS THEN
+ RAISE NOTICE 'Could not remove legacy policy: %', SQLERRM;
+ END;
+
+ BEGIN
+ -- remove_columnstore_policy is a PROCEDURE
+ -- Use EXECUTE 'CALL...' for maximum compatibility
+ EXECUTE 'CALL public.remove_columnstore_policy(''${schemaName}.asset_datapoint'', if_exists => true)';
+ EXCEPTION WHEN OTHERS THEN
+ RAISE NOTICE 'Could not remove columnstore policy: %', SQLERRM;
+ END;
+
+ END $$;
+
+-- 3. ENABLE COLUMNSTORE (Must be outside DO block if it requires its own transaction)
+ALTER TABLE ${schemaName}.asset_datapoint SET (
+ timescaledb.enable_columnstore = true,
+ timescaledb.orderby = 'timestamp DESC',
+ timescaledb.segmentby = 'entity_id,attribute_name',
+ timescaledb.chunk_interval = '7 days');
+
+-- ACTIVATE THE AUTOMATED POLICY
+CALL public.add_columnstore_policy('${schemaName}.asset_datapoint', after => INTERVAL '7 days');
diff --git a/model/src/main/java/org/openremote/model/value/MetaItemType.java b/model/src/main/java/org/openremote/model/value/MetaItemType.java
index a7ab01295..777a5ea40 100644
--- a/model/src/main/java/org/openremote/model/value/MetaItemType.java
+++ b/model/src/main/java/org/openremote/model/value/MetaItemType.java
@@ -92,8 +92,12 @@ public final class MetaItemType {
public static final MetaItemDescriptor<Boolean> STORE_DATA_POINTS = new MetaItemDescriptor<>("storeDataPoints", ValueType.BOOLEAN);
/**
- * How long to store attribute values in time series database (data older than this will be automatically purged)
+ * How long to store attribute values in time series database (data older than this will be automatically purged).
+ * @deprecated Per-attribute retention is no longer supported. Purging now uses TimescaleDB drop_chunks with
+ * a global week-based retention period configured via OR_DATA_POINTS_MAX_AGE_WEEKS. This meta item
+ * is retained for backward compatibility but has no effect.
*/
+ @Deprecated
public static final MetaItemDescriptor<Integer> DATA_POINTS_MAX_AGE_DAYS = new MetaItemDescriptor<>("dataPointsMaxAgeDays", ValueType.POSITIVE_INTEGER);
/**
diff --git a/profile/deploy.yml b/profile/deploy.yml
index 0a1dd7aa5..e12b8f0b4 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -486,7 +486,7 @@ services:
- deployment-data:/deployment
ports:
- "127.0.0.1:8406:9560" # Localhost metrics access
- - "${PRIVATE_IP:-127.0.0.1}:8406:8404" # Allows to also expose metrics on a given IP address,
+ - "${PRIVATE_IP:-127.0.0.1}:8406:9560" # Allows to also expose metrics on a given IP address,
# intended to be IP of the interface of the private subnet of the EC2 VM
environment:
TZ: ${TZ:-Europe/Amsterdam}
diff --git a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
index 85cadb4b3..b5d87839c 100644
--- a/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
+++ b/setup/src/integration/java/org/openremote/setup/integration/ManagerTestSetup.java
@@ -42,7 +42,6 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT;
import static org.openremote.model.Constants.*;
import static org.openremote.model.value.MetaItemType.*;
import static org.openremote.model.value.ValueType.*;
@@ -165,10 +164,6 @@ public class ManagerTestSetup extends ManagerSetup {
new MetaItem<>(
STORE_DATA_POINTS,
true),
- new MetaItem<>(
- DATA_POINTS_MAX_AGE_DAYS,
- OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT *7
- ),
new MetaItem<>(
AGENT_LINK,
new SimulatorAgentLink(agent.getId()))
diff --git a/test/load3/README.md b/test/load3/README.md
new file mode 100644
index 000000000..37a307ba3
--- /dev/null
+++ b/test/load3/README.md
@@ -0,0 +1,45 @@
+# TimescaleDB Benchmark
+
+This script is an automated load-generator designed to benchmark query performance on a TimescaleDB instance; it can be used to compare performance before and after a DB configuration change.
+
+## What it does
+Instead of testing generic, hardcoded queries, this script dynamically adapts to your actual data:
+1. **Dynamic Target Acquisition:** It queries the `asset_datapoint` table to find the 10 entities and attributes with the highest volume of numeric data over the last 3 days.
+2. **Heavy Load Generation:** For each of those 10 targets, it executes three realistic, heavy time-series queries:
+ * A 7-day hourly average.
+ * A 14-day Largest Triangle Three Buckets (LTTB) visual downsample (500 points).
+ * A 30-day daily Min/Max/Avg calculation.
+3. **Pure Execution Timing:** It wraps every query in an `EXPLAIN ANALYZE` command to extract the exact millisecond execution time directly from the PostgreSQL engine, completely eliminating Docker or network latency from the benchmark.
+4. **Excel-Ready Output:** It formats the results into a self-describing CSV file (`results.csv`), making it trivial to generate charts in Excel or Grafana.
+
+## Prerequisites
+* The script must be run on the host machine running the PostgreSQL Docker container.
+* The TimescaleDB instance must have the `timescaledb_toolkit` extension installed (required for the `lttb()` downsampling function).
+* The default Docker container name is expected to be `or-postgresql-1`. (This can be changed at the top of the script).
+* The DB must have the `datapoint_metric_query` function as added in this FlywayDB migration script (`V20260506_01__DatapointMetricQuery.sql`)
+
+## How to Run
+
+1. Make the script executable:
+ `chmod +x benchmark.sh`
+
+2. Execute the script:
+ `./benchmark.sh`
+
+## Customization
+You can easily adjust the behavior of the script by editing the configuration block at the top of `benchmark.sh`:
+* `ITERATIONS`: Controls how many times the full suite of queries runs (Default is `5`).
+* `CONTAINER_NAME`: Update this if your OpenRemote database container has a different name.
+
+## Understanding the Output (`results.csv`)
+The script generates a CSV file where the first column is the exact ISO8601 timestamp of when the iteration began.
+
+The subsequent columns contain the query execution times in milliseconds. The headers are dynamically generated using a combination of the Entity ID (first 4 characters), Attribute Name, and Query Type.
+
+**Example Header:** `4hOF-kwMax_Hourly_Avg_7d_ms`
+* `4hOF`: First 4 characters of the `entity_id`
+* `kwMax`: The `attribute_name`
+* `Hourly_Avg_7d`: The query template executed
+* `ms`: Value is measured in milliseconds
+
+To visualize your benchmark, simply open `results.csv` in Excel, select all data, and insert a Line Chart. The headers will automatically generate a perfectly formatted legend.
diff --git a/test/load3/benchmark.sh b/test/load3/benchmark.sh
new file mode 100644
index 000000000..b192c52ae
--- /dev/null
+++ b/test/load3/benchmark.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# Configuration
+CONTAINER_NAME="or-postgresql-1"
+DB_USER="postgres"
+DB_NAME="openremote"
+CSV_FILE="results.csv"
+ITERATIONS=5
+
+function run_psql() {
+ docker exec "$CONTAINER_NAME" psql -U "$DB_USER" -d "$DB_NAME" -t -A -c "$1"
+}
+
+echo "Starting Benchmark Initialization..."
+
+# 1. Find the top 10 entity_id & attribute_name values
+TOP_ENTITIES_QUERY="
+SELECT entity_id, attribute_name
+FROM asset_datapoint
+WHERE timestamp > now() - INTERVAL '3 days'
+ AND jsonb_typeof(value) = 'number'
+GROUP BY entity_id, attribute_name
+ORDER BY count(*) DESC
+LIMIT 10;
+"
+
+echo "Fetching top 10 entities..."
+ENTITIES=$(run_psql "$TOP_ENTITIES_QUERY")
+
+if [ -z "$ENTITIES" ]; then
+ echo "Error: No numeric entities found in the last 3 days. Exiting."
+ exit 1
+fi
+
+# 2. Define the Query Templates
+QUERY_TEMPLATES=(
+ "SELECT time_bucket('1 hour', timestamp) as bucket, avg((value::text)::numeric) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '7 days' GROUP BY bucket;"
+ "SELECT time, value FROM unnest((SELECT lttb(timestamp, (value::text)::numeric, 500) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '14 days'));"
+ "SELECT time_bucket('1 day', timestamp) as bucket, min((value::text)::numeric), max((value::text)::numeric), avg((value::text)::numeric) FROM asset_datapoint WHERE entity_id = '%s' AND attribute_name = '%s' AND timestamp > now() - INTERVAL '30 days' GROUP BY bucket;"
+)
+QUERY_NAMES=("Hourly_Avg_7d" "LTTB_500_14d" "Daily_MinMaxAvg_30d")
+
+# 3. Build the descriptive CSV headers
+echo "Generating headers..."
+CSV_HEADER="Timestamp"
+
+declare -a ENTITY_IDS
+declare -a ATTRIBUTE_NAMES
+
+while IFS='|' read -r entity_id attribute_name; do
+ ENTITY_IDS+=("$entity_id")
+ ATTRIBUTE_NAMES+=("$attribute_name")
+done <<< "$ENTITIES"
+
+# Create descriptive column headers
+for i in "${!ENTITY_IDS[@]}"; do
+ E_ID="${ENTITY_IDS[$i]}"
+ A_NAME="${ATTRIBUTE_NAMES[$i]}"
+
+ # SAFER PARSING: Grab first 4 chars of ID using 'cut' instead of bash string slicing
+ SHORT_ID="$(echo "$E_ID" | cut -c 1-4)-${A_NAME}"
+
+ for q in "${!QUERY_NAMES[@]}"; do
+ Q_NAME="${QUERY_NAMES[$q]}"
+ CSV_HEADER="${CSV_HEADER},${SHORT_ID}_${Q_NAME}_ms"
+ done
+done
+
+# Initialize the CSV file
+echo "$CSV_HEADER" > "$CSV_FILE"
+echo "Headers created successfully."
+
+# 4. Execute the load loop
+echo "Starting load generation ($ITERATIONS iterations)..."
+
+for (( iter=1; iter<=ITERATIONS; iter++ )); do
+ START_TIME=$(date -Iseconds)
+ CSV_ROW="$START_TIME"
+
+ echo "Iteration $iter / $ITERATIONS starting at $START_TIME"
+
+ for i in "${!ENTITY_IDS[@]}"; do
+ E_ID="${ENTITY_IDS[$i]}"
+ A_NAME="${ATTRIBUTE_NAMES[$i]}"
+
+ for q in "${!QUERY_TEMPLATES[@]}"; do
+ TEMPLATE="${QUERY_TEMPLATES[$q]}"
+
+ RAW_QUERY=$(printf "$TEMPLATE" "$E_ID" "$A_NAME")
+ ANALYZE_QUERY="EXPLAIN ANALYZE $RAW_QUERY"
+
+ # SAFER PARSING: Execute and extract execution time.
+ # Replaced 'awk' with 'tr' and 'cut' to avoid curly braces inside the subshell!
+ OUT=$(run_psql "$ANALYZE_QUERY" 2>&1)
+ EXEC_TIME=$(echo "$OUT" | grep "Execution Time" | tr -s ' ' | cut -d' ' -f3)
+
+ if [ -z "$EXEC_TIME" ]; then
+ EXEC_TIME="ERROR"
+ echo " [!] Error running $A_NAME (Iter $iter)"
+ fi
+
+ CSV_ROW="${CSV_ROW},${EXEC_TIME}"
+ done
+ done
+
+ # Write the row to the CSV
+ echo "$CSV_ROW" >> "$CSV_FILE"
+ echo " Completed iteration $iter."
+done
+
+echo "Benchmark complete! Results saved to $CSV_FILE"
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy
new file mode 100644
index 000000000..1e3ae5a23
--- /dev/null
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy
@@ -0,0 +1,246 @@
+package org.openremote.test.assets
+
+import org.hibernate.cfg.AvailableSettings
+import org.openremote.container.persistence.PersistenceService
+import org.openremote.manager.asset.AssetStorageService
+import org.openremote.manager.datapoint.AssetDatapointService
+import org.openremote.manager.datapoint.AssetPredictedDatapointService
+import org.openremote.manager.setup.SetupService
+import org.openremote.model.asset.impl.ThingAsset
+import org.openremote.model.attribute.Attribute
+import org.openremote.model.attribute.AttributeRef
+import org.openremote.model.datapoint.ValueDatapoint
+import org.openremote.model.value.ValueType
+import org.openremote.setup.integration.KeycloakTestSetup
+import org.openremote.test.ManagerContainerTrait
+import spock.lang.Specification
+import spock.util.concurrent.PollingConditions
+
+import java.time.Instant
+import java.time.temporal.ChronoUnit
+import java.util.concurrent.TimeUnit
+
+import static java.util.concurrent.TimeUnit.DAYS
+import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_WEEKS
+
+class AssetDatapointPurgeTest extends Specification implements ManagerContainerTrait {
+
+ def "Test TimescaleDB hypercore compression and purge"() {
+
+ given: "expected conditions"
+ def conditions = new PollingConditions(timeout: 30, delay: 1)
+
+ and: "the container is started with a 4-week retention period"
+ def config = defaultConfig()
+ config.put(OR_DATA_POINTS_MAX_AGE_WEEKS, "4")
+ def container = startContainer(config, defaultServices())
+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
+ def assetDatapointService = container.getService(AssetDatapointService.class)
+ def persistenceService = container.getService(PersistenceService.class)
+
+ and: "the schema name is retrieved"
+ def schemaName = persistenceService.persistenceUnitProperties.getProperty(AvailableSettings.DEFAULT_SCHEMA)
+ getLOG().info("Using schema: ${schemaName}")
+
+ and: "the clock is stopped and advanced to a known time"
+ stopPseudoClock()
+ advancePseudoClock(Instant.ofEpochMilli(getClockTimeOf(container)).truncatedTo(ChronoUnit.HOURS).plus(1, ChronoUnit.HOURS).toEpochMilli() - getClockTimeOf(container), TimeUnit.MILLISECONDS, container)
+
+ when: "a test asset is created with multiple attributes"
+ def testAsset = new ThingAsset("Hypercore Test Asset")
+ .setRealm(keycloakTestSetup.realmMaster.name)
+ testAsset.addOrReplaceAttributes(
+ new Attribute<>("temperature", ValueType.NUMBER),
+ new Attribute<>("humidity", ValueType.NUMBER),
+ new Attribute<>("pressure", ValueType.NUMBER),
+ new Attribute<>("co2", ValueType.NUMBER)
+ )
+ testAsset = assetStorageService.merge(testAsset)
+
+ then: "the asset should be created"
+ testAsset.id != null
+ getLOG().info("Created test asset with ID: ${testAsset.id}")
+
+ when: "25.000 datapoints are inserted across multiple attributes with some within the 4-week retention window"
+ def totalDatapoints = 25_000
+ def attributeNames = ["temperature", "humidity", "pressure", "co2"]
+ def datapointsPerAttribute = (totalDatapoints / attributeNames.size()) as int
+ def recentDatapointsPerAttribute = (datapointsPerAttribute * 0.2) as int
+ def oldDatapointsPerAttribute = datapointsPerAttribute - recentDatapointsPerAttribute
+ def expectedRecentDatapoints = recentDatapointsPerAttribute * attributeNames.size()
+ def currentTime = getClockTimeOf(container)
+ def oldBaseTimestamp = Instant.ofEpochMilli(currentTime).minus(365, ChronoUnit.DAYS)
+ def recentBaseTimestamp = Instant.ofEpochMilli(currentTime).minus(7, ChronoUnit.DAYS)
+
+ getLOG().info("Starting to insert ${totalDatapoints} datapoints...")
+ def insertStartTime = System.currentTimeMillis()
+
+ // Insert datapoints in batches for better performance
+ def batchSize = 10000
+ attributeNames.each { attributeName ->
+ getLOG().info("Inserting ${datapointsPerAttribute} datapoints for attribute: ${attributeName}")
+
+ for (int batchStart = 0; batchStart < datapointsPerAttribute; batchStart += batchSize) {
+ def batchEnd = Math.min(batchStart + batchSize, datapointsPerAttribute)
+ def datapoints = []
+ for (int index = batchStart; index < batchEnd; index++) {
+ def timestamp = index < oldDatapointsPerAttribute
+ ? oldBaseTimestamp.plus(index * 30, ChronoUnit.SECONDS).toEpochMilli()
+ : recentBaseTimestamp.plus((index - oldDatapointsPerAttribute) * 30, ChronoUnit.SECONDS).toEpochMilli()
+ def value = 20.0 + (Math.sin(index / 100.0) * 10.0) + (Math.random() * 2.0)
+ datapoints.add(new ValueDatapoint<>(timestamp, value))
+ }
+ assetDatapointService.upsertValues(testAsset.id, attributeName, datapoints)
+
+ if (batchEnd % (batchSize * 10) == 0 || batchEnd == datapointsPerAttribute) {
+ getLOG().info(" Progress: ${batchEnd} / ${datapointsPerAttribute} datapoints inserted for ${attributeName}")
+ }
+ }
+ }
+
+ def insertEndTime = System.currentTimeMillis()
+ def insertDuration = (insertEndTime - insertStartTime) / 1000.0
+ getLOG().info("Finished inserting ${totalDatapoints} datapoints in ${insertDuration} seconds")
+
+ then: "datapoints should be stored"
+ conditions.eventually {
+ def count = 0
+ attributeNames.each { attributeName ->
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))
+ count += datapoints.size()
+ }
+ assert count >= totalDatapoints * 0.99 // Allow for 1% margin
+ getLOG().info("Verified ${count} datapoints stored")
+ }
+
+ when: "storage usage is measured before enabling hypercore"
+ def orDatabaseSizeBefore = persistenceService.doReturningTransaction { em ->
+ def query = em.createNativeQuery("""
+ SELECT pg_database_size(pg_database.datname)
+ FROM pg_database
+ WHERE pg_database.datname = 'openremote';
+ """)
+ return query.getSingleResult() as Long
+ }
+
+ getLOG().info("\n=== Storage BEFORE Hypercore ===")
+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeBefore / (1024.0 * 1024.0))}")
+
+ and: "compression job is called manually"
+ def job_id = persistenceService.doReturningTransaction { em ->
+ def query = em.createNativeQuery("""
+ SELECT job_id
+ FROM timescaledb_information.jobs
+ WHERE proc_name = 'policy_compression' AND hypertable_name = 'asset_datapoint';
+ """)
+ return query.getSingleResult()
+ }
+
+ // run_job contains internal COMMIT statements, so it must be executed outside of a Hibernate transaction
+ def dataSource = persistenceService.persistenceUnitProperties.get(AvailableSettings.DATASOURCE) as javax.sql.DataSource
+ def connection = dataSource.getConnection()
+ try {
+ connection.setAutoCommit(true)
+ def stmt = connection.prepareCall("CALL public.run_job(?)")
+ stmt.setInt(1, job_id as int)
+ stmt.execute()
+ stmt.close()
+ } finally {
+ connection.close()
+ }
+
+ def orDatabaseSizeAfter = persistenceService.doReturningTransaction { em ->
+ def query = em.createNativeQuery("""
+ SELECT pg_database_size(pg_database.datname)
+ FROM pg_database
+ WHERE pg_database.datname = 'openremote';
+ """)
+ return query.getSingleResult() as Long
+ }
+
+ getLOG().info("\n=== Storage AFTER Hypercore ===")
+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeAfter / (1024.0 * 1024.0))}")
+
+ then: "storage should be smaller after hypercore"
+ orDatabaseSizeBefore > orDatabaseSizeAfter
+
+ when: "Purging will be called, count datapoints before purging"
+ def countBeforePurge = 0
+ attributeNames.each { attributeName ->
+ def dataPoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))
+ countBeforePurge += dataPoints.size()
+ }
+ getLOG().info("Datapoints before purge: ${countBeforePurge}")
+
+ and: "the purge routine is executed"
+ def deleteStartTime = System.currentTimeMillis()
+ assetDatapointService.purgeDataPoints()
+ def deleteEndTime = System.currentTimeMillis()
+ def deleteDuration = (deleteEndTime - deleteStartTime) / 1000.0
+
+ getLOG().info("Purge completed in ${deleteDuration} seconds")
+
+ then: "data points beyond the 4-week retention window should be purged via drop_chunks"
+ conditions.eventually {
+ def countAfterPurge = 0
+ attributeNames.each { attributeName ->
+ def dataPoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))
+ countAfterPurge += dataPoints.size()
+ }
+ def deletedCount = countBeforePurge - countAfterPurge
+
+ getLOG().info("Datapoints after purge: ${countAfterPurge}")
+ getLOG().info("Deleted ${deletedCount} datapoints")
+ getLOG().info("Deletion rate: ${String.format('%.0f', deletedCount / deleteDuration)} datapoints/second")
+
+ // With 365 days of data and 4-week (28-day) retention, most data should be purged
+ assert countAfterPurge < countBeforePurge
+ assert countAfterPurge >= expectedRecentDatapoints
+ assert deletedCount > 0
+ }
+
+ when: "storage usage is measured after deletion"
+ def orDatabaseSizeAfterDeletion = persistenceService.doReturningTransaction { em ->
+ def query = em.createNativeQuery("""
+ SELECT pg_database_size(pg_database.datname)
+ FROM pg_database
+ WHERE pg_database.datname = 'openremote';
+ """)
+ return query.getSingleResult() as Long
+ }
+
+ getLOG().info("\n=== Storage AFTER Deletion ===")
+ getLOG().info("\nDatabase size: ${String.format('%.2f MB', orDatabaseSizeAfterDeletion / (1024.0 * 1024.0))}")
+
+ then: "final storage should be measured"
+ true
+
+ when: "final verification of remaining datapoints"
+ def finalCount = 0
+ attributeNames.each { attributeName ->
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(testAsset.id, attributeName))
+ finalCount += datapoints.size()
+ }
+ def actualDeleted = countBeforePurge - finalCount
+ def compressionRatio = ((orDatabaseSizeBefore - orDatabaseSizeAfter) / (double) orDatabaseSizeBefore) * 100
+
+ getLOG().info("\n=== Final Summary ===")
+ getLOG().info("Total datapoints inserted: ${totalDatapoints}")
+ getLOG().info("Datapoints before purge: ${countBeforePurge}")
+ getLOG().info("Datapoints deleted by purge: ${actualDeleted}")
+ getLOG().info("Datapoints remaining: ${finalCount}")
+ getLOG().info("Deletion percentage: ${String.format('%.2f', (actualDeleted / totalDatapoints * 100.0))}%")
+ getLOG().info("\nStorage Summary:")
+ getLOG().info(" Before hypercore: ${String.format('%.2f MB', orDatabaseSizeBefore / (1024.0 * 1024.0))}")
+ getLOG().info(" After hypercore: ${String.format('%.2f MB', orDatabaseSizeAfter / (1024.0 * 1024.0))}")
+ getLOG().info(" After deletion: ${String.format('%.2f MB', orDatabaseSizeAfterDeletion / (1024.0 * 1024.0))}")
+ getLOG().info("\nPerformance Summary:")
+ getLOG().info(" Insert time: ${insertDuration} seconds")
+ getLOG().info(" Purge time: ${deleteDuration} seconds")
+ getLOG().info(" Compression: ${String.format('%.2f', compressionRatio)}%")
+
+ then: "No exception should have occurred"
+ true
+ }
+}
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
index 279e06c32..b740dae59 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointTest.groovy
@@ -44,7 +44,6 @@ import java.util.concurrent.TimeUnit
import java.util.function.Consumer
import static java.util.concurrent.TimeUnit.*
-import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT
import static org.openremote.model.Constants.KEYCLOAK_CLIENT_ID
import static org.openremote.model.Constants.MASTER_REALM
import static org.openremote.setup.integration.ManagerTestSetup.thingLightToggleAttributeName
@@ -109,7 +108,6 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
def conditions = new PollingConditions(timeout: 10, delay: 0.2)
when: "the demo agent and thing have been deployed"
- def datapointPurgeDays = OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT
def container = startContainer(defaultConfig(), defaultServices())
def managerTestSetup = container.getService(SetupService.class).getTaskOfType(ManagerTestSetup.class)
def agentService = container.getService(AgentService.class)
@@ -353,107 +351,6 @@ class AssetDatapointTest extends Specification implements ManagerContainerTrait
assert thing.getAttribute(thingLightToggleAttributeName).flatMap {it.getTimestamp()}.orElse(0) == getClockTimeOf(container)
}
- // ------------------------------------
- // Test purging of data points
- // ------------------------------------
-
- when: "time moves forward by more than purge days"
- advancePseudoClock(datapointPurgeDays + 1, DAYS, container)
-
- and: "the power sensor with default max age receives a new value"
- def datapoint4ExpectedTimestamp = getClockTimeOf(container)
- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"), 17.5d)
-
- and: "the toggle sensor with a custom max age of 7 days receives a new value"
- simulatorProtocol.updateSensor(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName), true)
-
- then: "the datapoints should be stored"
- conditions.eventually {
- //def powerDatapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List
- //def toggleDatapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List
- def powerDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))
- def toggleDatapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))
-
- assert powerDatapoints.size() >= 6
- assert powerDatapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value == 17.5 }
-
- assert toggleDatapoints.size() >= 4
- assert toggleDatapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}
- }
-
- when: "the clock advances to the next days purge routine execution time"
- advancePseudoClock(assetDatapointService.getFirstPurgeMillis(Instant.ofEpochMilli(getClockTimeOf(container))), TimeUnit.MILLISECONDS, container)
-
- and: "the purge routine runs"
- assetDatapointService.purgeDataPoints()
-
- then: "data points older than purge days should be purged for the power sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))
- assert datapoints.size() == 1
- assert ValueUtil.getValue(datapoints.get(0).value, Double.class).orElse(null) == 17.5d
- assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
- }
-
- and: "no data points should have been purged for the toggle sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))
- assert datapoints.size() >= 4
- assert ValueUtil.getBoolean(datapoints.get(0).value).orElse(false)
- assert datapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}
- }
-
- when: "the clock advances 3 times the purge duration"
- advancePseudoClock(3 * datapointPurgeDays, DAYS, container)
-
- and: "the purge routine runs"
- assetDatapointService.purgeDataPoints()
-
- then: "all data points should be purged for power sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, "light1PowerConsumption", null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, "light1PowerConsumption"))
- assert datapoints.isEmpty()
- }
-
- and: "no data points should have been purged for the toggle sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))
- assert datapoints.size() >= 4
- assert datapoints.any {it.timestamp == datapoint4ExpectedTimestamp && it.value}
- }
-
- when: "the clock advances 3 times the purge duration"
- advancePseudoClock(3 * datapointPurgeDays, DAYS, container)
-
- and: "the purge routine runs"
- assetDatapointService.purgeDataPoints()
-
- then: "data points older than 7 days should have been purged for the toggle sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))
- assert datapoints.size() == 1
- assert ValueUtil.getBoolean(datapoints.get(0).value).orElse(false)
- assert datapoints.get(0).timestamp == datapoint4ExpectedTimestamp
- }
-
- when: "the clock advances by the purge duration"
- advancePseudoClock(datapointPurgeDays, DAYS, container)
-
- and: "the purge routine runs"
- assetDatapointService.purgeDataPoints()
-
- then: "all data points should have been purged for the toggle sensor"
- conditions.eventually {
- //def datapoints = datapointResource.getDatapoints(null, managerTestSetup.thingId, thingLightToggleAttributeName, null) as List
- def datapoints = assetDatapointService.getDatapoints(new AttributeRef(managerTestSetup.thingId, thingLightToggleAttributeName))
- assert datapoints.isEmpty()
- }
-
when: "we subscribe to predicted data points events"
List<AssetPredictedDatapointEvent> predictedEvents = new CopyOnWriteArrayList<>()
Consumer<AssetPredictedDatapointEvent> predictedEventConsumer = { event ->