100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix JSON-null datapoint queries (#3447)
Eric Bariauxcffcdddd4f87
diff --git a/container/src/main/java/org/openremote/container/timer/TimerService.java b/container/src/main/java/org/openremote/container/timer/TimerService.javaindex f84840baa..561b3e847 100644--- a/container/src/main/java/org/openremote/container/timer/TimerService.java+++ b/container/src/main/java/org/openremote/container/timer/TimerService.java@@ -22,7 +22,6 @@ import static org.openremote.model.util.MapAccess.getString;import java.time.*;import java.time.temporal.Temporal;-import java.util.TimeZone;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicLong;import java.util.logging.Logger;@@ -83,13 +82,11 @@ public class TimerService implements ContainerService, RulesClock {PSEUDO {protected AtomicLong offset = new AtomicLong();protected volatile Long stopTime;- // Explicitly define a UTC Clock+ // Use UTC internally without changing the JVM's default time zone.protected java.time.Clock utcClock = java.time.Clock.systemUTC();@Overridepublic void init() {- // Force JVM timezone to UTC for legacy date conversions to avoid DST issues in tests- TimeZone.setDefault(TimeZone.getTimeZone("UTC"));long current = getCurrentTimeMillis();LOG.info("Initialized pseudo clock to: " + current + " / " + Instant.ofEpochMilli(current));}diff --git a/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy b/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovyindex 1bc52a805..0d60e599a 100644--- a/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy+++ b/container/src/test/groovy/org/openremote/container/timer/PseudoClockTest.groovy@@ -23,17 +23,62 @@ import spock.lang.Specificationimport java.time.LocalDateimport java.time.LocalTimeimport java.time.ZoneId+import java.util.concurrent.TimeUnitclass PseudoClockTest extends Specification {TimerService.Clock clock+ TimeZone originalTimeZonedef setup() {+ originalTimeZone = TimeZone.getDefault()clock = TimerService.Clock.PSEUDO+ clock.reset()clock.init()clock.stop()}+ def cleanup() {+ try {+ clock.reset()+ } finally {+ TimeZone.setDefault(originalTimeZone)+ }+ }++ def "initializing the pseudo clock preserves the JVM time zone #zone"() {+ given:+ def timeZone = TimeZone.getTimeZone(zone)+ TimeZone.setDefault(timeZone)++ when:+ clock.init()++ then:+ TimeZone.getDefault() == timeZone++ where:+ zone << ["Europe/Brussels", "America/New_York", "Asia/Kathmandu"]+ }++ def "setTime respects the #hours hour day on #date in Europe/Brussels"() {+ given:+ def zone = ZoneId.of("Europe/Brussels")+ def startDate = LocalDate.parse(date)+ def start = clock.setTime(startDate, LocalTime.MIDNIGHT, zone)++ when:+ def end = clock.setTime(startDate.plusDays(1), LocalTime.MIDNIGHT, zone)++ then:+ end - start == TimeUnit.HOURS.toMillis(hours)++ where:+ date | hours+ "2026-03-29" | 23+ "2026-10-25" | 25+ }+def "setTime sets #date at midnight in #zone to #expected milliseconds"() {expect:clock.setTime(LocalDate.parse(date), LocalTime.MIDNIGHT, ZoneId.of(zone)) == expecteddiff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.javaindex 26943b55b..d83cb080f 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java@@ -29,7 +29,9 @@ import org.openremote.model.attribute.AttributeRef;description ="Returns every stored datapoint in the inclusive range, ordered from newest to oldest. "+ "Use this when the complete series is needed without aggregation or downsampling. Supports numeric, "- + "boolean, text, and JSON-compatible complex attribute values.")+ + "boolean, text, and JSON-compatible complex attribute values. "+ + "Unavailable values retain their stored timestamps and are returned as null. "+ + "Boolean values are returned as 1 for true, 0 for false, and null for unavailable values.")public class AssetDatapointAllQuery extends AssetDatapointQuery {public AssetDatapointAllQuery() {}@@ -50,11 +52,15 @@ public class AssetDatapointAllQuery extends AssetDatapointQuery {String conditionSuffix =" where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";if (isNumber) {- return "select timestamp as X, cast(value as numeric) as Y from "+ return "select timestamp as X, cast("+ + NULLABLE_VALUE_SQL+ + " as numeric) as Y from "+ tableName+ conditionSuffix;} else if (isBoolean) {- return "select timestamp as X, (case when cast(cast(value as text) as boolean) is true then 1 else 0 end) as Y from "+ return "select timestamp as X, ("+ + BOOLEAN_AS_NUMBER_SQL+ + ") as Y from "+ tableName+ conditionSuffix;} else {diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.javaindex de10cb4ee..e792c4c08 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java@@ -47,7 +47,8 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {description ="When true, returns empty buckets across the requested range using time-bucket gap "+ "filling; their aggregate is generally null. When false, only buckets containing datapoints are "- + "returned. Defaults to false.",+ + "returned. MODE currently returns only buckets containing datapoints regardless of this flag. "+ + "Defaults to false.",example = "true")public boolean gapFill;@@ -55,9 +56,17 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {description ="Aggregation applied per bucket: `MIN`, `AVG`, and `MAX` calculate the corresponding "+ "value; `DIFFERENCE` calculates the change between the last values of consecutive buckets; "- + "`COUNT` counts datapoints; `SUM` totals numeric values (or true values for booleans); `MODE` "+ + "`COUNT` counts stored datapoints, including null-valued rows; `SUM` totals numeric values "+ + "(or true values for booleans); `MODE` "+ "selects the most frequent value; and `MEDIAN` selects the middle value. `DIFFERENCE` is numeric "- + "only.",+ + "only. For numeric and boolean attributes, MIN, AVG, MAX, SUM, MODE and MEDIAN ignore null values and return null "+ + "for buckets with no available values. Empty gap-filled buckets return null for COUNT. "+ + "MODE ties select the smaller numeric value or true for booleans. Boolean MEDIAN returns 1 "+ + "when the median of available 1/0 values is at least 0.5, otherwise 0; it returns null when unavailable. "+ + "DIFFERENCE uses the last stored value per bucket. With gap filling enabled, empty buckets carry "+ + "the previous bucket value, while explicit null values interrupt carry-forward. A difference is "+ + "null if either endpoint is unavailable. The first returned bucket has a difference of zero "+ + "if its value is available; observations outside the query range are not used.",requiredMode = Schema.RequiredMode.REQUIRED)public Formula formula;@@ -117,12 +126,14 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {return "WITH interval_data AS ("+ "SELECT "+ function- + "(cast(? as interval), timestamp) AS x, public.locf(public.last(value::DOUBLE PRECISION, timestamp)) AS numeric_value "+ + "(cast(? as interval), timestamp) AS x, public.locf(public.last("+ + NULLABLE_VALUE_SQL+ + "::DOUBLE PRECISION, timestamp)) AS numeric_value "+ "FROM "+ tableName+ " "+ "WHERE ENTITY_ID = ? AND ATTRIBUTE_NAME = ? AND TIMESTAMP >= ? AND TIMESTAMP <= ? GROUP BY x )"- + "SELECT x, COALESCE(numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x), numeric_value) AS delta FROM interval_data ORDER BY x ASC";+ + "SELECT x, numeric_value - LAG(numeric_value, 1, numeric_value) OVER (ORDER BY x) AS delta FROM interval_data ORDER BY x ASC";case COUNT:return "SELECT "+ function@@ -137,7 +148,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {return "SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + "SUM(cast(value as numeric)) AS total_sum "+ + "SUM(cast("+ + NULLABLE_VALUE_SQL+ + " as numeric)) AS total_sum "+ "FROM "+ tableName+ " "@@ -148,7 +161,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {+ " SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + " cast(value as numeric) AS num_value "+ + " cast("+ + NULLABLE_VALUE_SQL+ + " as numeric) AS num_value "+ " FROM "+ tableName+ " "@@ -158,7 +173,7 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {+ " SELECT num_value FROM ( "+ " SELECT num_value, COUNT(*) AS freq "+ " FROM bucketed b2 "- + " WHERE b2.x = d.x "+ + " WHERE b2.x = d.x AND num_value IS NOT NULL "+ " GROUP BY num_value "+ " ORDER BY freq DESC, num_value "+ " LIMIT 1 "@@ -170,7 +185,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {return "SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast(value as numeric)) as median_value "+ + "PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY cast("+ + NULLABLE_VALUE_SQL+ + " as numeric)) as median_value "+ "FROM "+ tableName+ " "@@ -181,7 +198,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {+ function+ "(cast(? as interval), timestamp) AS x, "+ this.formula.toString().toLowerCase()- + "(cast(value as numeric)) FROM "+ + "(cast("+ + NULLABLE_VALUE_SQL+ + " as numeric)) FROM "+ tableName+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";}@@ -206,7 +225,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {return "SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + "SUM(CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END) AS true_count_sum "+ + "SUM("+ + BOOLEAN_AS_NUMBER_SQL+ + ") AS true_count_sum "+ "FROM "+ tableName+ " "@@ -217,17 +238,19 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {+ " SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + " cast(value as text)::boolean AS bool_value "+ + " cast("+ + NULLABLE_VALUE_SQL+ + " as text)::boolean AS bool_value "+ " FROM "+ tableName+ " "+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? "+ ") "+ "SELECT d.x, ( "- + " SELECT bool_value FROM ( "+ + " SELECT bool_value::integer FROM ( "+ " SELECT bool_value, COUNT(*) AS freq "+ " FROM bucketed b2 "- + " WHERE b2.x = d.x "+ + " WHERE b2.x = d.x AND bool_value IS NOT NULL "+ " GROUP BY bool_value "+ " ORDER BY freq DESC, bool_value DESC "+ " LIMIT 1 "@@ -239,8 +262,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {return "SELECT "+ function+ "(cast(? as interval), timestamp) AS x, "- + "CASE WHEN PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY (CASE WHEN cast(value as text)::boolean THEN 1 ELSE 0 END)) >= 0.5 "- + "THEN true ELSE false END AS median_value "+ + "(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY ("+ + BOOLEAN_AS_NUMBER_SQL+ + ")) >= 0.5)::integer AS median_value "+ "FROM "+ tableName+ " "@@ -251,7 +275,9 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {+ function+ "(cast(? as interval), timestamp) AS x, "+ this.formula.toString().toLowerCase()- + "(case when cast(cast(value as text) as boolean) is true then 1 else 0 end) FROM "+ + "("+ + BOOLEAN_AS_NUMBER_SQL+ + ") FROM "+ tableName+ " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x ORDER by x ASC";}diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.javaindex 265a1767a..ad3bfc2aa 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java@@ -30,13 +30,15 @@ import org.openremote.model.attribute.AttributeRef;"Downsamples numeric or boolean datapoints from the inclusive range with the "+ "Largest-Triangle-Three-Buckets (LTTB) algorithm while preserving the series' visual shape. Use this "+ "to reduce a large series for charts. Results are chronological; boolean values are processed as "- + "1 for true and 0 for false. Other attribute value types are rejected.")+ + "1 for true and 0 for false. Null-valued observations are omitted before downsampling, so their "+ + "timestamps are not returned as gap markers. A range with no available values returns no points. "+ + "Other attribute value types are rejected.")public final class AssetDatapointLTTBQuery extends AssetDatapointQuery {@Schema(description ="Requested maximum number of representative datapoints after downsampling. Fewer "- + "points are returned when the range contains fewer datapoints.",+ + "points are returned when the range contains fewer available datapoints.",example = "500",requiredMode = Schema.RequiredMode.REQUIRED)public int amountOfPoints;@@ -60,11 +62,15 @@ public final class AssetDatapointLTTBQuery extends AssetDatapointQuery {boolean isNumber = Number.class.isAssignableFrom(attributeType);boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);if (isNumber) {- return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast(value as double precision), ?) from "+ return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), cast("+ + NULLABLE_VALUE_SQL+ + " as double precision), ?) from "+ tableName+ " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";} else if (isBoolean) {- return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), (case when cast(cast(value as text) as boolean) is true then 1 else 0 end), ?) from "+ return "select * from public.unnest((select public.lttb(cast(timestamp as timestamptz), ("+ + BOOLEAN_AS_NUMBER_SQL+ + "), ?) from "+ tableName+ " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";} else {diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.javaindex 484279468..5f62b10ac 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointNearestQuery.java@@ -31,7 +31,8 @@ import org.openremote.model.attribute.AttributeRef;description ="Returns at most one numeric datapoint: the closest value at or before the requested "+ "time. Use `fromTimestamp` as a Unix-seconds timestamp at the HTTP API boundary. `toTimestamp`, "- + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive.")+ + "`fromTime`, and `toTime` are ignored. The equality boundary is inclusive. "+ + "If the nearest datapoint has a null value, its timestamp and null value are returned.")public class AssetDatapointNearestQuery extends AssetDatapointQuery {public AssetDatapointNearestQuery() {}@@ -43,7 +44,9 @@ public class AssetDatapointNearestQuery extends AssetDatapointQuery {@Overridepublic String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {- return "SELECT timestamp as X, cast(value as numeric) as Y "+ return "SELECT timestamp as X, cast("+ + NULLABLE_VALUE_SQL+ + " as numeric) as Y "+ " FROM "+ tableName+ " "diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.javaindex 1d22cf9d6..6181e301c 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java@@ -40,9 +40,11 @@ import org.openremote.model.attribute.AttributeRef;defaultImpl = AssetDatapointAllQuery.class)@Schema(description =- "Polymorphic historical-datapoint query. Select `all` to return every value, "+ "Polymorphic historical or predicted datapoint query. Select `all` to return every value, "+ "`interval` to aggregate values into time buckets, `lttb` to downsample a numeric "- + "or boolean series, or `nearest` to retrieve the last value at or before a time.",+ + "or boolean series, or `nearest` to retrieve the last numeric value at or before a time. "+ + "Returned datapoints use x for the timestamp and y for the value. When a query retains an "+ + "unavailable datapoint, its JSON representation contains x but omits y.",discriminatorProperty = "type",discriminatorMapping = {@DiscriminatorMapping(value = "all", schema = AssetDatapointAllQuery.class),@@ -52,6 +54,15 @@ import org.openremote.model.attribute.AttributeRef;})public abstract class AssetDatapointQuery implements Serializable {+ /** Converts stored JSON null to SQL NULL before casting or aggregating a value. */+ protected static final String NULLABLE_VALUE_SQL = "nullif(value, 'null'::jsonb)";++ /** Maps true/false to 1/0 while preserving unavailable boolean values as SQL NULL. */+ protected static final String BOOLEAN_AS_NUMBER_SQL =+ "CASE cast("+ + NULLABLE_VALUE_SQL+ + " as text)::boolean WHEN true THEN 1 WHEN false THEN 0 ELSE NULL END";+@Schema(description ="Inclusive lower range bound as Unix time in milliseconds for `all`, `interval`, "diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovynew file mode 100644index 000000000..dfeacc9d8--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointNullQueryTest.groovy@@ -0,0 +1,446 @@+/*+ * Copyright 2026, OpenRemote Inc.+ *+ * This program is free software: you can redistribute it and/or modify+ * it under the terms of the GNU Affero General Public License as+ * published by the Free Software Foundation, either version 3 of the+ * License, or (at your option) any later version.+ *+ * This program is distributed in the hope that it will be useful,+ * but WITHOUT ANY WARRANTY; without even the implied warranty of+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+ * GNU Affero General Public License for more details.+ *+ * You should have received a copy of the GNU Affero General Public License+ * along with this program. If not, see <https://www.gnu.org/licenses/>.+ *+ * SPDX-License-Identifier: AGPL-3.0-or-later+ */+package org.openremote.test.assets++import org.openremote.manager.asset.AssetStorageService+import org.openremote.manager.datapoint.AssetDatapointService+import org.openremote.manager.datapoint.AssetPredictedDatapointService+import org.openremote.manager.setup.SetupService+import org.openremote.model.asset.impl.ThingAsset+import org.openremote.model.attribute.Attribute+import org.openremote.model.datapoint.ValueDatapoint+import org.openremote.model.datapoint.query.AssetDatapointAllQuery+import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery+import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery.Formula+import org.openremote.model.datapoint.query.AssetDatapointLTTBQuery+import org.openremote.model.datapoint.query.AssetDatapointNearestQuery+import org.openremote.model.value.ValueType+import org.openremote.setup.integration.KeycloakTestSetup+import org.openremote.test.ManagerContainerTrait+import spock.lang.Specification++import java.time.Instant+import java.time.temporal.ChronoUnit++class AssetDatapointNullQueryTest extends Specification implements ManagerContainerTrait {++ AssetStorageService assetStorageService+ AssetDatapointService historicalService+ AssetPredictedDatapointService predictedService+ ThingAsset asset+ String realm+ long start++ def setup() {+ def container = startContainer(defaultConfig(), defaultServices())+ stopPseudoClock()+ assetStorageService = container.getService(AssetStorageService.class)+ historicalService = container.getService(AssetDatapointService.class)+ predictedService = container.getService(AssetPredictedDatapointService.class)+ realm = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class).realmBuilding.name+ start = getInstantTimeOf(container).truncatedTo(ChronoUnit.SECONDS).toEpochMilli()+ }++ def cleanup() {+ if (asset?.id) {+ assert assetStorageService.delete([asset.id])+ }+ }++ def "numeric All query preserves #description for #storage #valueType.name datapoints"() {+ given: "numeric datapoints containing explicitly stored JSON nulls"+ asset = assetStorageService.merge(new ThingAsset("Null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def timestamps = (0..<values.size()).collect { start + it * 60_000L }+ def datapoints = values.withIndex().collect { value, index ->+ new ValueDatapoint<>(timestamps[index], value)+ }+ // The query must include both boundary timestamps and exclude their neighbours.+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, 111G))+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, 222G))+ service.upsertValues(asset.id, "value", datapoints)++ when: "the inclusive range is queried without aggregation"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointAllQuery(timestamps.first(), timestamps.last()))++ then: "every timestamp is returned newest first, retaining unavailable values"+ result.collect { it.timestamp } == timestamps.reverse()+ result.collect { it.value } == values.reverse()++ where:+ [storage, valueType, description, values] << ["historical", "predicted"].collectMany { storage ->+ [ValueType.NUMBER, ValueType.BIG_NUMBER].collectMany { type ->+ [+ [storage, type, "mixed values", [-5G, 0G, null, 90.125G]],+ [storage, type, "an entirely null range", [null, null, null]]+ ]+ }+ }+ }++ def "boolean All query preserves #description for #storage datapoints"() {+ given: "boolean datapoints containing explicitly stored JSON nulls"+ asset = assetStorageService.merge(new ThingAsset("Boolean null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", ValueType.BOOLEAN)))+ def service = storage == "historical" ? historicalService : predictedService+ def timestamps = (0..<values.size()).collect { start + it * 60_000L }+ def datapoints = values.withIndex().collect { value, index ->+ new ValueDatapoint<>(timestamps[index], value)+ }+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, true))+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, false))+ service.upsertValues(asset.id, "value", datapoints)++ when: "the inclusive range is queried without aggregation"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointAllQuery(timestamps.first(), timestamps.last()))++ then: "true, false, and unavailable remain distinct at their original timestamps"+ result.collect { it.timestamp } == timestamps.reverse()+ result.collect { it.value } == expectedValues.reverse()++ where:+ [storage, description, values, expectedValues] << ["historical", "predicted"].collectMany { storage ->+ [+ [storage, "mixed values", [true, false, null], [1D, 0D, null]],+ [storage, "an entirely null range", [null, null, null], [null, null, null]]+ ]+ }+ }++ def "numeric Nearest query preserves unavailable values for #storage #valueType.name datapoints"() {+ given: "an unavailable datapoint between older and newer available prices"+ asset = assetStorageService.merge(new ThingAsset("Nearest null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ service.upsertValues(asset.id, "value", [+ new ValueDatapoint<>(start, -5G),+ new ValueDatapoint<>(start + 60_000L, null),+ new ValueDatapoint<>(start + 120_000L, 0G),+ new ValueDatapoint<>(start + 180_000L, 90.125G)+ ])+ def queryAt = { long offset ->+ service.queryDatapoints(asset.id, "value", new AssetDatapointNearestQuery(start + offset)).collect {+ [it.timestamp, it.value]+ }+ }++ expect: "an exact match retains the null value and its timestamp"+ queryAt(60_000L) == [[start + 60_000L, null]]++ and: "a query between observations returns the null row, not an older or newer price"+ queryAt(90_000L) == [[start + 60_000L, null]]++ and: "a query before the first observation returns no datapoint"+ queryAt(-1_000L).isEmpty()++ and: "available observations remain selectable at and after their timestamps"+ queryAt(0) == [[start, -5G]]+ queryAt(120_000L) == [[start + 120_000L, 0G]]+ queryAt(240_000L) == [[start + 180_000L, 90.125G]]++ where:+ [storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER]].combinations()+ }++ def "numeric #formula handles unavailable values for #storage #valueType.name with gapFill=#gapFill"() {+ given: "mixed, entirely null, empty, and zero-valued minute buckets"+ asset = assetStorageService.merge(new ThingAsset("Interval null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()+ service.upsertValues(asset.id, "value", [+ [-1_000L, 999G],+ [0L, -6G],+ [10_000L, 0G],+ [20_000L, null],+ [30_000L, 6G],+ [40_000L, 12G],+ [70_000L, null],+ [100_000L, null],+ [190_000L, 0G],+ [240_000L, 999G]+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })++ when: "the four-minute range is aggregated"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 239_999L,+ "1 minute", formula, gapFill))++ then: "null-only buckets remain present and empty buckets follow the gap-fill setting"+ result.collect {+ it.timestamp+ } == (gapFill ? [0L, 60_000L, 120_000L, 180_000L] : [0L, 60_000L, 180_000L])+ .collect { bucketStart + it }+ result.collect { it.value } == (gapFill+ ? [mixedValue, nullBucketValue, null, zeroBucketValue]+ : [mixedValue, nullBucketValue, zeroBucketValue])++ where:+ [storage, valueType, gapFill, formula, mixedValue, nullBucketValue, zeroBucketValue] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER], [false, true]].combinations().collectMany { storage, type, gapFill ->+ [+ [Formula.MIN, -6G, null, 0G],+ [Formula.MAX, 12G, null, 0G],+ [Formula.AVG, 3G, null, 0G],+ [Formula.SUM, 12G, null, 0G],+ [Formula.MEDIAN, 3G, null, 0G],+ // COUNT retains row-count semantics, including stored nulls.+ [Formula.COUNT, 5G, 2G, 1G]+ ].collect { [storage, type, gapFill, *it] }+ }+ }++ def "boolean #formula handles unavailable values for #storage with gapFill=#gapFill"() {+ given: "mixed, entirely null, empty, false, and true minute buckets"+ asset = assetStorageService.merge(new ThingAsset("Boolean interval null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", ValueType.BOOLEAN)))+ def service = storage == "historical" ? historicalService : predictedService+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()+ service.upsertValues(asset.id, "value", [+ [-1_000L, true],+ [0L, true],+ [10_000L, false],+ [20_000L, null],+ [30_000L, null],+ [40_000L, null],+ [70_000L, null],+ [100_000L, null],+ [190_000L, false],+ [200_000L, null],+ [250_000L, true],+ [260_000L, null],+ [300_000L, false]+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })++ when: "the five-minute range is aggregated"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 299_999L,+ "1 minute", formula, gapFill))++ then: "unavailable values are distinct from false and empty buckets follow the gap-fill setting"+ result.collect { it.timestamp } == (gapFill+ ? [0L, 60_000L, 120_000L, 180_000L, 240_000L]+ : [0L, 60_000L, 180_000L, 240_000L]).collect { bucketStart + it }+ result.collect { it.value } == (gapFill+ ? [mixedValue, nullBucketValue, null, falseBucketValue, trueBucketValue]+ : [mixedValue, nullBucketValue, falseBucketValue, trueBucketValue])++ where:+ [storage, gapFill, formula, mixedValue, nullBucketValue, falseBucketValue, trueBucketValue] << [["historical", "predicted"], [false, true]].combinations().collectMany { storage, gapFill ->+ [+ [Formula.MIN, 0D, null, 0D, 1D],+ [Formula.MAX, 1D, null, 0D, 1D],+ [Formula.AVG, 0.5D, null, 0D, 1D],+ [Formula.SUM, 1D, null, 0D, 1D],+ // MEDIAN retains the existing true result when available values are tied.+ [Formula.MEDIAN, 1D, null, 0D, 1D],+ // COUNT retains row-count semantics, including stored nulls.+ [Formula.COUNT, 5D, 2D, 2D, 2D]+ ].collect { [storage, gapFill, *it] }+ }+ }++ def "MODE ignores unavailable values for #storage #valueType.name with gapFill=#gapFill"() {+ given: "nulls outnumber available values, followed by null-only, empty, tied, zero, and high-value buckets"+ asset = assetStorageService.merge(new ThingAsset("Mode null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()+ def buckets = [+ [low, low, high, null, null, null],+ [null, null],+ [],+ [low, high, null],+ [zero, null],+ [high, high, low, null]+ ]+ def datapoints = buckets.withIndex().collectMany { values, bucket ->+ values.withIndex().collect { value, index ->+ new ValueDatapoint<>(bucketStart + bucket * 60_000L + index * 1_000L, value)+ }+ }+ datapoints.add(new ValueDatapoint<>(bucketStart - 1_000L, high))+ datapoints.add(new ValueDatapoint<>(bucketStart + 360_000L, low))+ service.upsertValues(asset.id, "value", datapoints)++ when: "the six-minute range is queried for its most frequent available values"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 359_999L,+ "1 minute", Formula.MODE, gapFill))++ then: "nulls cannot win and existing numeric and boolean tie-breaking is preserved"+ // Preserve existing MODE behavior: empty buckets are omitted for either gapFill setting.+ result.collect { it.timestamp } == [0L, 60_000L, 180_000L, 240_000L, 300_000L]+ .collect { bucketStart + it }+ result.collect { it.value } == [lowResult, null, tieResult, 0D, highResult]++ when: "only the entirely unavailable bucket is queried"+ def unavailable = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart + 60_000L, bucketStart + 119_999L,+ "1 minute", Formula.MODE, gapFill))++ then: "the bucket and its null result remain present"+ unavailable.collect { [it.timestamp, it.value] } == [[bucketStart + 60_000L, null]]++ where:+ [storage, gapFill, valueType, low, high, zero, lowResult, highResult, tieResult] << [["historical", "predicted"], [false, true]].combinations().collectMany { storage, gapFill ->+ [+ [ValueType.NUMBER, -5G, 7G, 0G, -5D, 7D, -5D],+ [ValueType.BIG_NUMBER, -5G, 7G, 0G, -5D, 7D, -5D],+ [ValueType.BOOLEAN, false, true, false, 0D, 1D, 1D]+ ].collect { [storage, gapFill, *it] }+ }+ }++ def "DIFFERENCE preserves unavailable values for #storage #valueType.name with gapFill=#gapFill"() {+ given: "last values, explicit unavailability, recovery, and empty minute buckets"+ asset = assetStorageService.merge(new ThingAsset("Difference null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def bucketStart = Instant.ofEpochMilli(start).truncatedTo(ChronoUnit.MINUTES).toEpochMilli()+ service.upsertValues(asset.id, "value", [+ [-61_000L, 999G],+ [0L, 75G],+ [10_000L, 80G],+ [120_000L, 90G],+ [130_000L, null],+ [240_000L, null],+ [250_000L, 100G],+ [300_000L, 110G],+ [360_000L, 0G],+ [420_000L, -5G],+ [540_000L, -10G],+ [660_000L, 999G]+ ].collect { new ValueDatapoint<>(bucketStart + it[0], it[1]) })++ when: "the eleven-minute range is queried for changes between bucket values"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart, bucketStart + 659_999L,+ "1 minute", Formula.DIFFERENCE, gapFill))++ then: "explicit nulls interrupt differences while empty buckets carry forward when enabled"+ result.collect { it.timestamp } == (gapFill+ ? (0..10).toList() : [0, 2, 4, 5, 6, 7, 9]).collect { bucketStart + it * 60_000L }+ result.collect { it.value } == (gapFill+ ? [0D, 0D, null, null, null, 10D, -110D, -5D, 0D, -5D, 0D]+ : [0D, null, null, 10D, -110D, -5D, -5D])++ when: "the query starts with an empty bucket before the first available value"+ def leadingGap = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart - 60_000L, bucketStart + 119_999L,+ "1 minute", Formula.DIFFERENCE, gapFill))++ then: "an unavailable predecessor does not become an assumed zero baseline"+ leadingGap.collect { [it.timestamp, it.value] } == (gapFill+ ? [[bucketStart - 60_000L, null], [bucketStart, null], [bucketStart + 60_000L, 0D]]+ : [[bucketStart, 0D]])++ when: "only an empty bucket is requested"+ def empty = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart + 60_000L, bucketStart + 119_999L,+ "1 minute", Formula.DIFFERENCE, gapFill))++ then: "carry-forward does not read observations outside the query range"+ empty.collect { [it.timestamp, it.value] } == (gapFill ? [[bucketStart + 60_000L, null]] : [])++ when: "only explicitly unavailable observations are requested"+ def unavailable = service.queryDatapoints(asset.id, "value",+ new AssetDatapointIntervalQuery(bucketStart + 130_000L, bucketStart + 179_999L,+ "1 minute", Formula.DIFFERENCE, gapFill))++ then: "the initial difference stays unavailable"+ unavailable.collect { [it.timestamp, it.value] } == [[bucketStart + 120_000L, null]]++ where:+ [storage, valueType, gapFill] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER], [false, true]].combinations()+ }++ def "LTTB omits unavailable values in #description for #storage #valueType.name datapoints"() {+ given: "datapoints with unavailable observations and out-of-range neighbours"+ asset = assetStorageService.merge(new ThingAsset("LTTB null query test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def timestamps = (0..<values.size()).collect { start + it * 60_000L }+ def datapoints = values.withIndex().collect { value, index ->+ new ValueDatapoint<>(timestamps[index], value)+ }+ def outsideValue = valueType == ValueType.BOOLEAN ? true : 999G+ datapoints.add(new ValueDatapoint<>(timestamps.first() - 60_000L, outsideValue))+ datapoints.add(new ValueDatapoint<>(timestamps.last() + 60_000L, outsideValue))+ service.upsertValues(asset.id, "value", datapoints)++ when: "the requested resolution exceeds the available observations"+ def result = service.queryDatapoints(asset.id, "value",+ new AssetDatapointLTTBQuery(timestamps.first(), timestamps.last(), 10))++ then: "only available observations are returned, chronologically and at their original timestamps"+ result.collect { [it.timestamp, it.value] } == expected.collect { index, value ->+ [timestamps[index], value]+ }++ where:+ [storage, valueType, description, values, expected] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER, ValueType.BOOLEAN]].combinations().collectMany { storage, type ->+ def isBoolean = type == ValueType.BOOLEAN+ [+ [+ storage,+ type,+ "a mixed range",+ isBoolean ? [null, true, null, false, true, null, false, null] : [null, -5G, null, 0G, 90G, null, 40G, null],+ isBoolean ? [[1, 1D], [3, 0D], [4, 1D], [6, 0D]] : [[1, -5D], [3, 0D], [4, 90D], [6, 40D]]+ ],+ [storage, type, "an entirely null range", [null, null, null], []],+ [storage, type, "a single available observation", [null, isBoolean ? false : 0G, null], [[1, 0D]]]+ ]+ }+ }++ def "LTTB downsampling ignores unavailable observations for #storage #valueType.name datapoints"() {+ given: "the same available series stored with and without null observations"+ asset = assetStorageService.merge(new ThingAsset("LTTB downsampling null test")+ .setRealm(realm).addOrReplaceAttributes(new Attribute<>("value", valueType), new Attribute<>("available", valueType)))+ def service = storage == "historical" ? historicalService : predictedService+ def values = valueType == ValueType.BOOLEAN+ ? [null, false, true, null, false, true, null, true, false, null, true, false, null]+ : [null, 0G, 12G, null, -4G, 50G, null, 3G, 0G, null, 9G, 2G, null]+ def datapoints = values.withIndex().collect { value, index ->+ new ValueDatapoint<>(start + index * 60_000L, value)+ }+ service.upsertValues(asset.id, "value", datapoints)+ service.upsertValues(asset.id, "available", datapoints.findAll { it.value != null })++ when: "both series are reduced from eight available points to four"+ def query = new AssetDatapointLTTBQuery(start, start + 720_000L, 4)+ def result = service.queryDatapoints(asset.id, "value", query)+ def reference = service.queryDatapoints(asset.id, "available", query)++ then: "null observations do not affect point selection or the first and last available points"+ result.size() == 4+ result.collect { [it.timestamp, it.value] } == reference.collect { [it.timestamp, it.value] }+ result.first().timestamp == start + 60_000L+ result.last().timestamp == start + 660_000L+ result.every { it.value != null }+ result.collect { it.timestamp } == result.collect { it.timestamp }.sort()++ where:+ [storage, valueType] << [["historical", "predicted"], [ValueType.NUMBER, ValueType.BIG_NUMBER, ValueType.BOOLEAN]].combinations()+ }+}diff --git a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovyindex e3cd2b459..e79561aeb 100644--- a/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy+++ b/test/src/test/groovy/org/openremote/test/rules/residence/ResidenceNotifyAlarmTriggerTest.groovy@@ -54,9 +54,6 @@ import spock.lang.Specificationimport spock.util.concurrent.PollingConditionsimport java.time.Instant-import java.time.LocalDateTime-import java.time.ZoneId-import java.time.temporal.ChronoFieldimport java.time.temporal.ChronoUnitimport java.util.concurrent.CopyOnWriteArrayListimport java.util.concurrent.TimeUnit@@ -70,14 +67,10 @@ import static org.openremote.model.util.ValueUtil.parseclass ResidenceNotifyAlarmTriggerTest extends Specification implements ManagerContainerTrait {def getOccurrenceFromTo(long baseTime, int occurrence) {- def tomorrowMidnight = Instant.ofEpochMilli(baseTime).truncatedTo(ChronoUnit.DAYS).plus(1, ChronoUnit.DAYS).atZone(ZoneId.systemDefault())- def from = LocalDateTime.of(tomorrowMidnight.get(ChronoField.YEAR), tomorrowMidnight.get(ChronoField.MONTH_OF_YEAR), tomorrowMidnight.get(ChronoField.DAY_OF_MONTH), 6, 0, 0)- from = from.plus(occurrence, ChronoUnit.WEEKS)- def to = from.plus(2, ChronoUnit.HOURS)- return [- from.atZone(ZoneId.systemDefault()).toDate(),- to.atZone(ZoneId.systemDefault()).toDate()- ]+ // CalendarEvent evaluates recurrences in UTC, so expected windows must also stay in UTC across DST.+ def from = Instant.ofEpochMilli(baseTime).truncatedTo(ChronoUnit.DAYS)+ .plus(1 + occurrence * 7L, ChronoUnit.DAYS).plus(6, ChronoUnit.HOURS)+ return [Date.from(from), Date.from(from.plus(2, ChronoUnit.HOURS))]}def "Trigger notification when presence is detected and alarm enabled"() {