100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Bugfixes for several attribute types in datapoint query (#1029)
Martin Peeters4fe6a4f2073d
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javaindex cb8e7d759..a97c1a1d6 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -197,7 +197,14 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements Cboolean isNumber = Number.class.isAssignableFrom(attributeType);boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);- try (PreparedStatement st = connection.prepareStatement(datapointQuery.getSQLQuery(getDatapointTableName(), attributeType))) {+ String query;+ try {+ query = datapointQuery.getSQLQuery(getDatapointTableName(), attributeType);+ } catch (IllegalStateException ise) {+ getLogger().log(Level.WARNING, ise.getMessage());+ throw ise;+ }+ try (PreparedStatement st = connection.prepareStatement(query)) {if(parameters.size() > 0) {for(Map.Entry<Integer, Object> param : parameters.entrySet()) {diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.javaindex d7921e8a8..ef640cba6 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointAllQuery.java@@ -19,14 +19,18 @@ public class AssetDatapointAllQuery extends AssetDatapointQuery {this.toTime = toTime;}- public String getSQLQuery(String tableName, Class<?> attributeType) {+ public String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {boolean isNumber = Number.class.isAssignableFrom(attributeType);+ boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);if (isNumber) {- return "select timestamp, value::text::numeric from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";+ return "select timestamp as X, value::text::numeric as Y from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";+ } else if (isBoolean) {+ return "select timestamp as X, (case when VALUE::text::boolean is true then 1 else 0 end) as Y from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";} else {- return "select timestamp, (case when VALUE::text::boolean is true then 1 else 0 end) from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";+ return "select distinct timestamp as X, value as Y from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? order by timestamp desc";}}+public HashMap<Integer, Object> getSQLParameters(AttributeRef attributeRef) {HashMap<Integer, Object> parameters = new HashMap<>();LocalDateTime fromTimestamp = (this.fromTime != null) ? this.fromTime : LocalDateTime.ofInstant(Instant.ofEpochMilli(this.fromTimestamp), ZoneId.systemDefault());diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.javaindex c67012716..b8be9d020 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointIntervalQuery.java@@ -35,13 +35,16 @@ public class AssetDatapointIntervalQuery extends AssetDatapointQuery {}@Override- public String getSQLQuery(String tableName, Class<?> attributeType) {+ public String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {boolean isNumber = Number.class.isAssignableFrom(attributeType);+ boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);String function = (gapFill ? "public.time_bucket_gapfill" : "public.time_bucket");if (isNumber) {return "select " + function + "(?::interval, timestamp) AS x, " + this.formula.toString().toLowerCase() + "(value::text::numeric) FROM " + tableName + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x;";- } else {+ } else if (isBoolean) {return "select " + function + "(?::interval, timestamp) AS x, " + this.formula.toString().toLowerCase() + "(case when VALUE::text::boolean is true then 1 else 0 end) FROM " + tableName + " WHERE ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ? GROUP BY x;";+ } else {+ throw new IllegalStateException("Query of type Interval requires either a number or a boolean attribute.");}}diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.javaindex f8b888338..798fddcc4 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointLTTBQuery.java@@ -24,12 +24,15 @@ public final class AssetDatapointLTTBQuery extends AssetDatapointQuery {}@Override- public String getSQLQuery(String tableName, Class<?> attributeType) {+ public String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {boolean isNumber = Number.class.isAssignableFrom(attributeType);+ boolean isBoolean = Boolean.class.isAssignableFrom(attributeType);if (isNumber) {return "select * from public.unnest((select public.lttb(timestamp::timestamptz, value::double precision, ?) from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";- } else {+ } else if (isBoolean) {return "select * from public.unnest((select public.lttb(timestamp::timestamptz, (case when VALUE::text::boolean is true then 1 else 0 end), ?) from " + tableName + " where ENTITY_ID = ? and ATTRIBUTE_NAME = ? and TIMESTAMP >= ? and TIMESTAMP <= ?))";+ } else {+ throw new IllegalStateException("Query of type LTTB requires either a number or a boolean attribute.");}}diff --git a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.javaindex 52af33935..32a396290 100644--- a/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java+++ b/model/src/main/java/org/openremote/model/datapoint/query/AssetDatapointQuery.java@@ -44,7 +44,7 @@ public abstract class AssetDatapointQuery implements Serializable {public LocalDateTime fromTime;public LocalDateTime toTime;- public String getSQLQuery(String tableName, Class<?> attributeType) {+ public String getSQLQuery(String tableName, Class<?> attributeType) throws IllegalStateException {return null;}diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovynew file mode 100644index 000000000..0460c71f0--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointQueryTest.groovy@@ -0,0 +1,311 @@+package org.openremote.test.assets++import org.openremote.container.util.UniqueIdentifierGenerator+import org.openremote.manager.asset.AssetStorageService+import org.openremote.manager.datapoint.AssetDatapointService+import org.openremote.manager.setup.SetupService+import org.openremote.model.asset.impl.LightAsset+import org.openremote.model.asset.impl.ThingAsset+import org.openremote.model.attribute.Attribute+import org.openremote.model.attribute.AttributeRef+import org.openremote.model.attribute.MetaItem+import org.openremote.model.datapoint.AssetDatapoint+import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery+import org.openremote.model.datapoint.query.AssetDatapointLTTBQuery+import org.openremote.model.geo.GeoJSONPoint+import org.openremote.model.query.AssetQuery+import org.openremote.model.query.filter.RealmPredicate+import org.openremote.model.util.Pair+import org.openremote.model.value.MetaItemType+import org.openremote.model.value.ValueType+import org.openremote.setup.integration.KeycloakTestSetup+import org.openremote.test.ManagerContainerTrait+import spock.lang.Specification+import spock.util.concurrent.PollingConditions++import java.time.LocalDateTime++import static java.util.concurrent.TimeUnit.HOURS++class AssetDatapointQueryTest extends Specification implements ManagerContainerTrait {++ def "Test lttb datapoint query"() {++ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ and: "the container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)+++ when: "requesting the first light asset in City realm"+ def asset = assetStorageService.find(+ new AssetQuery()+ .types(LightAsset.class)+ .realm(new RealmPredicate(keycloakTestSetup.realmCity.name))+ .names("Light 1")+ )++ then: "asset should exist and be correct"+ def assetName = "Light 1"+ def attributeName = "brightness"+ def dateTime = LocalDateTime.now()+ assert asset != null // light 1, 2 and 3+ assert asset.name == assetName+ assert asset.attributes.has(attributeName)++ and: "no datapoints should exist"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))+ assert datapoints.isEmpty()+ }+++ /* ------------------------- */++ when: "datapoints are added to the asset"+ assetDatapointService.upsertValues(asset.getId(), attributeName,+ [+ new Pair<>(50d, dateTime.minusMinutes(25)),+ new Pair<>(40d, dateTime.minusMinutes(20)),+ new Pair<>(30d, dateTime.minusMinutes(15)),+ new Pair<>(20d, dateTime.minusMinutes(10)),+ new Pair<>(10d, dateTime.minusMinutes(5)),+ ]+ )++ then: "datapoints should exist"+ def allDatapoints = new ArrayList<AssetDatapoint>()+ conditions.eventually {+ allDatapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))+ assert allDatapoints.size() == 5+ }++ /* ------------------------- */++ and: "requesting 50 datapoints using the LTTB algorithm should return 5 values"+ def lttbDatapoints1 = assetDatapointService.queryDatapoints(+ asset.getId(),+ asset.getAttribute(attributeName).orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointLTTBQuery(dateTime.minusMinutes(30), dateTime, 5)+ )+ assert lttbDatapoints1.size() == 5++ and: "the 5 values are equal to 5 values in 'allDatapoints'"+ def index = 0+ Collections.reverse(allDatapoints) // since they're returned in reverse other (from new to old)+ allDatapoints.stream().forEach {dp -> {+ assert dp.timestamp == lttbDatapoints1[index].timestamp+ assert dp.value == lttbDatapoints1[index].value+ index+++ }}++ and: "requesting 3 datapoints using LTTB should return 50, 40 and 10 to reflect the algorithm spec."+ def lttbDatapoints2 = assetDatapointService.queryDatapoints(+ asset.getId(),+ asset.getAttribute(attributeName).orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointLTTBQuery(dateTime.minusMinutes(30), dateTime, 3)+ )+ assert lttbDatapoints2.size() == 3+ assert lttbDatapoints2[0].value == 50d+ assert lttbDatapoints2[1].value == 40d+ assert lttbDatapoints2[2].value == 10d+++ /* ------------------------- */++ when: "the datapoints are cleared"+ assetDatapointService.purgeDataPoints()++ and: "datapoints are added that have a spike in value, it should be included in any downsample"+ assetDatapointService.upsertValues(asset.getId(), attributeName,+ [+ // placing them in a random order to verify order that is returned with the query+ new Pair<>(25d, dateTime.minusMinutes(10)),+ new Pair<>(30d, dateTime.minusMinutes(5)),+ new Pair<>(10d, dateTime.minusMinutes(30)),+ new Pair<>(90d, dateTime.minusMinutes(20)),+ new Pair<>(15d, dateTime.minusMinutes(25)),+ new Pair<>(20d, dateTime.minusMinutes(15)),+ ]+ )++ then: "the spike should be present as the 2nd value"+ def lttbDatapoints3 = assetDatapointService.queryDatapoints(+ asset.getId(),+ asset.getAttribute(attributeName).orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointLTTBQuery(dateTime.minusMinutes(60), dateTime, 4)+ )+ assert lttbDatapoints3.size() == 4+ assert lttbDatapoints3[0].value == 10d+ assert lttbDatapoints3[1].value == 90d+ assert lttbDatapoints3[2].value == 20d+ assert lttbDatapoints3[3].value == 30d++ and: "returned datapoints should be in chronological order" // so from earliest to most recent+ def index2 = 0+ lttbDatapoints3.toList().stream().forEach { dp -> {+ if(index2 > 0) {+ assert dp.timestamp > lttbDatapoints3[index2 - 1].timestamp+ }+ index2+++ }}+ }++++ def "Lttb should not accept anything else than boolean and number"() {++ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ and: "the container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)++ when: "a thing asset is added to the building realm"+ def thingId = UniqueIdentifierGenerator.generateId("TestThing")+ def thingAsset = new ThingAsset("TestThing")+ .setId(thingId)+ .setRealm(keycloakTestSetup.realmBuilding.name)+ .setLocation(new GeoJSONPoint(0, 0))+ .addAttributes(+ new Attribute<>("status", ValueType.BOOLEAN, null).addMeta(new MetaItem<>(MetaItemType.STORE_DATA_POINTS)),+ new Attribute<>("deviceId", ValueType.TEXT, null).addMeta(new MetaItem<>(MetaItemType.STORE_DATA_POINTS)),+ )+ thingAsset = assetStorageService.merge(thingAsset)++ and: "add datapoints to the attributes"+ assetDatapointService.upsertValue(thingAsset.getId(), "status", true, LocalDateTime.now())+ assetDatapointService.upsertValue(thingAsset.getId(), "deviceId", "abcdefghijklmnopqrstuvwxyz", LocalDateTime.now())+++ then: "datapoints should exist"+ conditions.eventually {+ def booleanDatapoints = assetDatapointService.getDatapoints(new AttributeRef(thingAsset.getId(), "status"))+ assert booleanDatapoints.size() == 1+ def textDatapoints = assetDatapointService.getDatapoints(new AttributeRef(thingAsset.getId(), "deviceId"))+ assert textDatapoints.size() == 1+ }++ and: "lttbQuery with boolean should not throw exception"+ conditions.eventually {+ def lttbDatapoints = assetDatapointService.queryDatapoints(+ thingAsset.getId(),+ thingAsset.getAttribute("status").orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointLTTBQuery(LocalDateTime.now().minusMinutes(5), LocalDateTime.now(), 3)+ )+ assert lttbDatapoints.size() == 1+ assert lttbDatapoints[0].value == 1.0 // Aka true as boolean+ }++ when: "lttbQuery is created with text, it should throw exception"+ assetDatapointService.queryDatapoints(+ thingAsset.getId(),+ thingAsset.getAttribute("deviceId").orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointLTTBQuery(LocalDateTime.now().minusMinutes(5), LocalDateTime.now(), 3)+ )++ then: "exception to be thrown"+ IllegalStateException ex = thrown(IllegalStateException)+ assert ex.getMessage() == "Query of type LTTB requires either a number or a boolean attribute."+ }++++ def "Interval query should be returned in chronological order"() {++ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ and: "the container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)+++ when: "requesting the first light asset in City realm"+ def asset = assetStorageService.find(+ new AssetQuery()+ .types(LightAsset.class)+ .realm(new RealmPredicate(keycloakTestSetup.realmCity.name))+ .names("Light 1")+ )++ then: "asset should exist and be correct"+ def assetName = "Light 1"+ def attributeName = "brightness"+ def dateTime = LocalDateTime.now()+ assert asset != null // light 1, 2 and 3+ assert asset.name == assetName+ assert asset.attributes.has(attributeName)++ and: "no datapoints should exist"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))+ assert datapoints.isEmpty()+ }+++ /* ------------------------- */++ when: "the first datapoints are added to the asset"+ assetDatapointService.upsertValue(asset.getId(), attributeName, 25d, dateTime.minusMinutes(10))+ assetDatapointService.upsertValue(asset.getId(), attributeName, 30d, dateTime.minusMinutes(5))+ assetDatapointService.upsertValue(asset.getId(), attributeName, 10d, dateTime.minusMinutes(30))++ and: "the clock advances by an hour"+ advancePseudoClock(1, HOURS, container)++ and: "the other datapoints are added to the asset, using an timestamp earlier than the previous ones"+ assetDatapointService.upsertValue(asset.getId(), attributeName, 90d, dateTime.minusMinutes(20))+ assetDatapointService.upsertValue(asset.getId(), attributeName, 15d, dateTime.minusMinutes(25))+ assetDatapointService.upsertValue(asset.getId(), attributeName, 20d, dateTime.minusMinutes(15))++ then: "datapoints should exist"+ def allDatapoints = new ArrayList<AssetDatapoint>()+ conditions.eventually {+ allDatapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))+ assert allDatapoints.size() == 6+ }++ /* ------------------------- */++ and: "returned datapoints (without gapfill) should be in chronological order" // so from earliest to most recent+ def intervalDatapoints1 = assetDatapointService.queryDatapoints(+ asset.getId(),+ asset.getAttribute(attributeName).orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointIntervalQuery(dateTime.minusMinutes(60), dateTime, "1 minute", AssetDatapointIntervalQuery.Formula.AVG, false)+ )+ assert intervalDatapoints1.size() == 6+ def index = 0+ intervalDatapoints1.toList().stream().forEach { dp -> {+ if(index > 0) {+ assert dp.timestamp > intervalDatapoints1[index - 1].timestamp+ }+ index+++ }}++ and: "returned datapoints (with gapfill) should be in chronological order" // so from earliest to most recent+ def intervalDatapoints2 = assetDatapointService.queryDatapoints(+ asset.getId(),+ asset.getAttribute(attributeName).orElseThrow({ new RuntimeException("Missing attribute") }),+ new AssetDatapointIntervalQuery(dateTime.minusMinutes(60), dateTime, "1 minute", AssetDatapointIntervalQuery.Formula.AVG, true)+ )+ assert intervalDatapoints2.size() == 61+ def index2 = 0+ intervalDatapoints2.toList().stream().forEach { dp -> {+ if(index2 > 0) {+ assert dp.timestamp > intervalDatapoints2[index2 - 1].timestamp+ }+ index2+++ }}++ }+}diff --git a/ui/component/or-attribute-history/src/index.ts b/ui/component/or-attribute-history/src/index.tsindex f7555be60..957aed2de 100644--- a/ui/component/or-attribute-history/src/index.ts+++ b/ui/component/or-attribute-history/src/index.ts@@ -705,15 +705,30 @@ export class OrAttributeHistory extends translate(i18next)(LitElement) {this._timeUnits = lowerCaseInterval as TimeUnit;this._stepSize = stepSize;- const response = await manager.rest.api.AssetDatapointResource.getDatapoints(- assetId,- attributeName,- {- type: "all",- fromTimestamp: this._startOfPeriod,- toTimestamp: this._endOfPeriod- } // TODO: maybe optimize this?- );+ const isChart = this._type && (this._type.jsonType === "number" || this._type.jsonType === "boolean");+ let response;+ if(isChart) {+ response = await manager.rest.api.AssetDatapointResource.getDatapoints(+ assetId,+ attributeName,+ {+ type: "lttb",+ fromTimestamp: this._startOfPeriod,+ toTimestamp: this._endOfPeriod,+ amountOfPoints: 100+ }+ )+ } else {+ response = await manager.rest.api.AssetDatapointResource.getDatapoints(+ assetId,+ attributeName,+ {+ type: "all",+ fromTimestamp: this._startOfPeriod,+ toTimestamp: this._endOfPeriod+ }+ );+ }this._loading = false;