100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Added limit options on datapoint CSV export (#1558)
Martin Peeters8041636e5b3c
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javaindex 5a96343b5..73ecec621 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -38,6 +38,7 @@ import org.openremote.model.datapoint.DatapointPeriod;import org.openremote.model.datapoint.DatapointQueryTooLargeException;import org.openremote.model.datapoint.ValueDatapoint;import org.openremote.model.datapoint.query.AssetDatapointQuery;+import org.openremote.model.util.TextUtil;import org.openremote.model.util.ValueUtil;import org.postgresql.util.PGobject;@@ -203,32 +204,41 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C// Verify the query is 'legal' and can be executedtry {- if(canQueryDatapoints(assetId, attribute, query, parameters)) {+ if(canQueryDatapoints(query, parameters, maxAmountOfQueryPoints)) {getLogger().finest("Querying datapoints for: " + attributeRef);return doQueryDatapoints(assetId, attribute, query, parameters);}return Collections.emptyList();- } catch (IllegalStateException | IllegalArgumentException | DatapointQueryTooLargeException ex) {+ } catch (DatapointQueryTooLargeException dex) {+ String msg = "Could not query data points for " + assetId + ". It exceeds the data limit of " + maxAmountOfQueryPoints + " data points.";+ getLogger().log(Level.WARNING, msg, dex);+ throw dex;+ } catch (IllegalStateException | IllegalArgumentException ex) {getLogger().log(Level.WARNING, ex.getMessage());throw ex;}}- protected boolean canQueryDatapoints(String assetId, Attribute<?> attribute, String query, Map<Integer, Object> parameters) {+ protected boolean canQueryDatapoints(String query, Map<Integer, Object> parameters, int datapointLimit) {+ if(TextUtil.isNullOrEmpty(query)) {+ throw new IllegalArgumentException("Query is null or empty");+ }// If only a maximum amount of data points is allowed,// use SQL COUNT() to verify if the query does not exceed the maximum.- if(maxAmountOfQueryPoints > 0) {+ if(datapointLimit > 0) {String countQueryStr = "SELECT COUNT(*) FROM (" + query + ") AS count_query";int amount = persistenceService.doReturningTransaction(entityManager -> {Query countQuery = entityManager.createNativeQuery(countQueryStr);- parameters.forEach(countQuery::setParameter);+ if(parameters != null) {+ parameters.forEach(countQuery::setParameter);+ }return ((Number) countQuery.getSingleResult()).intValue();});- if (amount > maxAmountOfQueryPoints) {- throw new DatapointQueryTooLargeException("Could not query data points for " + assetId + ". It exceeds the data limit of " + maxAmountOfQueryPoints + " data points.");+ if (amount > datapointLimit) {+ throw new DatapointQueryTooLargeException();}}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.javaindex be8a38b27..05c655a18 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java@@ -137,7 +137,7 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As} catch (IllegalStateException | IllegalArgumentException ex) {throw new BadRequestException(ex);} catch (DatapointQueryTooLargeException dqex) {- throw new WebApplicationException(Response.Status.REQUEST_ENTITY_TOO_LARGE);+ throw new WebApplicationException(dqex, Response.Status.REQUEST_ENTITY_TOO_LARGE);} catch (UnsupportedOperationException ex) {throw new NotSupportedException(ex);}@@ -240,6 +240,8 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As}} catch (JsonProcessingException ex) {asyncResponse.resume(new BadRequestException(ex));+ } catch (DatapointQueryTooLargeException dqex) {+ asyncResponse.resume(new WebApplicationException(dqex, Response.Status.REQUEST_ENTITY_TOO_LARGE));}}}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex bc5f336f9..8159569ed 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -2,6 +2,7 @@ package org.openremote.manager.datapoint;import org.openremote.agent.protocol.ProtocolDatapointService;import org.openremote.container.timer.TimerService;+import org.openremote.model.datapoint.DatapointQueryTooLargeException;import org.openremote.model.util.UniqueIdentifierGenerator;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetStorageService;@@ -52,9 +53,12 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoipublic static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;+ public static final String OR_DATA_POINTS_EXPORT_LIMIT = "OR_DATA_POINTS_EXPORT_LIMIT";+ public static final int OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT = 1000000;private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());protected static final String EXPORT_STORAGE_DIR_NAME = "datapoint";protected int maxDatapointAgeDays;+ protected int datapointExportLimit;protected Path exportPath;@Override@@ -78,6 +82,14 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiLOG.log(Level.INFO, "Data point purge interval days = " + maxDatapointAgeDays);}+ datapointExportLimit = getInteger(container.getConfig(), OR_DATA_POINTS_EXPORT_LIMIT, OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT);++ if (datapointExportLimit <= 0) {+ LOG.warning(OR_DATA_POINTS_EXPORT_LIMIT + " value is not a valid value so the export data points won't be limited");+ } else {+ LOG.log(Level.INFO, "Data point export limit = " + datapointExportLimit);+ }+Path storageDir = persistenceService.getStorageDir();exportPath = storageDir.resolve(EXPORT_STORAGE_DIR_NAME);// Ensure export dir exists and is writable@@ -233,15 +245,35 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoipublic ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,long fromTimestamp,long toTimestamp) {+ try {+ String query = getSelectExportQuery(attributeRefs, fromTimestamp, toTimestamp);++ // Verify the query is 'legal' and can be executed+ if(canQueryDatapoints(query, null, datapointExportLimit)) {+ return doExportDatapoints(attributeRefs, fromTimestamp, toTimestamp);+ }+ throw new RuntimeException("Could not export datapoints.");++ } catch (DatapointQueryTooLargeException dqex) {+ String msg = "Could not export data points. It exceeds the data limit of " + datapointExportLimit + " data points.";+ getLogger().log(Level.WARNING, msg, dqex);+ throw dqex;+ }+ }++ protected ScheduledFuture<File> doExportDatapoints(AttributeRef[] attributeRefs,+ long fromTimestamp,+ long toTimestamp) {+return scheduledExecutorService.schedule(() -> {String fileName = UniqueIdentifierGenerator.generateId() + ".csv";- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))- .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))- .append(")) to '/storage/")- .append(EXPORT_STORAGE_DIR_NAME)- .append("/")- .append(fileName)- .append("' delimiter ',' CSV HEADER;");+ StringBuilder sb = new StringBuilder("copy (")+ .append(getSelectExportQuery(attributeRefs, fromTimestamp, toTimestamp))+ .append(") to '/storage/")+ .append(EXPORT_STORAGE_DIR_NAME)+ .append("/")+ .append(fileName)+ .append("' delimiter ',' CSV HEADER;");persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());@@ -249,4 +281,15 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoireturn exportPath.resolve(fileName).toFile();}, 0, TimeUnit.MILLISECONDS);}++ /**+ * Function for building an SQL query that SELECTs the content for the data export.+ * Currently, it includes timestamp, asset name, attribute name, and the data point value.+ * Be aware: this SQL query does NOT contain any CSV-related statements.+ */+ protected String getSelectExportQuery(AttributeRef[] attributeRefs, long fromTimestamp, long toTimestamp) {+ return new StringBuilder(String.format("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))+ .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))+ .append(")").toString();+ }}diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java b/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.javaindex 3f9658f3d..55f758187 100644--- a/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java@@ -21,6 +21,8 @@ package org.openremote.model.datapoint;public class DatapointQueryTooLargeException extends RuntimeException {+ public DatapointQueryTooLargeException() {}+public DatapointQueryTooLargeException(String message) {super(message);}diff --git a/profile/deploy.yml b/profile/deploy.ymlindex 314a3cce9..564ddd46a 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -339,6 +339,9 @@ services:# Defaults to 100.000. When set to 0, it disables the limit.# OR_DATA_POINTS_QUERY_LIMIT: 100000+ # Configure the limit of data points that can be exported to CSV. Defaults to 1 million data points.+ # OR_DATA_POINTS_EXPORT_LIMIT: 10000000+# App id for the API of OpenWeather: https://openweathermap.org# OR_OPEN_WEATHER_API_APP_IDdiff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovynew file mode 100644index 000000000..9a047c240--- /dev/null+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy@@ -0,0 +1,108 @@+package org.openremote.test.assets++import org.openremote.manager.asset.AssetStorageService+import org.openremote.manager.datapoint.AssetDatapointService+import org.openremote.manager.setup.SetupService+import org.openremote.model.asset.impl.LightAsset+import org.openremote.model.attribute.AttributeRef+import org.openremote.model.datapoint.DatapointQueryTooLargeException+import org.openremote.model.datapoint.ValueDatapoint+import org.openremote.model.query.AssetQuery+import org.openremote.model.query.filter.RealmPredicate+import org.openremote.setup.integration.KeycloakTestSetup+import org.openremote.test.ManagerContainerTrait+import spock.lang.Specification+import spock.util.concurrent.PollingConditions++import java.time.LocalDateTime+import java.time.ZoneId++class AssetDatapointExportTest extends Specification implements ManagerContainerTrait {++ def "Test CSV export functionality for asset data points"() {++ given: "expected conditions"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)++ and: "the container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)+ assetDatapointService.datapointExportLimit = 1000++ when: "requesting the first light asset in City realm"+ def asset = assetStorageService.find(+ new AssetQuery()+ .types(LightAsset.class)+ .realm(new RealmPredicate(keycloakTestSetup.realmCity.name))+ .names("Light 1")+ )++ then: "asset should exist and be correct"+ def assetName = "Light 1"+ def attributeName = "brightness"+ def dateTime = LocalDateTime.now()+ assert asset != null // light 1, 2 and 3+ assert asset.name == assetName+ assert asset.attributes.has(attributeName)++ and: "no datapoints should exist"+ conditions.eventually {+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))+ assert datapoints.isEmpty()+ }++ /* ------------------------- */++ when: "datapoints are added to the asset"+ assetDatapointService.upsertValues(asset.getId(), attributeName,+ [+ new ValueDatapoint<>(dateTime.minusMinutes(25).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 50d),+ new ValueDatapoint<>(dateTime.minusMinutes(20).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 40d),+ new ValueDatapoint<>(dateTime.minusMinutes(15).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 30d),+ new ValueDatapoint<>(dateTime.minusMinutes(10).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 20d),+ new ValueDatapoint<>(dateTime.minusMinutes(5).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 10d),+ ]+ )++ then: "the CSV export should return a file"+ def csvExport1Future = assetDatapointService.exportDatapoints(+ [new AttributeRef(asset.id, attributeName)] as AttributeRef[],+ dateTime.minusMinutes(30).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),+ dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()+ )+ assert csvExport1Future != null+ def csvExport1 = csvExport1Future.get()+ assert csvExport1 != null++ and: "the CSV export should contain all 5 data points"+ def csvExport1Lines = csvExport1.readLines()+ assert csvExport1Lines.size() == 6+ assert csvExport1Lines[1].endsWith("10.0")+ assert csvExport1Lines[2].endsWith("20.0")+ assert csvExport1Lines[3].endsWith("30.0")+ assert csvExport1Lines[4].endsWith("40.0")+ assert csvExport1Lines[5].endsWith("50.0")++ /* ------------------------- */++ when: "the limit of data export is lowered to 4"+ assetDatapointService.datapointExportLimit = 4++ and: "we try to export the same amount of data points"+ def csvExport2Future = assetDatapointService.exportDatapoints(+ [new AttributeRef(asset.id, attributeName)] as AttributeRef[],+ dateTime.minusMinutes(30).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),+ dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()+ )++ then: "the CSV export should throw an exception"+ thrown(DatapointQueryTooLargeException)++ /* ------------------------- */++ cleanup: "Remove the limit on datapoint querying"+ assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT;+ }+}diff --git a/ui/app/manager/src/pages/page-export.ts b/ui/app/manager/src/pages/page-export.tsindex fd44dae1a..254dd3ad3 100644--- a/ui/app/manager/src/pages/page-export.ts+++ b/ui/app/manager/src/pages/page-export.ts@@ -12,6 +12,8 @@ import { AttributeRef } from "@openremote/model";import moment from "moment";import { buttonStyle } from "@openremote/or-rules";import {createSelector} from "reselect";+import {isAxiosError} from "@openremote/rest";+import { showSnackbar } from "@openremote/or-mwc-components/or-mwc-snackbar";import { showDialog } from "@openremote/or-mwc-components/or-mwc-dialog";const tableStyle = require("@material/data-table/dist/mdc.data-table.css");@@ -389,6 +391,13 @@ export class PageExport extends Page<AppStateKeyed> {link.setAttribute("download", "dataexport.zip");document.body.appendChild(link);link.click();++ }).catch(ex => {+ if(isAxiosError(ex)) {+ if(ex.response?.status === 413) {+ showSnackbar(undefined, "exportTooLargeError");+ }+ }});}diff --git a/ui/app/shared/locales/en/or.json b/ui/app/shared/locales/en/or.jsonindex c821ca38c..5b2f75835 100644--- a/ui/app/shared/locales/en/or.json+++ b/ui/app/shared/locales/en/or.json@@ -574,6 +574,7 @@"attributeName": "Attribute name","oldestDatapoint": "Oldest datapoint","latestDatapoint": "Latest datapoint",+ "exportTooLargeError": "Could not export data: you've selected too much data!","arrayDimensions": "Array dimensions","addParameter": "Add parameter","addItem": "Add item",diff --git a/ui/app/shared/locales/nl/or.json b/ui/app/shared/locales/nl/or.jsonindex c6d9db105..58d5628e3 100644--- a/ui/app/shared/locales/nl/or.json+++ b/ui/app/shared/locales/nl/or.json@@ -572,6 +572,7 @@"attributeName": "Attribuut naam","oldestDatapoint": "Oudste datapunt","latestDatapoint": "Laatste datapunt",+ "exportTooLargeError": "Kon data niet exporteren: er is te veel data geselecteerd.","arrayDimensions": "Array dimensies","addParameter": "Voeg parameter toe","addItem": "Voeg item toe",