Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Added limit options on datapoint CSV export (#1558)

Martin Peeters8041636e5b3c
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
index 5a96343b5..73ecec621 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
@@ -38,6 +38,7 @@ import org.openremote.model.datapoint.DatapointPeriod;
import org.openremote.model.datapoint.DatapointQueryTooLargeException;
import org.openremote.model.datapoint.ValueDatapoint;
import org.openremote.model.datapoint.query.AssetDatapointQuery;
+import org.openremote.model.util.TextUtil;
import org.openremote.model.util.ValueUtil;
import org.postgresql.util.PGobject;
@@ -203,32 +204,41 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
// Verify the query is 'legal' and can be executed
try {
- if(canQueryDatapoints(assetId, attribute, query, parameters)) {
+ if(canQueryDatapoints(query, parameters, maxAmountOfQueryPoints)) {
getLogger().finest("Querying datapoints for: " + attributeRef);
return doQueryDatapoints(assetId, attribute, query, parameters);
}
return Collections.emptyList();
- } catch (IllegalStateException | IllegalArgumentException | DatapointQueryTooLargeException ex) {
+ } catch (DatapointQueryTooLargeException dex) {
+ String msg = "Could not query data points for " + assetId + ". It exceeds the data limit of " + maxAmountOfQueryPoints + " data points.";
+ getLogger().log(Level.WARNING, msg, dex);
+ throw dex;
+ } catch (IllegalStateException | IllegalArgumentException ex) {
getLogger().log(Level.WARNING, ex.getMessage());
throw ex;
}
}
- protected boolean canQueryDatapoints(String assetId, Attribute<?> attribute, String query, Map<Integer, Object> parameters) {
+ protected boolean canQueryDatapoints(String query, Map<Integer, Object> parameters, int datapointLimit) {
+ if(TextUtil.isNullOrEmpty(query)) {
+ throw new IllegalArgumentException("Query is null or empty");
+ }
// If only a maximum amount of data points is allowed,
// use SQL COUNT() to verify if the query does not exceed the maximum.
- if(maxAmountOfQueryPoints > 0) {
+ if(datapointLimit > 0) {
String countQueryStr = "SELECT COUNT(*) FROM (" + query + ") AS count_query";
int amount = persistenceService.doReturningTransaction(entityManager -> {
Query countQuery = entityManager.createNativeQuery(countQueryStr);
- parameters.forEach(countQuery::setParameter);
+ if(parameters != null) {
+ parameters.forEach(countQuery::setParameter);
+ }
return ((Number) countQuery.getSingleResult()).intValue();
});
- if (amount > maxAmountOfQueryPoints) {
- throw new DatapointQueryTooLargeException("Could not query data points for " + assetId + ". It exceeds the data limit of " + maxAmountOfQueryPoints + " data points.");
+ if (amount > datapointLimit) {
+ throw new DatapointQueryTooLargeException();
}
}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
index be8a38b27..05c655a18 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
@@ -137,7 +137,7 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
} catch (IllegalStateException | IllegalArgumentException ex) {
throw new BadRequestException(ex);
} catch (DatapointQueryTooLargeException dqex) {
- throw new WebApplicationException(Response.Status.REQUEST_ENTITY_TOO_LARGE);
+ throw new WebApplicationException(dqex, Response.Status.REQUEST_ENTITY_TOO_LARGE);
} catch (UnsupportedOperationException ex) {
throw new NotSupportedException(ex);
}
@@ -240,6 +240,8 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
}
} catch (JsonProcessingException ex) {
asyncResponse.resume(new BadRequestException(ex));
+ } catch (DatapointQueryTooLargeException dqex) {
+ asyncResponse.resume(new WebApplicationException(dqex, Response.Status.REQUEST_ENTITY_TOO_LARGE));
}
}
}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index bc5f336f9..8159569ed 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -2,6 +2,7 @@ package org.openremote.manager.datapoint;
import org.openremote.agent.protocol.ProtocolDatapointService;
import org.openremote.container.timer.TimerService;
+import org.openremote.model.datapoint.DatapointQueryTooLargeException;
import org.openremote.model.util.UniqueIdentifierGenerator;
import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetStorageService;
@@ -52,9 +53,12 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
public static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";
public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;
+ public static final String OR_DATA_POINTS_EXPORT_LIMIT = "OR_DATA_POINTS_EXPORT_LIMIT";
+ public static final int OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT = 1000000;
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
protected static final String EXPORT_STORAGE_DIR_NAME = "datapoint";
protected int maxDatapointAgeDays;
+ protected int datapointExportLimit;
protected Path exportPath;
@Override
@@ -78,6 +82,14 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
LOG.log(Level.INFO, "Data point purge interval days = " + maxDatapointAgeDays);
}
+ datapointExportLimit = getInteger(container.getConfig(), OR_DATA_POINTS_EXPORT_LIMIT, OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT);
+
+ if (datapointExportLimit <= 0) {
+ LOG.warning(OR_DATA_POINTS_EXPORT_LIMIT + " value is not a valid value so the export data points won't be limited");
+ } else {
+ LOG.log(Level.INFO, "Data point export limit = " + datapointExportLimit);
+ }
+
Path storageDir = persistenceService.getStorageDir();
exportPath = storageDir.resolve(EXPORT_STORAGE_DIR_NAME);
// Ensure export dir exists and is writable
@@ -233,15 +245,35 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,
long fromTimestamp,
long toTimestamp) {
+ try {
+ String query = getSelectExportQuery(attributeRefs, fromTimestamp, toTimestamp);
+
+ // Verify the query is 'legal' and can be executed
+ if(canQueryDatapoints(query, null, datapointExportLimit)) {
+ return doExportDatapoints(attributeRefs, fromTimestamp, toTimestamp);
+ }
+ throw new RuntimeException("Could not export datapoints.");
+
+ } catch (DatapointQueryTooLargeException dqex) {
+ String msg = "Could not export data points. It exceeds the data limit of " + datapointExportLimit + " data points.";
+ getLogger().log(Level.WARNING, msg, dqex);
+ throw dqex;
+ }
+ }
+
+ protected ScheduledFuture<File> doExportDatapoints(AttributeRef[] attributeRefs,
+ long fromTimestamp,
+ long toTimestamp) {
+
return scheduledExecutorService.schedule(() -> {
String fileName = UniqueIdentifierGenerator.generateId() + ".csv";
- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))
- .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))
- .append(")) to '/storage/")
- .append(EXPORT_STORAGE_DIR_NAME)
- .append("/")
- .append(fileName)
- .append("' delimiter ',' CSV HEADER;");
+ StringBuilder sb = new StringBuilder("copy (")
+ .append(getSelectExportQuery(attributeRefs, fromTimestamp, toTimestamp))
+ .append(") to '/storage/")
+ .append(EXPORT_STORAGE_DIR_NAME)
+ .append("/")
+ .append(fileName)
+ .append("' delimiter ',' CSV HEADER;");
persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
@@ -249,4 +281,15 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
return exportPath.resolve(fileName).toFile();
}, 0, TimeUnit.MILLISECONDS);
}
+
+ /**
+ * Function for building an SQL query that SELECTs the content for the data export.
+ * Currently, it includes timestamp, asset name, attribute name, and the data point value.
+ * Be aware: this SQL query does NOT contain any CSV-related statements.
+ */
+ protected String getSelectExportQuery(AttributeRef[] attributeRefs, long fromTimestamp, long toTimestamp) {
+ return new StringBuilder(String.format("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))
+ .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))
+ .append(")").toString();
+ }
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java b/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java
index 3f9658f3d..55f758187 100644
--- a/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java
+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointQueryTooLargeException.java
@@ -21,6 +21,8 @@ package org.openremote.model.datapoint;
public class DatapointQueryTooLargeException extends RuntimeException {
+ public DatapointQueryTooLargeException() {}
+
public DatapointQueryTooLargeException(String message) {
super(message);
}
diff --git a/profile/deploy.yml b/profile/deploy.yml
index 314a3cce9..564ddd46a 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -339,6 +339,9 @@ services:
# Defaults to 100.000. When set to 0, it disables the limit.
# OR_DATA_POINTS_QUERY_LIMIT: 100000
+ # Configure the limit of data points that can be exported to CSV. Defaults to 1 million data points.
+ # OR_DATA_POINTS_EXPORT_LIMIT: 10000000
+
# App id for the API of OpenWeather: https://openweathermap.org
# OR_OPEN_WEATHER_API_APP_ID
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
new file mode 100644
index 000000000..9a047c240
--- /dev/null
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
@@ -0,0 +1,108 @@
+package org.openremote.test.assets
+
+import org.openremote.manager.asset.AssetStorageService
+import org.openremote.manager.datapoint.AssetDatapointService
+import org.openremote.manager.setup.SetupService
+import org.openremote.model.asset.impl.LightAsset
+import org.openremote.model.attribute.AttributeRef
+import org.openremote.model.datapoint.DatapointQueryTooLargeException
+import org.openremote.model.datapoint.ValueDatapoint
+import org.openremote.model.query.AssetQuery
+import org.openremote.model.query.filter.RealmPredicate
+import org.openremote.setup.integration.KeycloakTestSetup
+import org.openremote.test.ManagerContainerTrait
+import spock.lang.Specification
+import spock.util.concurrent.PollingConditions
+
+import java.time.LocalDateTime
+import java.time.ZoneId
+
+class AssetDatapointExportTest extends Specification implements ManagerContainerTrait {
+
+ def "Test CSV export functionality for asset data points"() {
+
+ given: "expected conditions"
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+
+ and: "the container is started"
+ def container = startContainer(defaultConfig(), defaultServices())
+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
+ def assetDatapointService = container.getService(AssetDatapointService.class)
+ assetDatapointService.datapointExportLimit = 1000
+
+ when: "requesting the first light asset in City realm"
+ def asset = assetStorageService.find(
+ new AssetQuery()
+ .types(LightAsset.class)
+ .realm(new RealmPredicate(keycloakTestSetup.realmCity.name))
+ .names("Light 1")
+ )
+
+ then: "asset should exist and be correct"
+ def assetName = "Light 1"
+ def attributeName = "brightness"
+ def dateTime = LocalDateTime.now()
+ assert asset != null // light 1, 2 and 3
+ assert asset.name == assetName
+ assert asset.attributes.has(attributeName)
+
+ and: "no datapoints should exist"
+ conditions.eventually {
+ def datapoints = assetDatapointService.getDatapoints(new AttributeRef(asset.getId(), attributeName))
+ assert datapoints.isEmpty()
+ }
+
+ /* ------------------------- */
+
+ when: "datapoints are added to the asset"
+ assetDatapointService.upsertValues(asset.getId(), attributeName,
+ [
+ new ValueDatapoint<>(dateTime.minusMinutes(25).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 50d),
+ new ValueDatapoint<>(dateTime.minusMinutes(20).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 40d),
+ new ValueDatapoint<>(dateTime.minusMinutes(15).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 30d),
+ new ValueDatapoint<>(dateTime.minusMinutes(10).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 20d),
+ new ValueDatapoint<>(dateTime.minusMinutes(5).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 10d),
+ ]
+ )
+
+ then: "the CSV export should return a file"
+ def csvExport1Future = assetDatapointService.exportDatapoints(
+ [new AttributeRef(asset.id, attributeName)] as AttributeRef[],
+ dateTime.minusMinutes(30).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),
+ dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()
+ )
+ assert csvExport1Future != null
+ def csvExport1 = csvExport1Future.get()
+ assert csvExport1 != null
+
+ and: "the CSV export should contain all 5 data points"
+ def csvExport1Lines = csvExport1.readLines()
+ assert csvExport1Lines.size() == 6
+ assert csvExport1Lines[1].endsWith("10.0")
+ assert csvExport1Lines[2].endsWith("20.0")
+ assert csvExport1Lines[3].endsWith("30.0")
+ assert csvExport1Lines[4].endsWith("40.0")
+ assert csvExport1Lines[5].endsWith("50.0")
+
+ /* ------------------------- */
+
+ when: "the limit of data export is lowered to 4"
+ assetDatapointService.datapointExportLimit = 4
+
+ and: "we try to export the same amount of data points"
+ def csvExport2Future = assetDatapointService.exportDatapoints(
+ [new AttributeRef(asset.id, attributeName)] as AttributeRef[],
+ dateTime.minusMinutes(30).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),
+ dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()
+ )
+
+ then: "the CSV export should throw an exception"
+ thrown(DatapointQueryTooLargeException)
+
+ /* ------------------------- */
+
+ cleanup: "Remove the limit on datapoint querying"
+ assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT;
+ }
+}
diff --git a/ui/app/manager/src/pages/page-export.ts b/ui/app/manager/src/pages/page-export.ts
index fd44dae1a..254dd3ad3 100644
--- a/ui/app/manager/src/pages/page-export.ts
+++ b/ui/app/manager/src/pages/page-export.ts
@@ -12,6 +12,8 @@ import { AttributeRef } from "@openremote/model";
import moment from "moment";
import { buttonStyle } from "@openremote/or-rules";
import {createSelector} from "reselect";
+import {isAxiosError} from "@openremote/rest";
+import { showSnackbar } from "@openremote/or-mwc-components/or-mwc-snackbar";
import { showDialog } from "@openremote/or-mwc-components/or-mwc-dialog";
const tableStyle = require("@material/data-table/dist/mdc.data-table.css");
@@ -389,6 +391,13 @@ export class PageExport extends Page<AppStateKeyed> {
link.setAttribute("download", "dataexport.zip");
document.body.appendChild(link);
link.click();
+
+ }).catch(ex => {
+ if(isAxiosError(ex)) {
+ if(ex.response?.status === 413) {
+ showSnackbar(undefined, "exportTooLargeError");
+ }
+ }
});
}
diff --git a/ui/app/shared/locales/en/or.json b/ui/app/shared/locales/en/or.json
index c821ca38c..5b2f75835 100644
--- a/ui/app/shared/locales/en/or.json
+++ b/ui/app/shared/locales/en/or.json
@@ -574,6 +574,7 @@
"attributeName": "Attribute name",
"oldestDatapoint": "Oldest datapoint",
"latestDatapoint": "Latest datapoint",
+ "exportTooLargeError": "Could not export data: you've selected too much data!",
"arrayDimensions": "Array dimensions",
"addParameter": "Add parameter",
"addItem": "Add item",
diff --git a/ui/app/shared/locales/nl/or.json b/ui/app/shared/locales/nl/or.json
index c6d9db105..58d5628e3 100644
--- a/ui/app/shared/locales/nl/or.json
+++ b/ui/app/shared/locales/nl/or.json
@@ -572,6 +572,7 @@
"attributeName": "Attribuut naam",
"oldestDatapoint": "Oudste datapunt",
"latestDatapoint": "Laatste datapunt",
+ "exportTooLargeError": "Kon data niet exporteren: er is te veel data geselecteerd.",
"arrayDimensions": "Array dimensies",
"addParameter": "Voeg parameter toe",
"addItem": "Voeg item toe",