100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix timezone handling for datapoints export (#2882)
Eric Bariaux5b77bef9ea87
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 8dec9776e..828834010 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -31,6 +31,7 @@ import java.sql.Statement;import java.sql.Timestamp;import java.time.Duration;import java.time.Instant;+import java.time.LocalDateTime;import java.time.ZoneId;import java.util.*;import java.util.concurrent.TimeUnit;@@ -253,13 +254,19 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoitry {persistenceService.doTransaction(em -> em.unwrap(Session.class).doWork(connection -> {String tempTableName = "tmp_export_attributes";+ String tempBoundsTableName = "tmp_export_bounds";- // Create temp table and insert attribute refs safely via PreparedStatement- // to prevent SQL injection (instead of string-interpolating IDs into the query)+ // Create temp tables and insert parameters safely via PreparedStatement+ // to prevent SQL injection (instead of string-interpolating IDs into the query).try (Statement statement = connection.createStatement()) {+ // We have 2 different tables because for attribute_name, we have one entry per exported attributestatement.execute("create temp table " + tempTableName + " (entity_id text, attribute_name text) on commit drop");+ // but for timestamps, we always have only one row, as the export period is the same for all attributes+ statement.execute(+ "create temp table " + tempBoundsTableName + " (from_timestamp timestamp, to_timestamp timestamp) on commit drop"+ );}try (PreparedStatement insertStatement = connection.prepareStatement(@@ -273,8 +280,16 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiinsertStatement.executeBatch();}+ try (PreparedStatement boundsStatement = connection.prepareStatement(+ "insert into " + tempBoundsTableName + " (from_timestamp, to_timestamp) values (?, ?)"+ )) {+ boundsStatement.setObject(1, toDatapointLocalDateTime(fromTimestamp));+ boundsStatement.setObject(2, toDatapointLocalDateTime(toTimestamp));+ boundsStatement.executeUpdate();+ }+// Build the COPY ... TO STDOUT query based on format- String copyQuery = buildCopyToStdoutQuery(tempTableName, fromTimestamp, toTimestamp, format, attributeRefs);+ String copyQuery = buildCopyToStdoutQuery(tempTableName, tempBoundsTableName, format, attributeRefs);try {PGConnection pgConnection = connection.unwrap(PGConnection.class);@@ -311,10 +326,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi* Builds a COPY ... TO STDOUT query for streaming CSV data based on the export format.* Uses a temporary table for safe attribute filtering (SQL injection prevention).*/- private String buildCopyToStdoutQuery(String tempTableName, long fromTimestamp, long toTimestamp,+ private String buildCopyToStdoutQuery(String tempTableName, String tempBoundsTableName,DatapointExportFormat format, AttributeRef[] attributeRefs) {- long fromSeconds = fromTimestamp / 1000;- long toSeconds = toTimestamp / 1000;final String TO_STDOUT_WITH_CSV_FORMAT = ") TO STDOUT WITH (FORMAT CSV, HEADER, DELIMITER ',');";if (format == DatapointExportFormat.CSV_CROSSTAB || format == DatapointExportFormat.CSV_CROSSTAB_MINUTE) {@@ -345,19 +358,21 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi"from asset_datapoint ad " +"join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " ++ "cross join %s b " ++ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +"group by bucket_timestamp, header " +"order by bucket_timestamp, header",- tempTableName, fromSeconds, toSeconds);+ tempTableName, tempBoundsTableName);} else {innerQuery = String.format("select ad.timestamp, a.name || '' : '' || ad.attribute_name as header, ad.value " +"from asset_datapoint ad " +"join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " ++ "cross join %s b " ++ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +"order by ad.timestamp, header",- tempTableName, fromSeconds, toSeconds);+ tempTableName, tempBoundsTableName);}return String.format(@@ -370,9 +385,10 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi"from asset_datapoint ad " +"join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " ++ "cross join %s b " ++ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +"order by ad.timestamp desc",- tempTableName, fromSeconds, toSeconds);+ tempTableName, tempBoundsTableName);return "copy (" + innerQuery + TO_STDOUT_WITH_CSV_FORMAT;}}@@ -399,11 +415,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi* It returns a structure containing the parametrized query as a string and the parameter values.*/protected ExportQuery getSelectExportQuery(AttributeRef[] attributeRefs, long fromTimestamp, long toTimestamp) {- StringBuilder sb = new StringBuilder("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(?) and ad.timestamp <= to_timestamp(?) and ");+ StringBuilder sb = new StringBuilder("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= ? and ad.timestamp <= ? and ");Map<Integer, Object> parameters = new HashMap<>();int paramIndex = 1;- parameters.put(paramIndex++, fromTimestamp / 1000);- parameters.put(paramIndex++, toTimestamp / 1000);+ parameters.put(paramIndex++, toDatapointLocalDateTime(fromTimestamp));+ parameters.put(paramIndex++, toDatapointLocalDateTime(toTimestamp));sb.append("(ad.entity_id, ad.attribute_name) in (");for (int i = 0; i < attributeRefs.length; i++) {@@ -429,4 +445,10 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi}return assetId;}++ private static LocalDateTime toDatapointLocalDateTime(long timestamp) {+ // Datapoints are stored in a timestamp-without-time-zone column using this same JVM-local conversion.+ // Keep export bounds in that representation so PostgreSQL's session timezone does not affect filtering.+ return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDateTime();+ }}diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovyindex c634bb062..4955536c3 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy@@ -1,5 +1,8 @@package org.openremote.test.assets+import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider+import org.hibernate.engine.spi.SessionFactoryImplementor+import org.openremote.container.persistence.PersistenceServiceimport org.openremote.manager.asset.AssetStorageServiceimport org.openremote.manager.datapoint.AssetDatapointServiceimport org.openremote.manager.setup.SetupService@@ -27,18 +30,31 @@ import static org.openremote.model.value.ValueType.NUMBERclass AssetDatapointExportTest extends Specification implements ManagerContainerTrait {+ // Important for test is that we use a different TZ for DB than for JVM+ private static final String EXPORT_TEST_DATABASE_TIME_ZONE = ZoneId.systemDefault().id == "Europe/Amsterdam" ? "UTC" : "Europe/Amsterdam"+ private static final int EXPORT_TEST_DATABASE_POOL_SIZE = 5+def "Test CSV export functionality for asset data points"() {given: "expected conditions"def conditions = new PollingConditions(timeout: 10, delay: 0.2)- and: "the container is started"- def container = startContainer(defaultConfig(), defaultServices())+ and: "the container is started with a bounded DB connection pool"+ def config = defaultConfig()+ config.put(PersistenceService.OR_DB_POOL_MIN_SIZE, Integer.toString(EXPORT_TEST_DATABASE_POOL_SIZE))+ config.put(PersistenceService.OR_DB_POOL_MAX_SIZE, Integer.toString(EXPORT_TEST_DATABASE_POOL_SIZE))+ def container = startContainer(config, defaultServices())def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)def assetStorageService = container.getService(AssetStorageService.class)def assetDatapointService = container.getService(AssetDatapointService.class)+ def persistenceService = container.getService(PersistenceService.class)+ setDatabaseSessionTimeZone(persistenceService, EXPORT_TEST_DATABASE_TIME_ZONE, EXPORT_TEST_DATABASE_POOL_SIZE)assetDatapointService.datapointExportLimit = 1000+ and: "the database session timezone is different from the JVM timezone"+ assert getDatabaseSessionTimeZone(persistenceService) == EXPORT_TEST_DATABASE_TIME_ZONE+ assert ZoneId.systemDefault() != ZoneId.of(EXPORT_TEST_DATABASE_TIME_ZONE)+when: "requesting the first light asset in City realm"def asset = assetStorageService.find(new AssetQuery()@@ -159,7 +175,12 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer/* ------------------------- */cleanup: "Remove the limit on datapoint querying"- assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT;+ if (assetDatapointService != null) {+ assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT+ }+ if (persistenceService != null) {+ setDatabaseSessionTimeZone(persistenceService, "UTC", EXPORT_TEST_DATABASE_POOL_SIZE)+ }}def "Export query is not vulnerable to SQL injection via attributeRefs"() {@@ -379,4 +400,60 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer}}+ private static void setDatabaseSessionTimeZone(PersistenceService persistenceService, String timeZone, int connectionCount) {+ def connectionProvider = getConnectionProvider(persistenceService)+ def connections = []+ try {+ connectionCount.times {+ connections.add(connectionProvider.getConnection())+ }+ connections.each { connection ->+ def statement = connection.createStatement()+ try {+ statement.execute("set time zone '${timeZone}'")+ } finally {+ statement.close()+ }+ }+ } finally {+ connections.each { connection ->+ connectionProvider.closeConnection(connection)+ }+ }+ }++ private static String getDatabaseSessionTimeZone(PersistenceService persistenceService) {+ withConnection(persistenceService) { connection ->+ def statement = connection.createStatement()+ try {+ def resultSet = statement.executeQuery("select current_setting('TimeZone')")+ try {+ resultSet.next()+ return resultSet.getString(1)+ } finally {+ resultSet.close()+ }+ } finally {+ statement.close()+ }+ }+ }++ private static <T> T withConnection(PersistenceService persistenceService, Closure<T> closure) {+ def connectionProvider = getConnectionProvider(persistenceService)+ def connection = connectionProvider.getConnection()+ try {+ return closure.call(connection)+ } finally {+ connectionProvider.closeConnection(connection)+ }+ }++ private static ConnectionProvider getConnectionProvider(PersistenceService persistenceService) {+ return persistenceService.entityManagerFactory+ .unwrap(SessionFactoryImplementor.class)+ .serviceRegistry+ .requireService(ConnectionProvider.class)+ }+}