Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Fix timezone handling for datapoints export (#2882)

Eric Bariaux5b77bef9ea87
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 8dec9776e..828834010 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -31,6 +31,7 @@ import java.sql.Statement;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
+import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -253,13 +254,19 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
try {
persistenceService.doTransaction(em -> em.unwrap(Session.class).doWork(connection -> {
String tempTableName = "tmp_export_attributes";
+ String tempBoundsTableName = "tmp_export_bounds";
- // Create temp table and insert attribute refs safely via PreparedStatement
- // to prevent SQL injection (instead of string-interpolating IDs into the query)
+ // Create temp tables and insert parameters safely via PreparedStatement
+ // to prevent SQL injection (instead of string-interpolating IDs into the query).
try (Statement statement = connection.createStatement()) {
+ // We have 2 different tables because for attribute_name, we have one entry per exported attribute
statement.execute(
"create temp table " + tempTableName + " (entity_id text, attribute_name text) on commit drop"
);
+ // but for timestamps, we always have only one row, as the export period is the same for all attributes
+ statement.execute(
+ "create temp table " + tempBoundsTableName + " (from_timestamp timestamp, to_timestamp timestamp) on commit drop"
+ );
}
try (PreparedStatement insertStatement = connection.prepareStatement(
@@ -273,8 +280,16 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
insertStatement.executeBatch();
}
+ try (PreparedStatement boundsStatement = connection.prepareStatement(
+ "insert into " + tempBoundsTableName + " (from_timestamp, to_timestamp) values (?, ?)"
+ )) {
+ boundsStatement.setObject(1, toDatapointLocalDateTime(fromTimestamp));
+ boundsStatement.setObject(2, toDatapointLocalDateTime(toTimestamp));
+ boundsStatement.executeUpdate();
+ }
+
// Build the COPY ... TO STDOUT query based on format
- String copyQuery = buildCopyToStdoutQuery(tempTableName, fromTimestamp, toTimestamp, format, attributeRefs);
+ String copyQuery = buildCopyToStdoutQuery(tempTableName, tempBoundsTableName, format, attributeRefs);
try {
PGConnection pgConnection = connection.unwrap(PGConnection.class);
@@ -311,10 +326,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
* Builds a COPY ... TO STDOUT query for streaming CSV data based on the export format.
* Uses a temporary table for safe attribute filtering (SQL injection prevention).
*/
- private String buildCopyToStdoutQuery(String tempTableName, long fromTimestamp, long toTimestamp,
+ private String buildCopyToStdoutQuery(String tempTableName, String tempBoundsTableName,
DatapointExportFormat format, AttributeRef[] attributeRefs) {
- long fromSeconds = fromTimestamp / 1000;
- long toSeconds = toTimestamp / 1000;
final String TO_STDOUT_WITH_CSV_FORMAT = ") TO STDOUT WITH (FORMAT CSV, HEADER, DELIMITER ',');";
if (format == DatapointExportFormat.CSV_CROSSTAB || format == DatapointExportFormat.CSV_CROSSTAB_MINUTE) {
@@ -345,19 +358,21 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
"from asset_datapoint ad " +
"join asset a on ad.entity_id = a.id " +
"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +
- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " +
+ "cross join %s b " +
+ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +
"group by bucket_timestamp, header " +
"order by bucket_timestamp, header",
- tempTableName, fromSeconds, toSeconds);
+ tempTableName, tempBoundsTableName);
} else {
innerQuery = String.format(
"select ad.timestamp, a.name || '' : '' || ad.attribute_name as header, ad.value " +
"from asset_datapoint ad " +
"join asset a on ad.entity_id = a.id " +
"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +
- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " +
+ "cross join %s b " +
+ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +
"order by ad.timestamp, header",
- tempTableName, fromSeconds, toSeconds);
+ tempTableName, tempBoundsTableName);
}
return String.format(
@@ -370,9 +385,10 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
"from asset_datapoint ad " +
"join asset a on ad.entity_id = a.id " +
"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +
- "where ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) " +
+ "cross join %s b " +
+ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +
"order by ad.timestamp desc",
- tempTableName, fromSeconds, toSeconds);
+ tempTableName, tempBoundsTableName);
return "copy (" + innerQuery + TO_STDOUT_WITH_CSV_FORMAT;
}
}
@@ -399,11 +415,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
* It returns a structure containing the parametrized query as a string and the parameter values.
*/
protected ExportQuery getSelectExportQuery(AttributeRef[] attributeRefs, long fromTimestamp, long toTimestamp) {
- StringBuilder sb = new StringBuilder("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(?) and ad.timestamp <= to_timestamp(?) and ");
+ StringBuilder sb = new StringBuilder("select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= ? and ad.timestamp <= ? and ");
Map<Integer, Object> parameters = new HashMap<>();
int paramIndex = 1;
- parameters.put(paramIndex++, fromTimestamp / 1000);
- parameters.put(paramIndex++, toTimestamp / 1000);
+ parameters.put(paramIndex++, toDatapointLocalDateTime(fromTimestamp));
+ parameters.put(paramIndex++, toDatapointLocalDateTime(toTimestamp));
sb.append("(ad.entity_id, ad.attribute_name) in (");
for (int i = 0; i < attributeRefs.length; i++) {
@@ -429,4 +445,10 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
}
return assetId;
}
+
+ private static LocalDateTime toDatapointLocalDateTime(long timestamp) {
+ // Datapoints are stored in a timestamp-without-time-zone column using this same JVM-local conversion.
+ // Keep export bounds in that representation so PostgreSQL's session timezone does not affect filtering.
+ return Instant.ofEpochMilli(timestamp).atZone(ZoneId.systemDefault()).toLocalDateTime();
+ }
}
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
index c634bb062..4955536c3 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy
@@ -1,5 +1,8 @@
package org.openremote.test.assets
+import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider
+import org.hibernate.engine.spi.SessionFactoryImplementor
+import org.openremote.container.persistence.PersistenceService
import org.openremote.manager.asset.AssetStorageService
import org.openremote.manager.datapoint.AssetDatapointService
import org.openremote.manager.setup.SetupService
@@ -27,18 +30,31 @@ import static org.openremote.model.value.ValueType.NUMBER
class AssetDatapointExportTest extends Specification implements ManagerContainerTrait {
+ // Important for test is that we use a different TZ for DB than for JVM
+ private static final String EXPORT_TEST_DATABASE_TIME_ZONE = ZoneId.systemDefault().id == "Europe/Amsterdam" ? "UTC" : "Europe/Amsterdam"
+ private static final int EXPORT_TEST_DATABASE_POOL_SIZE = 5
+
def "Test CSV export functionality for asset data points"() {
given: "expected conditions"
def conditions = new PollingConditions(timeout: 10, delay: 0.2)
- and: "the container is started"
- def container = startContainer(defaultConfig(), defaultServices())
+ and: "the container is started with a bounded DB connection pool"
+ def config = defaultConfig()
+ config.put(PersistenceService.OR_DB_POOL_MIN_SIZE, Integer.toString(EXPORT_TEST_DATABASE_POOL_SIZE))
+ config.put(PersistenceService.OR_DB_POOL_MAX_SIZE, Integer.toString(EXPORT_TEST_DATABASE_POOL_SIZE))
+ def container = startContainer(config, defaultServices())
def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
def assetStorageService = container.getService(AssetStorageService.class)
def assetDatapointService = container.getService(AssetDatapointService.class)
+ def persistenceService = container.getService(PersistenceService.class)
+ setDatabaseSessionTimeZone(persistenceService, EXPORT_TEST_DATABASE_TIME_ZONE, EXPORT_TEST_DATABASE_POOL_SIZE)
assetDatapointService.datapointExportLimit = 1000
+ and: "the database session timezone is different from the JVM timezone"
+ assert getDatabaseSessionTimeZone(persistenceService) == EXPORT_TEST_DATABASE_TIME_ZONE
+ assert ZoneId.systemDefault() != ZoneId.of(EXPORT_TEST_DATABASE_TIME_ZONE)
+
when: "requesting the first light asset in City realm"
def asset = assetStorageService.find(
new AssetQuery()
@@ -159,7 +175,12 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer
/* ------------------------- */
cleanup: "Remove the limit on datapoint querying"
- assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT;
+ if (assetDatapointService != null) {
+ assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT
+ }
+ if (persistenceService != null) {
+ setDatabaseSessionTimeZone(persistenceService, "UTC", EXPORT_TEST_DATABASE_POOL_SIZE)
+ }
}
def "Export query is not vulnerable to SQL injection via attributeRefs"() {
@@ -379,4 +400,60 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer
}
}
+ private static void setDatabaseSessionTimeZone(PersistenceService persistenceService, String timeZone, int connectionCount) {
+ def connectionProvider = getConnectionProvider(persistenceService)
+ def connections = []
+ try {
+ connectionCount.times {
+ connections.add(connectionProvider.getConnection())
+ }
+ connections.each { connection ->
+ def statement = connection.createStatement()
+ try {
+ statement.execute("set time zone '${timeZone}'")
+ } finally {
+ statement.close()
+ }
+ }
+ } finally {
+ connections.each { connection ->
+ connectionProvider.closeConnection(connection)
+ }
+ }
+ }
+
+ private static String getDatabaseSessionTimeZone(PersistenceService persistenceService) {
+ withConnection(persistenceService) { connection ->
+ def statement = connection.createStatement()
+ try {
+ def resultSet = statement.executeQuery("select current_setting('TimeZone')")
+ try {
+ resultSet.next()
+ return resultSet.getString(1)
+ } finally {
+ resultSet.close()
+ }
+ } finally {
+ statement.close()
+ }
+ }
+ }
+
+ private static <T> T withConnection(PersistenceService persistenceService, Closure<T> closure) {
+ def connectionProvider = getConnectionProvider(persistenceService)
+ def connection = connectionProvider.getConnection()
+ try {
+ return closure.call(connection)
+ } finally {
+ connectionProvider.closeConnection(connection)
+ }
+ }
+
+ private static ConnectionProvider getConnectionProvider(PersistenceService persistenceService) {
+ return persistenceService.entityManagerFactory
+ .unwrap(SessionFactoryImplementor.class)
+ .serviceRegistry
+ .requireService(ConnectionProvider.class)
+ }
+
}