100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Merge commit from fork
Eric Bariaux02ac83074b81
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 828834010..06f9833d5 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -26,6 +26,7 @@ import org.postgresql.copy.CopyManager;import java.io.IOException;import java.io.PipedInputStream;import java.io.PipedOutputStream;+import java.nio.charset.StandardCharsets;import java.sql.PreparedStatement;import java.sql.Statement;import java.sql.Timestamp;@@ -255,13 +256,14 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoipersistenceService.doTransaction(em -> em.unwrap(Session.class).doWork(connection -> {String tempTableName = "tmp_export_attributes";String tempBoundsTableName = "tmp_export_bounds";+ List<ExportColumn> exportColumns = getExportColumns(attributeRefs);// Create temp tables and insert parameters safely via PreparedStatement// to prevent SQL injection (instead of string-interpolating IDs into the query).try (Statement statement = connection.createStatement()) {// We have 2 different tables because for attribute_name, we have one entry per exported attributestatement.execute(- "create temp table " + tempTableName + " (entity_id text, attribute_name text) on commit drop"+ "create temp table " + tempTableName + " (ordinal integer, entity_id text, attribute_name text, column_key text) on commit drop");// but for timestamps, we always have only one row, as the export period is the same for all attributesstatement.execute(@@ -270,11 +272,14 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi}try (PreparedStatement insertStatement = connection.prepareStatement(- "insert into " + tempTableName + " (entity_id, attribute_name) values (?, ?)"+ "insert into " + tempTableName + " (ordinal, entity_id, attribute_name, column_key) values (?, ?, ?, ?)")) {- for (AttributeRef attributeRef : attributeRefs) {- insertStatement.setString(1, validateAssetId(attributeRef.getId()));- insertStatement.setString(2, attributeRef.getName());+ for (ExportColumn exportColumn : exportColumns) {+ AttributeRef attributeRef = exportColumn.attributeRef();+ insertStatement.setInt(1, exportColumn.ordinal());+ insertStatement.setString(2, validateAssetId(attributeRef.getId()));+ insertStatement.setString(3, attributeRef.getName());+ insertStatement.setString(4, exportColumn.columnKey());insertStatement.addBatch();}insertStatement.executeBatch();@@ -289,11 +294,14 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi}// Build the COPY ... TO STDOUT query based on format- String copyQuery = buildCopyToStdoutQuery(tempTableName, tempBoundsTableName, format, attributeRefs);+ String copyQuery = buildCopyToStdoutQuery(tempTableName, tempBoundsTableName, format, exportColumns);try {PGConnection pgConnection = connection.unwrap(PGConnection.class);CopyManager copyManager = pgConnection.getCopyAPI();+ if (isCrosstabFormat(format)) {+ out.write(buildCrosstabCsvHeader(exportColumns).getBytes(StandardCharsets.UTF_8));+ }copyManager.copyOut(copyQuery, out);out.flush();} catch (IOException e) {@@ -327,21 +335,16 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi* Uses a temporary table for safe attribute filtering (SQL injection prevention).*/private String buildCopyToStdoutQuery(String tempTableName, String tempBoundsTableName,- DatapointExportFormat format, AttributeRef[] attributeRefs) {- final String TO_STDOUT_WITH_CSV_FORMAT = ") TO STDOUT WITH (FORMAT CSV, HEADER, DELIMITER ',');";-- if (format == DatapointExportFormat.CSV_CROSSTAB || format == DatapointExportFormat.CSV_CROSSTAB_MINUTE) {- Set<String> headers = getAttributeHeaders(attributeRefs);+ DatapointExportFormat format, List<ExportColumn> exportColumns) {+ final String TO_STDOUT_WITH_CSV_HEADER_FORMAT = ") TO STDOUT WITH (FORMAT CSV, HEADER, DELIMITER ',');";+ final String TO_STDOUT_WITH_CSV_FORMAT = ") TO STDOUT WITH (FORMAT CSV, DELIMITER ',');";- // Category query using VALUES clause with dollar-quoting to avoid escaping issues- String categoryValues = headers.stream()- .map(header -> "('" + header.replace("'", "''") + "')")- .collect(Collectors.joining(", "));- String categoryQuery = "SELECT header FROM (VALUES " + categoryValues + ") AS t(header)";+ if (isCrosstabFormat(format)) {+ String categoryQuery = "select column_key from " + tempTableName + " order by ordinal";- // Column definitions for crosstab result- String attributeColumns = headers.stream()- .map(header -> "\"" + header + "\" text")+ // Column definitions for crosstab result use generated identifiers only.+ String attributeColumns = exportColumns.stream()+ .map(exportColumn -> exportColumn.columnKey() + " text").collect(Collectors.joining(", "));// Inner SELECT query for crosstab (uses escaped quotes since it's inside a string argument)@@ -349,34 +352,32 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiif (format == DatapointExportFormat.CSV_CROSSTAB_MINUTE) {innerQuery = String.format("select public.time_bucket(''1 minute'', ad.timestamp) as bucket_timestamp, " +- "a.name || '' : '' || ad.attribute_name as header, " ++ "t.column_key as category, " +"CASE " +" WHEN jsonb_typeof((array_agg(ad.value))[1]) = ''number'' THEN " +" round(avg((ad.value#>>''{}'')::numeric) FILTER (WHERE jsonb_typeof(ad.value) = ''number''), 3)::text " +" ELSE (array_agg(ad.value ORDER BY ad.timestamp DESC) FILTER (WHERE jsonb_typeof(ad.value) != ''number''))[1]#>>''{}''" +"END as value " +"from asset_datapoint ad " +- "join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +"cross join %s b " +"where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +- "group by bucket_timestamp, header " +- "order by bucket_timestamp, header",+ "group by bucket_timestamp, category " ++ "order by bucket_timestamp, category",tempTableName, tempBoundsTableName);} else {innerQuery = String.format(- "select ad.timestamp, a.name || '' : '' || ad.attribute_name as header, ad.value " ++ "select ad.timestamp, t.column_key as category, ad.value " +"from asset_datapoint ad " +- "join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +"cross join %s b " +"where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +- "order by ad.timestamp, header",+ "order by ad.timestamp, category",tempTableName, tempBoundsTableName);}return String.format(- "copy (select * from crosstab('%s', $cat$%s$cat$) as ct(timestamp timestamp, %s)%s",+ "copy (select * from crosstab('%s', '%s') as ct(timestamp timestamp, %s)%s",innerQuery, categoryQuery, attributeColumns, TO_STDOUT_WITH_CSV_FORMAT);} else {// Plain CSV format using temp table join@@ -385,26 +386,70 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi"from asset_datapoint ad " +"join asset a on ad.entity_id = a.id " +"join %s t on ad.entity_id = t.entity_id and ad.attribute_name = t.attribute_name " +- "cross join %s b " +- "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " +- "order by ad.timestamp desc",- tempTableName, tempBoundsTableName);- return "copy (" + innerQuery + TO_STDOUT_WITH_CSV_FORMAT;+ "cross join %s b " ++ "where ad.timestamp >= b.from_timestamp and ad.timestamp <= b.to_timestamp " ++ "order by ad.timestamp desc",+ tempTableName, tempBoundsTableName);+ return "copy (" + innerQuery + TO_STDOUT_WITH_CSV_HEADER_FORMAT;}}/**- * Function for building CSV attribute headers when format will make- * a separate column per attribute.- * Uses TreeSet for alphabetical ordering required by crosstab queries.+ * Function for building crosstab columns when format will make a separate column per attribute.+ * Uses display-label ordering to preserve existing crosstab output ordering, but keeps those labels+ * out of SQL syntax by assigning generated column keys.*/- private Set<String> getAttributeHeaders(AttributeRef[] attributeRefs) {- Set<String> headers = new TreeSet<>();- Arrays.stream(attributeRefs).forEach(attr -> {- String assetName = assetStorageService.findNames(attr.getId()).toString().replaceAll("(^\\[|\\]$)", "");- headers.add(assetName + " : " + attr.getName());- });- return headers;+ private List<ExportColumn> getExportColumns(AttributeRef[] attributeRefs) {+ List<ExportColumn> sortedColumns = Arrays.stream(attributeRefs)+ .map(attr -> new ExportColumn(-1, attr, getDisplayHeader(attr), null))+ .sorted(Comparator+ .comparing(ExportColumn::displayHeader)+ .thenComparing(exportColumn -> exportColumn.attributeRef().getId())+ .thenComparing(exportColumn -> exportColumn.attributeRef().getName()))+ .collect(Collectors.toList());++ List<ExportColumn> exportColumns = new ArrayList<>(sortedColumns.size());+ for (int i = 0; i < sortedColumns.size(); i++) {+ ExportColumn exportColumn = sortedColumns.get(i);+ exportColumns.add(new ExportColumn(+ i,+ exportColumn.attributeRef(),+ exportColumn.displayHeader(),+ "col_" + i+ ));+ }+ return exportColumns;+ }++ private String getDisplayHeader(AttributeRef attributeRef) {+ List<String> assetNames = assetStorageService.findNames(attributeRef.getId());+ String assetName = assetNames.isEmpty() ? "" : assetNames.get(0);+ return assetName + " : " + attributeRef.getName();+ }++ private static String buildCrosstabCsvHeader(List<ExportColumn> exportColumns) {+ List<String> headers = new ArrayList<>(exportColumns.size() + 1);+ headers.add("timestamp");+ exportColumns.stream()+ .map(ExportColumn::displayHeader)+ .forEach(headers::add);+ return headers.stream()+ .map(AssetDatapointService::escapeCsvField)+ .collect(Collectors.joining(",")) + "\n";+ }++ private static String escapeCsvField(String field) {+ if (field == null) {+ return "";+ }+ if (field.indexOf(',') < 0 && field.indexOf('"') < 0 && field.indexOf('\n') < 0 && field.indexOf('\r') < 0) {+ return field;+ }+ return "\"" + field.replace("\"", "\"\"") + "\"";+ }++ private static boolean isCrosstabFormat(DatapointExportFormat format) {+ return format == DatapointExportFormat.CSV_CROSSTAB || format == DatapointExportFormat.CSV_CROSSTAB_MINUTE;}/**@@ -439,6 +484,9 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiprotected record ExportQuery(String query, Map<Integer, Object> parameters) {}+ private record ExportColumn(int ordinal, AttributeRef attributeRef, String displayHeader, String columnKey) {+ }+private static String validateAssetId(String assetId) {if (assetId == null || !Asset.matchesAssetIdPattern(assetId)) {throw new IllegalArgumentException("Invalid asset id");diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovyindex 4955536c3..49119dc2c 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointExportTest.groovy@@ -222,6 +222,68 @@ class AssetDatapointExportTest extends Specification implements ManagerContainerassert csvExportLines.size() == 1}+ def "Crosstab exports treat asset names with SQL delimiters as CSV labels"() {++ given: "the container is started"+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def assetStorageService = container.getService(AssetStorageService.class)+ def assetDatapointService = container.getService(AssetDatapointService.class)+ assetDatapointService.datapointExportLimit = 10000++ and: "ensure there are no datapoints"+ assetDatapointService.purgeDataPoints()++ and: "an asset name contains SQL delimiter syntax"+ def asset = assetStorageService.find(+ new AssetQuery()+ .types(LightAsset.class)+ .realm(new RealmPredicate(keycloakTestSetup.realmCity.name))+ .names("Light 1")+ )+ assert asset != null+ def originalName = asset.name+ def maliciousAssetName = 'SQL "delimiter" $cat$ -- label'+ def attributeName = "brightness"+ asset.name = maliciousAssetName+ asset = assetStorageService.merge(asset)++ and: "the asset has one datapoint for a real exported attribute"+ def dateTime = LocalDateTime.now()+ assetDatapointService.upsertValues(+ asset.getId(),+ attributeName,+ [new ValueDatapoint<>(dateTime.minusMinutes(1).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(), 42d)]+ )++ expect: "both crosstab formats export the datapoint without treating the asset name as SQL"+ [DatapointExportFormat.CSV_CROSSTAB, DatapointExportFormat.CSV_CROSSTAB_MINUTE].each { format ->+ def csvExport = assetDatapointService.exportDatapoints(+ [new AttributeRef(asset.id, attributeName)] as AttributeRef[],+ dateTime.minusMinutes(5).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),+ dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(),+ format+ ).getText(StandardCharsets.UTF_8.name())+ def csvExportLines = csvExport.readLines()++ assert csvExportLines.size() == 2+ assert parseCsvLine(csvExportLines[0]) == ["timestamp", maliciousAssetName + " : " + attributeName]+ assert csvExportLines[1].contains("42")+ }++ cleanup: "restore the fixture asset name"+ if (assetStorageService != null && asset != null && originalName != null) {+ def currentAsset = assetStorageService.find(asset.id, true)+ if (currentAsset != null) {+ currentAsset.name = originalName+ assetStorageService.merge(currentAsset)+ }+ }+ if (assetDatapointService != null) {+ assetDatapointService.datapointExportLimit = assetDatapointService.OR_DATA_POINTS_EXPORT_LIMIT_DEFAULT+ }+ }+def "Export query is not vulnerable to SQL injection via REST API"() {given: "the container is started"@@ -400,6 +462,38 @@ class AssetDatapointExportTest extends Specification implements ManagerContainer}}+ private static List<String> parseCsvLine(String line) {+ def fields = []+ def current = new StringBuilder()+ boolean quoted = false++ for (int i = 0; i < line.length(); i++) {+ char c = line.charAt(i)+ if (quoted) {+ if (c == '"') {+ if (i + 1 < line.length() && line.charAt(i + 1) == '"') {+ current.append('"')+ i+++ } else {+ quoted = false+ }+ } else {+ current.append(c)+ }+ } else if (c == '"') {+ quoted = true+ } else if (c == ',') {+ fields.add(current.toString())+ current.setLength(0)+ } else {+ current.append(c)+ }+ }++ fields.add(current.toString())+ return fields+ }+private static void setDatabaseSessionTimeZone(PersistenceService persistenceService, String timeZone, int connectionCount) {def connectionProvider = getConnectionProvider(persistenceService)def connections = []