100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Improved asset datapoint export
Rich Turner1f3a38aab1e0
diff --git a/docker-compose.yml b/docker-compose.ymlindex 6af5abb0e..56f4bbea8 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -10,6 +10,9 @@#version: '2.4'+volumes:+ temp-data:+services:proxy:@@ -26,6 +29,8 @@ services:postgresql:restart: alwaysimage: openremote/postgresql:${POSTGRESQL_VERSION:-9.6.21.0}+ volumes:+ - temp-data:/tmpkeycloak:restart: always@@ -46,6 +51,8 @@ services:environment:DEV_MODE: ${DEV_MODE:-false}SETUP_ADMIN_PASSWORD: ${SETUP_ADMIN_PASSWORD:-secret}+ volumes:+ - temp-data:/tmp# devices:# - /dev/ttyACM0:/dev/ttyS0diff --git a/manager/Dockerfile b/manager/Dockerfileindex 9fc1964c6..a22328dd3 100644--- a/manager/Dockerfile+++ b/manager/Dockerfile@@ -64,6 +64,7 @@ ENV PROVISIONING_DOCROOT ${PROVISIONING_DOCROOT:-/deployment/manager/provisioninENV ROOT_REDIRECT_PATH ${ROOT_REDIRECT_PATH:-/manager}ENV MAP_TILES_PATH ${MAP_TILES_PATH:-/deployment/map/mapdata.mbtiles}ENV MAP_SETTINGS_PATH ${MAP_SETTINGS_PATH:-/deployment/map/mapsettings.json}+ENV DATA_POINTS_EXPORT_DIR ${DATA_POINTS_EXPORT_DIR:-/tmp}ENV LOGGING_CONFIG_FILE ${LOGGING_CONFIG_FILE}ENV MAP_TILESERVER_HOST ${MAP_TILESERVER_HOST}ENV MAP_TILESERVER_PORT ${MAP_TILESERVER_PORT:-8082}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javaindex 9684f05f8..8baf7fc26 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -5,6 +5,7 @@ import org.hibernate.Session;import org.hibernate.jdbc.AbstractReturningWork;import org.openremote.container.persistence.PersistenceService;import org.openremote.container.timer.TimerService;+import org.openremote.container.util.UniqueIdentifierGenerator;import org.openremote.manager.asset.AssetStorageService;import org.openremote.model.Container;import org.openremote.model.ContainerService;@@ -22,6 +23,7 @@ import org.postgresql.util.PGobject;import javax.persistence.TypedQuery;import java.io.File;+import java.nio.file.Paths;import java.sql.Connection;import java.sql.PreparedStatement;import java.sql.ResultSet;@@ -54,8 +56,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements Cprotected ScheduledExecutorService executorService;protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;- private String managerFilePath;-@Overridepublic int getPriority() {return PRIORITY;@@ -67,8 +67,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements CassetStorageService = container.getService(AssetStorageService.class);timerService = container.getService(TimerService.class);executorService = container.getExecutorService();-- managerFilePath = container.isDevMode() ? "./postgresql" : "/postgresql";}@Override@@ -366,7 +364,7 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements Cif (rs.next()) {return new DatapointPeriod(rs.getString(1), rs.getString(2), rs.getTimestamp(3).getTime(), rs.getTimestamp(4).getTime());}- return null;+ return new DatapointPeriod(assetId, attributeName, null, null);}}}@@ -374,27 +372,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C);}- public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,- long fromTimestamp,- long toTimestamp) {- return executorService.schedule(() -> {- int count = attributeRefs.length;- LocalDateTime fromInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault());- LocalDateTime toInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault());- String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());-- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));-- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));-- sb.append(String.format(")) to '/var/lib/postgresql/%s' delimiter ',' CSV HEADER;", fileName));-- persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());-- return new File(String.format("%s/%s", managerFilePath, fileName));- }, 0, TimeUnit.MILLISECONDS);- }-protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " +"VALUES (?, ?, ?, ?) " +diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.javaindex 923e5bb72..b9133bca4 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java@@ -41,6 +41,7 @@ import javax.ws.rs.WebApplicationException;import javax.ws.rs.container.AsyncResponse;import javax.ws.rs.container.ConnectionCallback;import javax.ws.rs.core.HttpHeaders;+import javax.ws.rs.core.MediaType;import javax.ws.rs.core.Response;import java.io.*;import java.time.Instant;@@ -161,7 +162,7 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements Asthrow new WebApplicationException(Response.Status.FORBIDDEN);}- Attribute<?> attribute = asset.getAttribute(attributeRef.getName()).orElseThrow(() ->+ asset.getAttribute(attributeRef.getName()).orElseThrow(() ->new WebApplicationException(Response.Status.NOT_FOUND));}@@ -170,25 +171,37 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements AsasyncResponse.register((ConnectionCallback) disconnected -> {exportFuture.cancel(true);});++ File exportFile = null;+try {- File file = exportFuture.get();- response.setContentType("text/csv");- response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());+ exportFile = exportFuture.get();+ response.setContentType(MediaType.APPLICATION_OCTET_STREAM);+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + exportFile.getName());OutputStream out = response.getOutputStream();-- FileInputStream in = new FileInputStream(file);+ FileInputStream in = new FileInputStream(exportFile);IOUtils.copy(in, out);-out.close();in.close();- file.delete();- asyncResponse.resume(response);+ asyncResponse.resume(+ Response.ok(exportFile)+ .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"dataexport.csv\"" )+ .build()+ );} catch (Exception ex) {exportFuture.cancel(true);asyncResponse.resume(new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR));LOG.log(Level.WARNING, "Exception in ScheduledFuture: ", ex);+ } finally {+ if (exportFile != null && exportFile.exists()) {+ try {+ exportFile.delete();+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Failed to delete temporary export file: " + exportFile.getPath(), e);+ }+ }}} catch (JsonProcessingException ex) {asyncResponse.resume(new BadRequestException(ex));diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex ebf09c469..e96c00be4 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -1,6 +1,7 @@package org.openremote.manager.datapoint;import org.openremote.container.timer.TimerService;+import org.openremote.container.util.UniqueIdentifierGenerator;import org.openremote.manager.asset.AssetProcessingException;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.asset.AssetUpdateProcessor;@@ -10,6 +11,7 @@ import org.openremote.model.Container;import org.openremote.model.asset.Asset;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent.Source;+import org.openremote.model.attribute.AttributeRef;import org.openremote.model.attribute.AttributeWriteFailure;import org.openremote.model.datapoint.AssetDatapoint;import org.openremote.model.query.AssetQuery;@@ -19,13 +21,18 @@ import org.openremote.model.util.Pair;import org.openremote.model.value.MetaItemType;import javax.persistence.EntityManager;+import java.io.File;+import java.nio.file.Path;+import java.nio.file.Paths;import java.sql.Date;import java.time.Duration;import java.time.Instant;import java.time.LocalDateTime;import java.time.ZoneId;+import java.util.Arrays;import java.util.List;import java.util.Map;+import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;@@ -35,6 +42,7 @@ import static java.time.temporal.ChronoUnit.DAYS;import static java.util.stream.Collectors.groupingBy;import static java.util.stream.Collectors.toList;import static org.openremote.container.util.MapAccess.getInteger;+import static org.openremote.container.util.MapAccess.getString;import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;/**@@ -47,8 +55,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoipublic static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";public static final int DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;+ public static final String DATA_POINTS_EXPORT_DIR = "DATA_POINTS_EXPORT_DIR";+ public static final String DATA_POINTS_EXPORT_DIR_DEFAULT = "profile/tmp";private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());protected int maxDatapointAgeDays;+ protected Path exportPath;@Overridepublic void init(Container container) throws Exception {@@ -68,6 +79,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiif (maxDatapointAgeDays <= 0) {LOG.warning(DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");}++ exportPath = Paths.get(getString(container.getConfig(), DATA_POINTS_EXPORT_DIR, DATA_POINTS_EXPORT_DIR_DEFAULT));}@Override@@ -186,4 +199,22 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoireturn " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";}++ public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,+ long fromTimestamp,+ long toTimestamp) {+ return executorService.schedule(() -> {+ String fileName = UniqueIdentifierGenerator.generateId() + ".csv";+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));++ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));++ sb.append(String.format(")) to '/tmp/%s' delimiter ',' CSV HEADER;", fileName));++ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());++ return exportPath.resolve(fileName).toFile();+ }, 0, TimeUnit.MILLISECONDS);+ }+}diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java b/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.javaindex 751f4b306..203c143de 100644--- a/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java@@ -9,9 +9,9 @@ public class DatapointPeriod {protected String attributeName;- protected long oldest;+ protected Long oldest;- protected long latest;+ protected Long latest;protected DatapointPeriod() {}@@ -19,8 +19,8 @@ public class DatapointPeriod {@JsonCreatorpublic DatapointPeriod(@JsonProperty("assetId") String assetId,@JsonProperty("attributeName") String attributeName,- @JsonProperty("oldestTimestamp") long oldest,- @JsonProperty("latestTimestamp") long latest) {+ @JsonProperty("oldestTimestamp") Long oldest,+ @JsonProperty("latestTimestamp") Long latest) {this.assetId = assetId;this.attributeName = attributeName;this.oldest = oldest;@@ -38,12 +38,12 @@ public class DatapointPeriod {}@JsonProperty("oldestTimestamp")- public long getOldest() {+ public Long getOldest() {return oldest;}@JsonProperty("latestTimestamp")- public long getLatest() {+ public Long getLatest() {return latest;}}diff --git a/profile/basic-identity.yml b/profile/basic-identity.ymlindex c3556a79b..4c8ede7a2 100644--- a/profile/basic-identity.yml+++ b/profile/basic-identity.yml@@ -12,6 +12,7 @@version: '2.4'volumes:+ temp-data:postgresql-data:services:@@ -30,6 +31,7 @@ services:service: postgresqlvolumes:- postgresql-data:/var/lib/postgresql/data+ - temp-data:/tmpmanager:extends:@@ -38,5 +40,7 @@ services:depends_on:postgresql:condition: service_healthy+ volumes:+ - temp-data:/tmpenvironment:IDENTITY_PROVIDER: basicdiff --git a/profile/deploy.yml b/profile/deploy.ymlindex effb41aab..694ac78e8 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -23,9 +23,9 @@#version: '2.4'-# If you enable volume mappings in services, you need this. Otherwise, remove the volumesvolumes:- postgresql-data:+ postgresql-data: # Needed if you want to persist postgres data outside of container+ temp-data: # Needed to share temp export data between postgres and manager containersservices:@@ -105,7 +105,8 @@ services:# /deployment/manager/provisioning - contains JSON files for provisioning the system# /deployment/map/mapdata.mbtiles - map tile data# /deployment/map/mapsettings.json - map styling- #volumes:+ volumes:+ - temp-data:/tmp# - deployment-data:/deployment# - postgresql-data:/postgres# Expose any devices required by agents@@ -307,7 +308,8 @@ services:image: openremote/postgresql:${POSTGRESQL_VERSION:-9.6.21.0}restart: always# Docker volume for persistent storage of DB; can alternatively map to host- #volumes:+ volumes:+ - temp-data:/tmp# - postgresql-data:/var/lib/postgresql/data#ports:# - "5432:5432" # DB serverdiff --git a/profile/dev-testing.yml b/profile/dev-testing.ymlindex 42752d9ae..b45a3dbc9 100644--- a/profile/dev-testing.yml+++ b/profile/dev-testing.yml@@ -31,7 +31,7 @@ services:file: deploy.ymlservice: postgresqlvolumes:- - ./postgresql:/var/lib/postgresql+ - ./tmp:/tmp# Access directly if needed on localhostports:- "5432:5432"