Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Improved asset datapoint export

Rich Turner1f3a38aab1e0
diff --git a/docker-compose.yml b/docker-compose.yml
index 6af5abb0e..56f4bbea8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,6 +10,9 @@
#
version: '2.4'
+volumes:
+ temp-data:
+
services:
proxy:
@@ -26,6 +29,8 @@ services:
postgresql:
restart: always
image: openremote/postgresql:${POSTGRESQL_VERSION:-9.6.21.0}
+ volumes:
+ - temp-data:/tmp
keycloak:
restart: always
@@ -46,6 +51,8 @@ services:
environment:
DEV_MODE: ${DEV_MODE:-false}
SETUP_ADMIN_PASSWORD: ${SETUP_ADMIN_PASSWORD:-secret}
+ volumes:
+ - temp-data:/tmp
# devices:
# - /dev/ttyACM0:/dev/ttyS0
diff --git a/manager/Dockerfile b/manager/Dockerfile
index 9fc1964c6..a22328dd3 100644
--- a/manager/Dockerfile
+++ b/manager/Dockerfile
@@ -64,6 +64,7 @@ ENV PROVISIONING_DOCROOT ${PROVISIONING_DOCROOT:-/deployment/manager/provisionin
ENV ROOT_REDIRECT_PATH ${ROOT_REDIRECT_PATH:-/manager}
ENV MAP_TILES_PATH ${MAP_TILES_PATH:-/deployment/map/mapdata.mbtiles}
ENV MAP_SETTINGS_PATH ${MAP_SETTINGS_PATH:-/deployment/map/mapsettings.json}
+ENV DATA_POINTS_EXPORT_DIR ${DATA_POINTS_EXPORT_DIR:-/tmp}
ENV LOGGING_CONFIG_FILE ${LOGGING_CONFIG_FILE}
ENV MAP_TILESERVER_HOST ${MAP_TILESERVER_HOST}
ENV MAP_TILESERVER_PORT ${MAP_TILESERVER_PORT:-8082}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
index 9684f05f8..8baf7fc26 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
@@ -5,6 +5,7 @@ import org.hibernate.Session;
import org.hibernate.jdbc.AbstractReturningWork;
import org.openremote.container.persistence.PersistenceService;
import org.openremote.container.timer.TimerService;
+import org.openremote.container.util.UniqueIdentifierGenerator;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.model.Container;
import org.openremote.model.ContainerService;
@@ -22,6 +23,7 @@ import org.postgresql.util.PGobject;
import javax.persistence.TypedQuery;
import java.io.File;
+import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -54,8 +56,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
protected ScheduledExecutorService executorService;
protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
- private String managerFilePath;
-
@Override
public int getPriority() {
return PRIORITY;
@@ -67,8 +67,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
assetStorageService = container.getService(AssetStorageService.class);
timerService = container.getService(TimerService.class);
executorService = container.getExecutorService();
-
- managerFilePath = container.isDevMode() ? "./postgresql" : "/postgresql";
}
@Override
@@ -366,7 +364,7 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
if (rs.next()) {
return new DatapointPeriod(rs.getString(1), rs.getString(2), rs.getTimestamp(3).getTime(), rs.getTimestamp(4).getTime());
}
- return null;
+ return new DatapointPeriod(assetId, attributeName, null, null);
}
}
}
@@ -374,27 +372,6 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
);
}
- public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,
- long fromTimestamp,
- long toTimestamp) {
- return executorService.schedule(() -> {
- int count = attributeRefs.length;
- LocalDateTime fromInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault());
- LocalDateTime toInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault());
- String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());
-
- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));
-
- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
-
- sb.append(String.format(")) to '/var/lib/postgresql/%s' delimiter ',' CSV HEADER;", fileName));
-
- persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
-
- return new File(String.format("%s/%s", managerFilePath, fileName));
- }, 0, TimeUnit.MILLISECONDS);
- }
-
protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {
return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " +
"VALUES (?, ?, ?, ?) " +
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
index 923e5bb72..b9133bca4 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
@@ -41,6 +41,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.container.AsyncResponse;
import javax.ws.rs.container.ConnectionCallback;
import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.io.*;
import java.time.Instant;
@@ -161,7 +162,7 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
throw new WebApplicationException(Response.Status.FORBIDDEN);
}
- Attribute<?> attribute = asset.getAttribute(attributeRef.getName()).orElseThrow(() ->
+ asset.getAttribute(attributeRef.getName()).orElseThrow(() ->
new WebApplicationException(Response.Status.NOT_FOUND)
);
}
@@ -170,25 +171,37 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
asyncResponse.register((ConnectionCallback) disconnected -> {
exportFuture.cancel(true);
});
+
+ File exportFile = null;
+
try {
- File file = exportFuture.get();
- response.setContentType("text/csv");
- response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
+ exportFile = exportFuture.get();
+ response.setContentType(MediaType.APPLICATION_OCTET_STREAM);
+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + exportFile.getName());
OutputStream out = response.getOutputStream();
-
- FileInputStream in = new FileInputStream(file);
+ FileInputStream in = new FileInputStream(exportFile);
IOUtils.copy(in, out);
-
out.close();
in.close();
- file.delete();
- asyncResponse.resume(response);
+ asyncResponse.resume(
+ Response.ok(exportFile)
+ .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"dataexport.csv\"" )
+ .build()
+ );
} catch (Exception ex) {
exportFuture.cancel(true);
asyncResponse.resume(new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR));
LOG.log(Level.WARNING, "Exception in ScheduledFuture: ", ex);
+ } finally {
+ if (exportFile != null && exportFile.exists()) {
+ try {
+ exportFile.delete();
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to delete temporary export file: " + exportFile.getPath(), e);
+ }
+ }
}
} catch (JsonProcessingException ex) {
asyncResponse.resume(new BadRequestException(ex));
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index ebf09c469..e96c00be4 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -1,6 +1,7 @@
package org.openremote.manager.datapoint;
import org.openremote.container.timer.TimerService;
+import org.openremote.container.util.UniqueIdentifierGenerator;
import org.openremote.manager.asset.AssetProcessingException;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.asset.AssetUpdateProcessor;
@@ -10,6 +11,7 @@ import org.openremote.model.Container;
import org.openremote.model.asset.Asset;
import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeEvent.Source;
+import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.attribute.AttributeWriteFailure;
import org.openremote.model.datapoint.AssetDatapoint;
import org.openremote.model.query.AssetQuery;
@@ -19,13 +21,18 @@ import org.openremote.model.util.Pair;
import org.openremote.model.value.MetaItemType;
import javax.persistence.EntityManager;
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.sql.Date;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -35,6 +42,7 @@ import static java.time.temporal.ChronoUnit.DAYS;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static org.openremote.container.util.MapAccess.getInteger;
+import static org.openremote.container.util.MapAccess.getString;
import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;
/**
@@ -47,8 +55,11 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
public static final String DATA_POINTS_MAX_AGE_DAYS = "DATA_POINTS_MAX_AGE_DAYS";
public static final int DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;
+ public static final String DATA_POINTS_EXPORT_DIR = "DATA_POINTS_EXPORT_DIR";
+ public static final String DATA_POINTS_EXPORT_DIR_DEFAULT = "profile/tmp";
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
protected int maxDatapointAgeDays;
+ protected Path exportPath;
@Override
public void init(Container container) throws Exception {
@@ -68,6 +79,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
if (maxDatapointAgeDays <= 0) {
LOG.warning(DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");
}
+
+ exportPath = Paths.get(getString(container.getConfig(), DATA_POINTS_EXPORT_DIR, DATA_POINTS_EXPORT_DIR_DEFAULT));
}
@Override
@@ -186,4 +199,22 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";
}
+
+ public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,
+ long fromTimestamp,
+ long toTimestamp) {
+ return executorService.schedule(() -> {
+ String fileName = UniqueIdentifierGenerator.generateId() + ".csv";
+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));
+
+ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
+
+ sb.append(String.format(")) to '/tmp/%s' delimiter ',' CSV HEADER;", fileName));
+
+ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
+
+ return exportPath.resolve(fileName).toFile();
+ }, 0, TimeUnit.MILLISECONDS);
+ }
+
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java b/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java
index 751f4b306..203c143de 100644
--- a/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java
+++ b/model/src/main/java/org/openremote/model/datapoint/DatapointPeriod.java
@@ -9,9 +9,9 @@ public class DatapointPeriod {
protected String attributeName;
- protected long oldest;
+ protected Long oldest;
- protected long latest;
+ protected Long latest;
protected DatapointPeriod() {
}
@@ -19,8 +19,8 @@ public class DatapointPeriod {
@JsonCreator
public DatapointPeriod(@JsonProperty("assetId") String assetId,
@JsonProperty("attributeName") String attributeName,
- @JsonProperty("oldestTimestamp") long oldest,
- @JsonProperty("latestTimestamp") long latest) {
+ @JsonProperty("oldestTimestamp") Long oldest,
+ @JsonProperty("latestTimestamp") Long latest) {
this.assetId = assetId;
this.attributeName = attributeName;
this.oldest = oldest;
@@ -38,12 +38,12 @@ public class DatapointPeriod {
}
@JsonProperty("oldestTimestamp")
- public long getOldest() {
+ public Long getOldest() {
return oldest;
}
@JsonProperty("latestTimestamp")
- public long getLatest() {
+ public Long getLatest() {
return latest;
}
}
diff --git a/profile/basic-identity.yml b/profile/basic-identity.yml
index c3556a79b..4c8ede7a2 100644
--- a/profile/basic-identity.yml
+++ b/profile/basic-identity.yml
@@ -12,6 +12,7 @@
version: '2.4'
volumes:
+ temp-data:
postgresql-data:
services:
@@ -30,6 +31,7 @@ services:
service: postgresql
volumes:
- postgresql-data:/var/lib/postgresql/data
+ - temp-data:/tmp
manager:
extends:
@@ -38,5 +40,7 @@ services:
depends_on:
postgresql:
condition: service_healthy
+ volumes:
+ - temp-data:/tmp
environment:
IDENTITY_PROVIDER: basic
diff --git a/profile/deploy.yml b/profile/deploy.yml
index effb41aab..694ac78e8 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -23,9 +23,9 @@
#
version: '2.4'
-# If you enable volume mappings in services, you need this. Otherwise, remove the volumes
volumes:
- postgresql-data:
+ postgresql-data: # Needed if you want to persist postgres data outside of container
+ temp-data: # Needed to share temp export data between postgres and manager containers
services:
@@ -105,7 +105,8 @@ services:
# /deployment/manager/provisioning - contains JSON files for provisioning the system
# /deployment/map/mapdata.mbtiles - map tile data
# /deployment/map/mapsettings.json - map styling
- #volumes:
+ volumes:
+ - temp-data:/tmp
# - deployment-data:/deployment
# - postgresql-data:/postgres
# Expose any devices required by agents
@@ -307,7 +308,8 @@ services:
image: openremote/postgresql:${POSTGRESQL_VERSION:-9.6.21.0}
restart: always
# Docker volume for persistent storage of DB; can alternatively map to host
- #volumes:
+ volumes:
+ - temp-data:/tmp
# - postgresql-data:/var/lib/postgresql/data
#ports:
# - "5432:5432" # DB server
diff --git a/profile/dev-testing.yml b/profile/dev-testing.yml
index 42752d9ae..b45a3dbc9 100644
--- a/profile/dev-testing.yml
+++ b/profile/dev-testing.yml
@@ -31,7 +31,7 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - ./postgresql:/var/lib/postgresql
+ - ./tmp:/tmp
# Access directly if needed on localhost
ports:
- "5432:5432"