Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Added data export endpoint

Michael Rademaker5b081fa39994
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
index d16e7a157..b32d8596b 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
@@ -21,6 +21,7 @@ import org.postgresql.util.PGInterval;
import org.postgresql.util.PGobject;
import javax.persistence.TypedQuery;
+import java.io.File;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -29,11 +30,13 @@ import java.time.Instant;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.logging.Level;
import java.util.logging.Logger;
+import java.util.stream.Collectors;
import static java.time.temporal.ChronoUnit.DAYS;
@@ -320,7 +323,7 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
value = Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null);
} else {
if (rs.getObject(2) instanceof PGobject) {
- value = Values.parse(((PGobject)rs.getObject(2)).getValue()).orElse(null);
+ value = Values.parse(((PGobject) rs.getObject(2)).getValue()).orElse(null);
} else {
value = Values.getValueCoerced(rs.getObject(2), JsonNode.class).orElse(null);
}
@@ -365,6 +368,26 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
);
}
+ public File exportDatapoints(AttributeRef[] attributeRefs,
+ long fromTimestamp,
+ long toTimestamp) {
+
+ int count = attributeRefs.length;
+ Instant fromInstant = Instant.ofEpochSecond(fromTimestamp);
+ Instant toInstant = Instant.ofEpochSecond(toTimestamp);
+ String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());
+
+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp, toTimestamp));
+
+ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
+
+ sb.append(String.format(")) to '/var/lib/postgresql/data/%s' delimiter ',' CSV HEADER;", fileName));
+
+ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
+
+ return new File(String.format("/postgresql/%s", fileName));
+ }
+
protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {
return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " +
"VALUES (?, ?, ?, ?) " +
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
index 12af9be0c..3cf37c80b 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
@@ -19,12 +19,15 @@
*/
package org.openremote.manager.datapoint;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.commons.io.IOUtils;
import org.openremote.container.timer.TimerService;
import org.openremote.manager.asset.AssetStorageService;
import org.openremote.manager.security.ManagerIdentityService;
import org.openremote.manager.web.ManagerWebResource;
import org.openremote.model.asset.Asset;
import org.openremote.model.attribute.Attribute;
+import org.openremote.model.attribute.AttributeRef;
import org.openremote.model.datapoint.AssetDatapointResource;
import org.openremote.model.datapoint.DatapointInterval;
import org.openremote.model.datapoint.DatapointPeriod;
@@ -35,12 +38,16 @@ import javax.ws.rs.BadRequestException;
import javax.ws.rs.BeanParam;
import javax.ws.rs.NotSupportedException;
import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
+import java.io.*;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.logging.Logger;
+import static org.openremote.model.value.Values.JSON;
+
public class AssetDatapointResourceImpl extends ManagerWebResource implements AssetDatapointResource {
private static final Logger LOG = Logger.getLogger(AssetDatapointResourceImpl.class.getName());
@@ -129,4 +136,50 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
}
}
+ @Override
+ public void getDatapointExport(RequestParams requestParams, String attributeRefsString, long fromTimestamp, long toTimestamp) {
+ try {
+ AttributeRef[] attributeRefs = JSON.readValue(attributeRefsString, AttributeRef[].class);
+
+ for (AttributeRef attributeRef : attributeRefs
+ ) {
+ if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), attributeRef.getId())) {
+ throw new WebApplicationException(Response.Status.FORBIDDEN);
+ }
+
+ Asset<?> asset = assetStorageService.find(attributeRef.getId(), true);
+
+ if (asset == null) {
+ throw new WebApplicationException(Response.Status.NOT_FOUND);
+ }
+
+ if (!isTenantActiveAndAccessible(asset.getRealm())) {
+ LOG.info("Forbidden access for user '" + getUsername() + "': " + asset);
+ throw new WebApplicationException(Response.Status.FORBIDDEN);
+ }
+
+ Attribute<?> attribute = asset.getAttribute(attributeRef.getName()).orElseThrow(() ->
+ new WebApplicationException(Response.Status.NOT_FOUND)
+ );
+ }
+
+ File file = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);
+
+ response.setContentType("text/csv");
+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
+
+ OutputStream out = response.getOutputStream();
+ FileInputStream in = new FileInputStream(file);
+ IOUtils.copy(in, out);
+
+ out.close();
+ in.close();
+ file.delete();
+
+ } catch (JsonProcessingException ex) {
+ throw new BadRequestException(ex);
+ } catch (IOException ex) {
+ throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
}
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
index 2b700e1c9..95cddd163 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
@@ -57,4 +57,13 @@ public interface AssetDatapointResource {
DatapointPeriod getDatapointPeriod(@BeanParam RequestParams requestParams,
@QueryParam("assetId") String assetId,
@QueryParam("attributeName") String attributeName);
+
+ @GET
+ @Path("export")
+ @Produces("text/csv")
+ @RolesAllowed({Constants.READ_ASSETS_ROLE})
+ void getDatapointExport(@BeanParam RequestParams requestParams,
+ @QueryParam("attributeRefs") String attributeRefsString,
+ @QueryParam("fromTimestamp") long fromTimestamp,
+ @QueryParam("toTimestamp") long toTimestamp);
}