100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Added data export endpoint
Michael Rademaker5b081fa39994
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javaindex d16e7a157..b32d8596b 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -21,6 +21,7 @@ import org.postgresql.util.PGInterval;import org.postgresql.util.PGobject;import javax.persistence.TypedQuery;+import java.io.File;import java.sql.Connection;import java.sql.PreparedStatement;import java.sql.ResultSet;@@ -29,11 +30,13 @@ import java.time.Instant;import java.time.LocalDateTime;import java.time.temporal.ChronoUnit;import java.util.ArrayList;+import java.util.Arrays;import java.util.List;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ScheduledFuture;import java.util.logging.Level;import java.util.logging.Logger;+import java.util.stream.Collectors;import static java.time.temporal.ChronoUnit.DAYS;@@ -320,7 +323,7 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements Cvalue = Values.getValueCoerced(rs.getObject(2), Double.class).orElse(null);} else {if (rs.getObject(2) instanceof PGobject) {- value = Values.parse(((PGobject)rs.getObject(2)).getValue()).orElse(null);+ value = Values.parse(((PGobject) rs.getObject(2)).getValue()).orElse(null);} else {value = Values.getValueCoerced(rs.getObject(2), JsonNode.class).orElse(null);}@@ -365,6 +368,26 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C);}+ public File exportDatapoints(AttributeRef[] attributeRefs,+ long fromTimestamp,+ long toTimestamp) {++ int count = attributeRefs.length;+ Instant fromInstant = Instant.ofEpochSecond(fromTimestamp);+ Instant toInstant = Instant.ofEpochSecond(toTimestamp);+ String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());++ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp, toTimestamp));++ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));++ sb.append(String.format(")) to '/var/lib/postgresql/data/%s' delimiter ',' CSV HEADER;", fileName));++ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());++ return new File(String.format("/postgresql/%s", fileName));+ }+protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {return connection.prepareStatement("INSERT INTO " + getDatapointTableName() + " (entity_id, attribute_name, value, timestamp) " +"VALUES (?, ?, ?, ?) " +diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.javaindex 12af9be0c..3cf37c80b 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java@@ -19,12 +19,15 @@*/package org.openremote.manager.datapoint;+import com.fasterxml.jackson.core.JsonProcessingException;+import org.apache.commons.io.IOUtils;import org.openremote.container.timer.TimerService;import org.openremote.manager.asset.AssetStorageService;import org.openremote.manager.security.ManagerIdentityService;import org.openremote.manager.web.ManagerWebResource;import org.openremote.model.asset.Asset;import org.openremote.model.attribute.Attribute;+import org.openremote.model.attribute.AttributeRef;import org.openremote.model.datapoint.AssetDatapointResource;import org.openremote.model.datapoint.DatapointInterval;import org.openremote.model.datapoint.DatapointPeriod;@@ -35,12 +38,16 @@ import javax.ws.rs.BadRequestException;import javax.ws.rs.BeanParam;import javax.ws.rs.NotSupportedException;import javax.ws.rs.WebApplicationException;+import javax.ws.rs.core.HttpHeaders;import javax.ws.rs.core.Response;+import java.io.*;import java.time.Instant;import java.time.LocalDateTime;import java.time.ZoneId;import java.util.logging.Logger;+import static org.openremote.model.value.Values.JSON;+public class AssetDatapointResourceImpl extends ManagerWebResource implements AssetDatapointResource {private static final Logger LOG = Logger.getLogger(AssetDatapointResourceImpl.class.getName());@@ -129,4 +136,50 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As}}+ @Override+ public void getDatapointExport(RequestParams requestParams, String attributeRefsString, long fromTimestamp, long toTimestamp) {+ try {+ AttributeRef[] attributeRefs = JSON.readValue(attributeRefsString, AttributeRef[].class);++ for (AttributeRef attributeRef : attributeRefs+ ) {+ if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), attributeRef.getId())) {+ throw new WebApplicationException(Response.Status.FORBIDDEN);+ }++ Asset<?> asset = assetStorageService.find(attributeRef.getId(), true);++ if (asset == null) {+ throw new WebApplicationException(Response.Status.NOT_FOUND);+ }++ if (!isTenantActiveAndAccessible(asset.getRealm())) {+ LOG.info("Forbidden access for user '" + getUsername() + "': " + asset);+ throw new WebApplicationException(Response.Status.FORBIDDEN);+ }++ Attribute<?> attribute = asset.getAttribute(attributeRef.getName()).orElseThrow(() ->+ new WebApplicationException(Response.Status.NOT_FOUND)+ );+ }++ File file = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);++ response.setContentType("text/csv");+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());++ OutputStream out = response.getOutputStream();+ FileInputStream in = new FileInputStream(file);+ IOUtils.copy(in, out);++ out.close();+ in.close();+ file.delete();++ } catch (JsonProcessingException ex) {+ throw new BadRequestException(ex);+ } catch (IOException ex) {+ throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);+ }+ }}diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.javaindex 2b700e1c9..95cddd163 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java@@ -57,4 +57,13 @@ public interface AssetDatapointResource {DatapointPeriod getDatapointPeriod(@BeanParam RequestParams requestParams,@QueryParam("assetId") String assetId,@QueryParam("attributeName") String attributeName);++ @GET+ @Path("export")+ @Produces("text/csv")+ @RolesAllowed({Constants.READ_ASSETS_ROLE})+ void getDatapointExport(@BeanParam RequestParams requestParams,+ @QueryParam("attributeRefs") String attributeRefsString,+ @QueryParam("fromTimestamp") long fromTimestamp,+ @QueryParam("toTimestamp") long toTimestamp);}