100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Made export AssetDatapoint async, so closed connections will cancel the query to create the CSV file
Michael Rademaker8d990e165751
diff --git a/container/src/main/java/org/openremote/container/web/WebService.java b/container/src/main/java/org/openremote/container/web/WebService.javaindex 09a0bd5a5..6031dc172 100644--- a/container/src/main/java/org/openremote/container/web/WebService.java+++ b/container/src/main/java/org/openremote/container/web/WebService.java@@ -295,7 +295,8 @@ public abstract class WebService implements ContainerService {if (corsFilter != null) {CORSFilter finalCorsFilter = corsFilter;- corsFilterInfo = Servlets.filter("CORS Filter", CORSFilter.class, () -> new ImmediateInstanceHandle<>(finalCorsFilter));+ corsFilterInfo = Servlets.filter("CORS Filter", CORSFilter.class, () -> new ImmediateInstanceHandle<>(finalCorsFilter))+ .setAsyncSupported(true);}}}diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.javaindex b32d8596b..9684f05f8 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java@@ -28,12 +28,14 @@ import java.sql.ResultSet;import java.sql.SQLException;import java.time.Instant;import java.time.LocalDateTime;+import java.time.ZoneId;import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;import java.util.stream.Collectors;@@ -52,6 +54,8 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements Cprotected ScheduledExecutorService executorService;protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;+ private String managerFilePath;+@Overridepublic int getPriority() {return PRIORITY;@@ -63,6 +67,8 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements CassetStorageService = container.getService(AssetStorageService.class);timerService = container.getService(TimerService.class);executorService = container.getExecutorService();++ managerFilePath = container.isDevMode() ? "./postgresql" : "/postgresql";}@Override@@ -368,24 +374,25 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C);}- public File exportDatapoints(AttributeRef[] attributeRefs,- long fromTimestamp,- long toTimestamp) {-- int count = attributeRefs.length;- Instant fromInstant = Instant.ofEpochSecond(fromTimestamp);- Instant toInstant = Instant.ofEpochSecond(toTimestamp);- String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());+ public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,+ long fromTimestamp,+ long toTimestamp) {+ return executorService.schedule(() -> {+ int count = attributeRefs.length;+ LocalDateTime fromInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault());+ LocalDateTime toInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault());+ String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp, toTimestamp));+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));+ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));- sb.append(String.format(")) to '/var/lib/postgresql/data/%s' delimiter ',' CSV HEADER;", fileName));+ sb.append(String.format(")) to '/var/lib/postgresql/%s' delimiter ',' CSV HEADER;", fileName));- persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());+ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());- return new File(String.format("/postgresql/%s", fileName));+ return new File(String.format("%s/%s", managerFilePath, fileName));+ }, 0, TimeUnit.MILLISECONDS);}protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.javaindex 3cf37c80b..923e5bb72 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java@@ -38,12 +38,16 @@ import javax.ws.rs.BadRequestException;import javax.ws.rs.BeanParam;import javax.ws.rs.NotSupportedException;import javax.ws.rs.WebApplicationException;+import javax.ws.rs.container.AsyncResponse;+import javax.ws.rs.container.ConnectionCallback;import javax.ws.rs.core.HttpHeaders;import javax.ws.rs.core.Response;import java.io.*;import java.time.Instant;import java.time.LocalDateTime;import java.time.ZoneId;+import java.util.concurrent.ScheduledFuture;+import java.util.logging.Level;import java.util.logging.Logger;import static org.openremote.model.value.Values.JSON;@@ -137,12 +141,11 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As}@Override- public void getDatapointExport(RequestParams requestParams, String attributeRefsString, long fromTimestamp, long toTimestamp) {+ public void getDatapointExport(AsyncResponse asyncResponse, String attributeRefsString, long fromTimestamp, long toTimestamp) {try {AttributeRef[] attributeRefs = JSON.readValue(attributeRefsString, AttributeRef[].class);- for (AttributeRef attributeRef : attributeRefs- ) {+ for (AttributeRef attributeRef : attributeRefs) {if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), attributeRef.getId())) {throw new WebApplicationException(Response.Status.FORBIDDEN);}@@ -163,23 +166,32 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As);}- File file = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);-- response.setContentType("text/csv");- response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());-- OutputStream out = response.getOutputStream();- FileInputStream in = new FileInputStream(file);- IOUtils.copy(in, out);-- out.close();- in.close();- file.delete();-+ ScheduledFuture<File> exportFuture = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);+ asyncResponse.register((ConnectionCallback) disconnected -> {+ exportFuture.cancel(true);+ });+ try {+ File file = exportFuture.get();+ response.setContentType("text/csv");+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());++ OutputStream out = response.getOutputStream();++ FileInputStream in = new FileInputStream(file);+ IOUtils.copy(in, out);++ out.close();+ in.close();+ file.delete();++ asyncResponse.resume(response);+ } catch (Exception ex) {+ exportFuture.cancel(true);+ asyncResponse.resume(new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR));+ LOG.log(Level.WARNING, "Exception in ScheduledFuture: ", ex);+ }} catch (JsonProcessingException ex) {- throw new BadRequestException(ex);- } catch (IOException ex) {- throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);+ asyncResponse.resume(new BadRequestException(ex));}}}diff --git a/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java b/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.javaindex c940ac3bc..da540e305 100644--- a/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java+++ b/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java@@ -103,7 +103,8 @@ public class ManagerFileServlet extends FileServlet {servletInfo.addMapping("/*");Filter gzipFilter = new GzipResponseFilter(MIME_TYPES_TO_ZIP);- FilterInfo gzipFilterInfo = Servlets.filter("Gzip Filter", GzipResponseFilter.class, () -> new ImmediateInstanceHandle<>(gzipFilter));+ FilterInfo gzipFilterInfo = Servlets.filter("Gzip Filter", GzipResponseFilter.class, () -> new ImmediateInstanceHandle<>(gzipFilter))+ .setAsyncSupported(true);return new DeploymentInfo().setDeploymentName(contextPath + " File Servlet Deployment")diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.javaindex 95cddd163..9107e43b9 100644--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java@@ -24,6 +24,8 @@ import org.openremote.model.Constants;import org.openremote.model.http.RequestParams;import javax.annotation.security.RolesAllowed;import javax.ws.rs.*;+import javax.ws.rs.container.AsyncResponse;+import javax.ws.rs.container.Suspended;import static javax.ws.rs.core.MediaType.APPLICATION_JSON;@@ -62,8 +64,8 @@ public interface AssetDatapointResource {@Path("export")@Produces("text/csv")@RolesAllowed({Constants.READ_ASSETS_ROLE})- void getDatapointExport(@BeanParam RequestParams requestParams,- @QueryParam("attributeRefs") String attributeRefsString,- @QueryParam("fromTimestamp") long fromTimestamp,- @QueryParam("toTimestamp") long toTimestamp);+ void getDatapointExport(@Suspended AsyncResponse asyncResponse,+ @QueryParam("attributeRefs") String attributeRefsString,+ @QueryParam("fromTimestamp") long fromTimestamp,+ @QueryParam("toTimestamp") long toTimestamp);}diff --git a/profile/deploy.yml b/profile/deploy.ymlindex d163a0bcd..9df7971c0 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -107,6 +107,7 @@ services:# /deployment/map/mapsettings.json - map styling#volumes:# - deployment-data:/deployment+ # - postgresql-data:/postgres# Expose any devices required by agents#devices:# - /dev/ttyACM0diff --git a/profile/dev-testing.yml b/profile/dev-testing.ymlindex 2920c45cd..42752d9ae 100644--- a/profile/dev-testing.yml+++ b/profile/dev-testing.yml@@ -30,6 +30,8 @@ services:extends:file: deploy.ymlservice: postgresql+ volumes:+ - ./postgresql:/var/lib/postgresql# Access directly if needed on localhostports:- "5432:5432"