Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Made export AssetDatapoint async, so closed connections will cancel the query to create the CSV file

Michael Rademaker8d990e165751
diff --git a/container/src/main/java/org/openremote/container/web/WebService.java b/container/src/main/java/org/openremote/container/web/WebService.java
index 09a0bd5a5..6031dc172 100644
--- a/container/src/main/java/org/openremote/container/web/WebService.java
+++ b/container/src/main/java/org/openremote/container/web/WebService.java
@@ -295,7 +295,8 @@ public abstract class WebService implements ContainerService {
if (corsFilter != null) {
CORSFilter finalCorsFilter = corsFilter;
- corsFilterInfo = Servlets.filter("CORS Filter", CORSFilter.class, () -> new ImmediateInstanceHandle<>(finalCorsFilter));
+ corsFilterInfo = Servlets.filter("CORS Filter", CORSFilter.class, () -> new ImmediateInstanceHandle<>(finalCorsFilter))
+ .setAsyncSupported(true);
}
}
}
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
index b32d8596b..9684f05f8 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AbstractDatapointService.java
@@ -28,12 +28,14 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.Instant;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -52,6 +54,8 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
protected ScheduledExecutorService executorService;
protected ScheduledFuture<?> dataPointsPurgeScheduledFuture;
+ private String managerFilePath;
+
@Override
public int getPriority() {
return PRIORITY;
@@ -63,6 +67,8 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
assetStorageService = container.getService(AssetStorageService.class);
timerService = container.getService(TimerService.class);
executorService = container.getExecutorService();
+
+ managerFilePath = container.isDevMode() ? "./postgresql" : "/postgresql";
}
@Override
@@ -368,24 +374,25 @@ public abstract class AbstractDatapointService<T extends Datapoint> implements C
);
}
- public File exportDatapoints(AttributeRef[] attributeRefs,
- long fromTimestamp,
- long toTimestamp) {
-
- int count = attributeRefs.length;
- Instant fromInstant = Instant.ofEpochSecond(fromTimestamp);
- Instant toInstant = Instant.ofEpochSecond(toTimestamp);
- String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());
+ public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,
+ long fromTimestamp,
+ long toTimestamp) {
+ return executorService.schedule(() -> {
+ int count = attributeRefs.length;
+ LocalDateTime fromInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(fromTimestamp), ZoneId.systemDefault());
+ LocalDateTime toInstant = LocalDateTime.ofInstant(Instant.ofEpochMilli(toTimestamp), ZoneId.systemDefault());
+ String fileName = String.format("%s_%s_%d_%s.csv", fromInstant, toInstant, count, getDatapointTableName());
- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp, toTimestamp));
+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));
- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
+ sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
- sb.append(String.format(")) to '/var/lib/postgresql/data/%s' delimiter ',' CSV HEADER;", fileName));
+ sb.append(String.format(")) to '/var/lib/postgresql/%s' delimiter ',' CSV HEADER;", fileName));
- persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
+ persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
- return new File(String.format("/postgresql/%s", fileName));
+ return new File(String.format("%s/%s", managerFilePath, fileName));
+ }, 0, TimeUnit.MILLISECONDS);
}
protected PreparedStatement getUpsertPreparedStatement(Connection connection) throws SQLException {
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
index 3cf37c80b..923e5bb72 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointResourceImpl.java
@@ -38,12 +38,16 @@ import javax.ws.rs.BadRequestException;
import javax.ws.rs.BeanParam;
import javax.ws.rs.NotSupportedException;
import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.container.AsyncResponse;
+import javax.ws.rs.container.ConnectionCallback;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import java.io.*;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.util.concurrent.ScheduledFuture;
+import java.util.logging.Level;
import java.util.logging.Logger;
import static org.openremote.model.value.Values.JSON;
@@ -137,12 +141,11 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
}
@Override
- public void getDatapointExport(RequestParams requestParams, String attributeRefsString, long fromTimestamp, long toTimestamp) {
+ public void getDatapointExport(AsyncResponse asyncResponse, String attributeRefsString, long fromTimestamp, long toTimestamp) {
try {
AttributeRef[] attributeRefs = JSON.readValue(attributeRefsString, AttributeRef[].class);
- for (AttributeRef attributeRef : attributeRefs
- ) {
+ for (AttributeRef attributeRef : attributeRefs) {
if (isRestrictedUser() && !assetStorageService.isUserAsset(getUserId(), attributeRef.getId())) {
throw new WebApplicationException(Response.Status.FORBIDDEN);
}
@@ -163,23 +166,32 @@ public class AssetDatapointResourceImpl extends ManagerWebResource implements As
);
}
- File file = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);
-
- response.setContentType("text/csv");
- response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
-
- OutputStream out = response.getOutputStream();
- FileInputStream in = new FileInputStream(file);
- IOUtils.copy(in, out);
-
- out.close();
- in.close();
- file.delete();
-
+ ScheduledFuture<File> exportFuture = assetDatapointService.exportDatapoints(attributeRefs, fromTimestamp, toTimestamp);
+ asyncResponse.register((ConnectionCallback) disconnected -> {
+ exportFuture.cancel(true);
+ });
+ try {
+ File file = exportFuture.get();
+ response.setContentType("text/csv");
+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + file.getName());
+
+ OutputStream out = response.getOutputStream();
+
+ FileInputStream in = new FileInputStream(file);
+ IOUtils.copy(in, out);
+
+ out.close();
+ in.close();
+ file.delete();
+
+ asyncResponse.resume(response);
+ } catch (Exception ex) {
+ exportFuture.cancel(true);
+ asyncResponse.resume(new WebApplicationException(Response.Status.INTERNAL_SERVER_ERROR));
+ LOG.log(Level.WARNING, "Exception in ScheduledFuture: ", ex);
+ }
} catch (JsonProcessingException ex) {
- throw new BadRequestException(ex);
- } catch (IOException ex) {
- throw new WebApplicationException(ex, Response.Status.INTERNAL_SERVER_ERROR);
+ asyncResponse.resume(new BadRequestException(ex));
}
}
}
diff --git a/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java b/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java
index c940ac3bc..da540e305 100644
--- a/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java
+++ b/manager/src/main/java/org/openremote/manager/web/ManagerFileServlet.java
@@ -103,7 +103,8 @@ public class ManagerFileServlet extends FileServlet {
servletInfo.addMapping("/*");
Filter gzipFilter = new GzipResponseFilter(MIME_TYPES_TO_ZIP);
- FilterInfo gzipFilterInfo = Servlets.filter("Gzip Filter", GzipResponseFilter.class, () -> new ImmediateInstanceHandle<>(gzipFilter));
+ FilterInfo gzipFilterInfo = Servlets.filter("Gzip Filter", GzipResponseFilter.class, () -> new ImmediateInstanceHandle<>(gzipFilter))
+ .setAsyncSupported(true);
return new DeploymentInfo()
.setDeploymentName(contextPath + " File Servlet Deployment")
diff --git a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
index 95cddd163..9107e43b9 100644
--- a/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
+++ b/model/src/main/java/org/openremote/model/datapoint/AssetDatapointResource.java
@@ -24,6 +24,8 @@ import org.openremote.model.Constants;
import org.openremote.model.http.RequestParams;
import javax.annotation.security.RolesAllowed;
import javax.ws.rs.*;
+import javax.ws.rs.container.AsyncResponse;
+import javax.ws.rs.container.Suspended;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -62,8 +64,8 @@ public interface AssetDatapointResource {
@Path("export")
@Produces("text/csv")
@RolesAllowed({Constants.READ_ASSETS_ROLE})
- void getDatapointExport(@BeanParam RequestParams requestParams,
- @QueryParam("attributeRefs") String attributeRefsString,
- @QueryParam("fromTimestamp") long fromTimestamp,
- @QueryParam("toTimestamp") long toTimestamp);
+ void getDatapointExport(@Suspended AsyncResponse asyncResponse,
+ @QueryParam("attributeRefs") String attributeRefsString,
+ @QueryParam("fromTimestamp") long fromTimestamp,
+ @QueryParam("toTimestamp") long toTimestamp);
}
diff --git a/profile/deploy.yml b/profile/deploy.yml
index d163a0bcd..9df7971c0 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -107,6 +107,7 @@ services:
# /deployment/map/mapsettings.json - map styling
#volumes:
# - deployment-data:/deployment
+ # - postgresql-data:/postgres
# Expose any devices required by agents
#devices:
# - /dev/ttyACM0
diff --git a/profile/dev-testing.yml b/profile/dev-testing.yml
index 2920c45cd..42752d9ae 100644
--- a/profile/dev-testing.yml
+++ b/profile/dev-testing.yml
@@ -30,6 +30,8 @@ services:
extends:
file: deploy.yml
service: postgresql
+ volumes:
+ - ./postgresql:/var/lib/postgresql
# Access directly if needed on localhost
ports:
- "5432:5432"