Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Manager persistence updates (standardised storage volume and Flyway env variable) (#1004)

Rich Turnerb217d555845b
diff --git a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java
index 9d894e279..6abb304e4 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java
@@ -27,6 +27,7 @@ import com.welie.blessed.ScanResult;
import org.openremote.agent.protocol.AbstractProtocol;
import org.openremote.agent.protocol.bluetooth.mesh.models.SigModelParser;
import org.openremote.agent.protocol.bluetooth.mesh.utils.MeshParserUtils;
+import org.openremote.container.persistence.PersistenceService;
import org.openremote.model.Container;
import org.openremote.model.asset.Asset;
import org.openremote.model.asset.AssetTreeNode;
@@ -43,6 +44,7 @@ import org.openremote.model.value.MetaItemType;
import org.openremote.model.value.ValueFormat;
import org.openremote.model.value.ValueType;
+import java.nio.file.Path;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -74,9 +76,9 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,
public static final Logger LOG = SyslogCategory.getLogger(SyslogCategory.PROTOCOL, BluetoothMeshProtocol.class.getName());
- private static MainThreadManager mainThread = new MainThreadManager();
+ private static final MainThreadManager mainThread = new MainThreadManager();
private static ScheduledFuture<?> mainThreadFuture = null;
- private static BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {
+ private static final BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {
@Override
public void onConnectedPeripheral(BluetoothPeripheral peripheral) {
LOG.info("BluetoothCentralManager::onConnectedPeripheral: [Name=" + peripheral.getName() + ", Address=" + peripheral.getAddress() + "]");
@@ -132,10 +134,10 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,
}
}
};
- private static BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);
- private static List<BluetoothMeshNetwork> networkList = new LinkedList<>();
- private static SequenceNumberPersistencyManager sequenceNumberManager = new SequenceNumberPersistencyManager();
-
+ private static final BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);
+ private static final List<BluetoothMeshNetwork> networkList = new LinkedList<>();
+ // Not ideal this but will do for now
+ private static SequenceNumberPersistencyManager sequenceNumberManager;
public synchronized static void initMainThread(ScheduledExecutorService executorService) {
if (mainThreadFuture == null) {
@@ -180,6 +182,15 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,
@Override
protected synchronized void doStart(Container container) throws Exception {
+
+ synchronized (BluetoothMeshProtocol.mainThread) {
+ if (BluetoothMeshProtocol.sequenceNumberManager == null) {
+ Path storagePath = container.getService(PersistenceService.class).getStorageDir();
+ BluetoothMeshProtocol.sequenceNumberManager = new SequenceNumberPersistencyManager(storagePath.resolve("bluetoothmesh"));
+ BluetoothMeshProtocol.sequenceNumberManager.load();
+ }
+ }
+
LOG.info("Starting Bluetooth Mesh protocol.");
String meshNetKeyParam = agent.getNetworkKey().orElseThrow(() -> {
String msg = "No Bluetooth Mesh network key provided for protocol: " + this;
@@ -243,7 +254,7 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,
}
}
};
- BluetoothMeshProtocol.sequenceNumberManager.load();
+
Integer oldSequenceNumber = BluetoothMeshProtocol.sequenceNumberManager.getSequenceNumber(networkKey, sourceAddress);
if (oldSequenceNumber == null) {
oldSequenceNumber = sequenceNumberParam;
diff --git a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java
index 364d92923..71c9c9b49 100644
--- a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java
+++ b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java
@@ -31,10 +31,11 @@ import org.openremote.model.syslog.SyslogCategory;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.net.URI;
+import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.logging.Level;
import java.util.logging.Logger;
public class SequenceNumberPersistencyManager {
@@ -48,7 +49,11 @@ public class SequenceNumberPersistencyManager {
public static String XML_TAG_SEQUENCE_NUMBER = "SequenceNumber";
private final Map<String, Map<String, Integer>> map = new HashMap<>();
- private final URI filePath = new File("/btmesh/sequencenumber.xml").toURI();
+ private final Path filePath;
+
+ public SequenceNumberPersistencyManager(Path storagePath) {
+ filePath = storagePath.resolve("sequencenumber.xml");
+ }
public synchronized void save(NetworkKey networkKey, int unicastAddress, int sequenceNumber) {
String key = MeshParserUtils.bytesToHex(networkKey.key, false);
@@ -65,7 +70,7 @@ public class SequenceNumberPersistencyManager {
save();
}
- public synchronized void save() {
+ protected synchronized void save() {
Document doc = new Document();
Element root = new Element(XML_TAG_SEQUENCE_NUMBER_LIST);
@@ -96,8 +101,8 @@ public class SequenceNumberPersistencyManager {
FileWriter writer = null;
File file = null;
try {
- file = new File(filePath);
- writer = new FileWriter(new File(filePath));
+ file = filePath.toFile();
+ writer = new FileWriter(file);
outputter.output(doc, writer);
writer.flush();
} catch (IOException e) {
@@ -114,21 +119,16 @@ public class SequenceNumberPersistencyManager {
}
public synchronized void load() {
- if (!fileExists()) {
+ if (!filePath.toFile().exists()) {
return;
}
- File file = new File(filePath);
+ File file = filePath.toFile();
SAXBuilder builder = new SAXBuilder();
- Document doc = null;
+ Document doc;
try {
doc = builder.build(file);
- } catch (JDOMException e) {
- LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
- e.printStackTrace();
- return;
- } catch (IOException e) {
- LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
- e.printStackTrace();
+ } catch (JDOMException | IOException e) {
+ LOG.log(Level.SEVERE, "Failed to load sequence number file: '" + file.getPath() + "'", e);
return;
}
map.clear();
@@ -147,14 +147,13 @@ public class SequenceNumberPersistencyManager {
String number = numberElement.getText();
if (!map.containsKey(key)) {
- map.put(key, new HashMap<String, Integer>());
+ map.put(key, new HashMap<>());
}
try {
map.get(key).put(address, Integer.valueOf(number));
} catch (NumberFormatException e) {
- LOG.severe("Error while loading sequence number file: '" + file.getPath() + "' because: " + e.getMessage());
+ LOG.log(Level.SEVERE, "Error while loading sequence number file: '" + file.getPath() + "'", e);
e.printStackTrace();
- continue;
}
}
}
@@ -168,10 +167,4 @@ public class SequenceNumberPersistencyManager {
}
return number;
}
-
- // Private Instance Methods -------------------------------------------------------------------
-
- private boolean fileExists() {
- return new File(filePath).exists();
- }
}
diff --git a/build.gradle b/build.gradle
index 16f113e86..eeb41e13f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -35,3 +35,9 @@ task checkFilesGitIgnoredNew(type: Exec) {
commandLine args
}
+
+task clean() {
+ doLast {
+ delete "tmp"
+ }
+}
diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
index 1a0e77311..2c2bb22b9 100644
--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
@@ -63,8 +63,13 @@ import javax.persistence.spi.ClassTransformer;
import javax.persistence.spi.PersistenceUnitTransactionType;
import javax.sql.DataSource;
import javax.ws.rs.core.UriBuilder;
+import java.io.File;
import java.lang.reflect.Field;
import java.net.URL;
+import java.nio.file.FileSystemNotFoundException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
@@ -225,6 +230,9 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
public static final int OR_DB_MAX_POOL_SIZE_DEFAULT = 20;
public static final String OR_DB_CONNECTION_TIMEOUT_SECONDS = "OR_DB_CONNECTION_TIMEOUT_SECONDS";
public static final int OR_DB_CONNECTION_TIMEOUT_SECONDS_DEFAULT = 300;
+ public static final String OR_STORAGE_DIR = "OR_STORAGE_DIR";
+ public static final String OR_STORAGE_DIR_DEFAULT = "tmp";
+ public static final String OR_DB_FLYWAY_OUT_OF_ORDER = "OR_DB_FLYWAY_OUT_OF_ORDER";
public static final int PRIORITY = Integer.MIN_VALUE + 100;
protected MessageBrokerService messageBrokerService;
@@ -236,6 +244,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
protected boolean forceClean;
protected Set<String> defaultSchemaLocations = new HashSet<>();
protected Set<String> schemas = new HashSet<>();
+ protected Path storageDir;
public static Predicate isPersistenceEventForEntityType(Class<?> type) {
return exchange -> {
@@ -292,8 +301,24 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
forceClean = getBoolean(container.getConfig(), OR_SETUP_RUN_ON_RESTART, container.isDevMode());
+ storageDir = Paths.get(getString(container.getConfig(), OR_STORAGE_DIR, OR_STORAGE_DIR_DEFAULT));
+ LOG.log(Level.INFO, "Setting storage directory to '" + storageDir.toAbsolutePath() + "'");
+
+ if (!Files.exists(storageDir) || !Files.isDirectory(storageDir)) {
+ String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' doesn't exist or is not a folder";
+ LOG.log(Level.SEVERE, msg);
+ throw new FileSystemNotFoundException(msg);
+ } else {
+ File testFile = storageDir.toFile();
+ if (!testFile.canRead() || !testFile.canWrite()) {
+ String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' is not writable";
+ LOG.log(Level.SEVERE, msg);
+ throw new FileSystemNotFoundException(msg);
+ }
+ }
+
openDatabase(container, database, dbUsername, dbPassword, connectionUrl);
- prepareSchema(connectionUrl, dbUsername, dbPassword, dbSchema);
+ prepareSchema(container, connectionUrl, dbUsername, dbPassword, dbSchema);
// Register standard entity classes and also any Entity ClassProviders
List<String> entityClasses = new ArrayList<>(50);
@@ -480,7 +505,10 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
database.open(persistenceUnitProperties, connectionUrl, username, password, connectionTimeoutSeconds, databaseMinPoolSize, databaseMaxPoolSize);
}
- protected void prepareSchema(String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {
+ protected void prepareSchema(Container container, String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {
+
+ boolean outOfOrder = getBoolean(container.getConfig(), OR_DB_FLYWAY_OUT_OF_ORDER, false);
+
LOG.fine("Preparing database schema");
List<String> locations = new ArrayList<>();
List<String> schemas = new ArrayList<>();
@@ -503,6 +531,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
.locations(locations.toArray(new String[0]))
.initSql(initSql.toString())
.baselineOnMigrate(true)
+ .outOfOrder(outOfOrder)
.load();
MigrationInfo currentMigration;
@@ -565,6 +594,10 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
'}';
}
+ public Path getStorageDir() {
+ return storageDir;
+ }
+
public static Field[] getEntityPropertyFields(Class<?> clazz, List<String> includeFields, List<String> excludeFields) {
return Arrays.stream(clazz.getDeclaredFields())
.filter(field -> ((field.isAnnotationPresent(Column.class) || field.isAnnotationPresent(EmbeddedId.class) || field.isAnnotationPresent(JoinColumn.class) || field.isAnnotationPresent(Formula.class)) && (excludeFields == null || !excludeFields.contains(field.getName())))
diff --git a/deployment/manager/app/images/.appignore b/deployment/manager/app/images/.appignore
new file mode 100644
index 000000000..e69de29bb
diff --git a/docker-compose.yml b/docker-compose.yml
index c9bfa90e0..61a972c81 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,9 +9,8 @@ version: '2.4'
volumes:
proxy-data:
- temp-data:
+ manager-data:
postgresql-data:
-# btmesh-data:
services:
@@ -39,7 +38,7 @@ services:
image: openremote/postgresql:${POSTGRESQL_VERSION:-latest}
volumes:
- postgresql-data:/var/lib/postgresql/data
- - temp-data:/tmp
+ - manager-data:/storage
keycloak:
restart: always
@@ -83,9 +82,4 @@ services:
OR_SETUP_IMPORT_DEMO_AGENT_KNX:
OR_SETUP_IMPORT_DEMO_AGENT_VELBUS:
volumes:
- - temp-data:/tmp
-# - /var/run/dbus:/var/run/dbus
-# # Bluetooth mesh volume
-# - btmesh-data:/btmesh
-# devices:
-# - /dev/ttyACM0:/dev/ttyS0
+ - manager-data:/storage
diff --git a/manager/Dockerfile b/manager/Dockerfile
index 92ca3f810..f564f9178 100644
--- a/manager/Dockerfile
+++ b/manager/Dockerfile
@@ -35,6 +35,7 @@ ENV OR_DB_PASSWORD ${OR_DB_PASSWORD:-postgres}
ENV OR_DB_MIN_POOL_SIZE ${OR_DB_MIN_POOL_SIZE:-5}
ENV OR_DB_MAX_POOL_SIZE ${OR_DB_MAX_POOL_SIZE:-20}
ENV OR_DB_CONNECTION_TIMEOUT_SECONDS ${OR_DB_CONNECTION_TIMEOUT_SECONDS:-300}
+ENV OR_DB_FLYWAY_OUT_OF_ORDER ${OR_DB_FLYWAY_OUT_OF_ORDER}
ENV OR_KEYCLOAK_HOST ${OR_KEYCLOAK_HOST:-keycloak}
ENV OR_KEYCLOAK_PORT ${OR_KEYCLOAK_PORT:-8080}
ENV OR_KEYCLOAK_GRANT_FILE ${OR_KEYCLOAK_GRANT_FILE:-/deployment/manager/keycloak.json}
@@ -42,7 +43,6 @@ ENV OR_APP_DOCROOT ${OR_APP_DOCROOT:-/opt/web}
ENV OR_CUSTOM_APP_DOCROOT ${OR_CUSTOM_APP_DOCROOT:-/deployment/manager/app}
ENV OR_PROVISIONING_DOCROOT ${OR_PROVISIONING_DOCROOT:-/deployment/manager/provisioning}
ENV OR_ROOT_REDIRECT_PATH ${OR_ROOT_REDIRECT_PATH:-/manager}
-ENV OR_DATA_POINTS_EXPORT_DIR ${OR_DATA_POINTS_EXPORT_DIR:-/tmp}
ENV OR_LOGGING_CONFIG_FILE ${OR_LOGGING_CONFIG_FILE}
ENV OR_MAP_TILES_PATH ${OR_MAP_TILES_PATH:-/deployment.local/mapdata/mapdata.mbtiles}
ENV OR_MAP_SETTINGS_PATH ${OR_MAP_SETTINGS_PATH:-/deployment/map/mapsettings.json}
@@ -54,6 +54,7 @@ ENV OR_RULE_EVENT_EXPIRES ${OR_RULE_EVENT_EXPIRES:-PT1H}
ENV OR_IDENTITY_PROVIDER ${OR_IDENTITY_PROVIDER:-keycloak}
ENV OR_IDENTITY_SESSION_MAX_MINUTES ${OR_IDENTITY_SESSION_MAX_MINUTES:-1440}
ENV OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES ${OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES:-2628000}
+ENV OR_STORAGE_DIR ${OR_STORAGE_DIR:-/storage}
ENV OR_JAVA_OPTS ${OR_JAVA_OPTS:--Xms500m -Xmx2g \
-XX:NativeMemoryTracking=summary \
-Xlog:all=warning:stdout:uptime,level,tags \
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 82ae6fd6b..2fc5e1974 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -23,8 +23,8 @@ import org.openremote.model.value.MetaItemType;
import javax.persistence.EntityManager;
import java.io.File;
+import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.sql.Date;
import java.time.Duration;
import java.time.Instant;
@@ -43,7 +43,6 @@ import static java.time.temporal.ChronoUnit.DAYS;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
import static org.openremote.container.util.MapAccess.getInteger;
-import static org.openremote.container.util.MapAccess.getString;
import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;
/**
@@ -56,9 +55,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
public static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";
public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;
- public static final String OR_DATA_POINTS_EXPORT_DIR = "OR_DATA_POINTS_EXPORT_DIR";
- public static final String OR_DATA_POINTS_EXPORT_DIR_DEFAULT = "tmp";
private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());
+ protected static final String EXPORT_STORAGE_DIR_NAME = "datapoint";
protected int maxDatapointAgeDays;
protected Path exportPath;
@@ -81,7 +79,13 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
LOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");
}
- exportPath = Paths.get(getString(container.getConfig(), OR_DATA_POINTS_EXPORT_DIR, OR_DATA_POINTS_EXPORT_DIR_DEFAULT));
+ Path storageDir = persistenceService.getStorageDir();
+ exportPath = storageDir.resolve(EXPORT_STORAGE_DIR_NAME);
+ // Ensure export dir exists and is writable
+ Files.createDirectories(exportPath);
+ if (!exportPath.toFile().setWritable(true, false)) {
+ LOG.log(Level.WARNING, "Failed to set export dir write flag; data export may not work");
+ }
}
@Override
@@ -183,6 +187,34 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
} catch (Exception e) {
LOG.log(Level.WARNING, "Failed to run data points purge", e);
}
+
+ // Purge old exports
+ try {
+ long oneDayMillis = 24*60*60*1000;
+ File[] obsoleteExports = exportPath.toFile()
+ .listFiles(file ->
+ file.isFile()
+ && file.getName().endsWith("csv")
+ && file.lastModified() < timerService.getCurrentTimeMillis() - oneDayMillis
+ );
+
+ if (obsoleteExports != null) {
+ Arrays.stream(obsoleteExports)
+ .forEach(file -> {
+ boolean success = false;
+ try {
+ success = file.delete();
+ } catch (SecurityException e) {
+ LOG.log(Level.WARNING, "Cannot access the export file to delete it", e);
+ }
+ if (!success) {
+ LOG.log(Level.WARNING, "Failed to delete obsolete export '" + file.getName() + "'");
+ }
+ });
+ }
+ } catch (Exception e) {
+ LOG.log(Level.WARNING, "Failed to purge old exports", e);
+ }
}
protected String buildWhereClause(List<Pair<String, Attribute<?>>> attributes, boolean negate) {
@@ -198,21 +230,27 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
return " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";
}
+ /**
+ * Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped into the manager
+ * container so it can be accessed by this process.
+ */
public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,
long fromTimestamp,
long toTimestamp) {
return executorService.schedule(() -> {
String fileName = UniqueIdentifierGenerator.generateId() + ".csv";
- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));
-
- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));
-
- sb.append(String.format(")) to '" + exportPath.toString() + "/%s' delimiter ',' CSV HEADER;", fileName));
+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))
+ .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))
+ .append(")) to '/storage/")
+ .append(EXPORT_STORAGE_DIR_NAME)
+ .append("/")
+ .append(fileName)
+ .append("' delimiter ',' CSV HEADER;");
persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());
+ // The same path must resolve in both the postgresql container and the manager container
return exportPath.resolve(fileName).toFile();
}, 0, TimeUnit.MILLISECONDS);
}
-
}
diff --git a/profile/basic-identity.yml b/profile/basic-identity.yml
index 69881fcb4..23566203d 100644
--- a/profile/basic-identity.yml
+++ b/profile/basic-identity.yml
@@ -12,7 +12,7 @@
version: '2.4'
volumes:
- temp-data:
+ manager-data:
postgresql-data:
services:
@@ -31,7 +31,7 @@ services:
service: postgresql
volumes:
- postgresql-data:/var/lib/postgresql/data
- - temp-data:/tmp
+ - manager-data:/storage
manager:
extends:
@@ -41,6 +41,6 @@ services:
postgresql:
condition: service_healthy
volumes:
- - temp-data:/tmp
+ - manager-data:/storage
environment:
OR_IDENTITY_PROVIDER: basic
diff --git a/profile/demo.yml b/profile/demo.yml
index 74fbd45c9..d06a9a486 100644
--- a/profile/demo.yml
+++ b/profile/demo.yml
@@ -16,7 +16,7 @@ version: '2.4'
volumes:
proxy-data:
- temp-data:
+ manager-data:
postgresql-data:
deployment-data:
@@ -61,7 +61,7 @@ services:
image: openremote/postgresql:${POSTGRESQL_VERSION:-latest}
volumes:
- postgresql-data:/var/lib/postgresql/data
- - temp-data:/tmp
+ - manager-data:/storage
<<: *awslogs
keycloak:
@@ -106,7 +106,7 @@ services:
OR_SETUP_IMPORT_DEMO_AGENT_KNX:
OR_SETUP_IMPORT_DEMO_AGENT_VELBUS:
volumes:
- - temp-data:/tmp
+ - manager-data:/storage
- deployment-data:/deployment
<<: *awslogs
diff --git a/profile/deploy.yml b/profile/deploy.yml
index caa0b36a1..24ad04314 100644
--- a/profile/deploy.yml
+++ b/profile/deploy.yml
@@ -23,9 +23,8 @@ version: '2.4'
volumes:
# postgresql-data: # Needed if you want to persist postgres data outside of container
- temp-data: # Needed to share temp export data between postgres and manager containers
+ manager-data: # A storage volume for file persistence in the manager (can be used by services and protocols)
proxy-data: #Needed to persist SSL certificates to avoid excessive requests to Letsencrypt
- btmesh-data: #Needed to persist Bluetooth mesh data (if using Bluetooth mesh)
services:
@@ -115,9 +114,10 @@ services:
# /deployment/map/mapdata.mbtiles - map tile data
# /deployment/map/mapsettings.json - map styling
volumes:
- - temp-data:/tmp
+ - manager-data:/storage
# - deployment-data:/deployment
# - postgresql-data:/postgres
+ # - /var/run/dbus:/var/run/dbus # EXPOSE DBUS
# Expose any devices required by agents
#devices:
# - /dev/ttyACM0
@@ -212,6 +212,8 @@ services:
OR_DB_MIN_POOL_SIZE: ${OR_DB_MIN_POOL_SIZE:-5}
OR_DB_MAX_POOL_SIZE: ${OR_DB_MAX_POOL_SIZE:-20}
OR_DB_CONNECTION_TIMEOUT_SECONDS: ${OR_DB_CONNECTION_TIMEOUT_SECONDS:-300}
+ # Allow migration scripts to be applied out of order (see flyway DB docs)
+ OR_DB_FLYWAY_OUT_OF_ORDER: ${OR_DB_FLYWAY_OUT_OF_ORDER}
# The service-internal host name and port of the keycloak identity provider; defaults:
OR_KEYCLOAK_HOST: ${OR_KEYCLOAK_HOST:-keycloak}
@@ -366,7 +368,8 @@ services:
restart: always
# Docker volume for persistent storage of DB; can alternatively map to host
volumes:
- - temp-data:/tmp
+ # Needs to be available on the same path as the manager for data exports to work
+ - manager-data:/storage
# - postgresql-data:/var/lib/postgresql/data
#ports:
# - "5432:5432" # DB server
@@ -382,6 +385,11 @@ services:
POSTGRES_USER: ${OR_DB_USER:-postgres}
POSTGRES_PASSWORD: ${OR_DB_PASSWORD:-postgres}
+ # When a newer POSTGRESQL image is used with a higher OR_REINDEX_COUNTER and PGDATA is
+ # preserved then the container will REINDEX the DB automatically unless the following
+ # is set to false
+ #OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false}
+
# Map tile server
tileserver:
image: openremote/tileserver:${TILESERVER_VERSION:-latest}
diff --git a/profile/dev-proxy.yml b/profile/dev-proxy.yml
index faf6cec45..68d634bd8 100644
--- a/profile/dev-proxy.yml
+++ b/profile/dev-proxy.yml
@@ -43,7 +43,7 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - ../tmp:/tmp
+ - ../tmp:/storage
# Access directly if needed on localhost
ports:
- "5432:5432"
diff --git a/profile/dev-testing.yml b/profile/dev-testing.yml
index 3e892a3b6..990532d9a 100644
--- a/profile/dev-testing.yml
+++ b/profile/dev-testing.yml
@@ -31,7 +31,7 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - ../tmp:/tmp
+ - ../tmp:/storage
# Access directly if needed on localhost
ports:
- "5432:5432"
diff --git a/profile/dev-ui.yml b/profile/dev-ui.yml
index 228338aaa..d484451e3 100644
--- a/profile/dev-ui.yml
+++ b/profile/dev-ui.yml
@@ -7,7 +7,7 @@
version: '2.4'
volumes:
- temp-data:
+ manager-data:
services:
@@ -35,7 +35,7 @@ services:
file: deploy.yml
service: postgresql
volumes:
- - temp-data:/tmp
+ - manager-data:/storage
# Access directly if needed on localhost
ports:
- "5432:5432"
@@ -52,7 +52,7 @@ services:
postgresql:
condition: service_healthy
volumes:
- - temp-data:/tmp
+ - manager-data:/storage
environment:
OR_SETUP_RUN_ON_RESTART: ${OR_SETUP_RUN_ON_RESTART:-true}
OR_DEV_MODE: ${OR_DEV_MODE:-true}
diff --git a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
index b07feacb7..3621008a8 100644
--- a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy
@@ -11,7 +11,6 @@ import org.openremote.manager.setup.SetupService
import org.openremote.model.asset.impl.*
import org.openremote.model.attribute.AttributeEvent
import org.openremote.model.attribute.AttributeRef
-import org.openremote.model.datapoint.DatapointInterval
import org.openremote.model.datapoint.query.AssetDatapointIntervalQuery
import org.openremote.model.util.ValueUtil
import org.openremote.test.ManagerContainerTrait
diff --git a/ui/component/or-icon/package.json b/ui/component/or-icon/package.json
index 5219f622b..f03b47d47 100644
--- a/ui/component/or-icon/package.json
+++ b/ui/component/or-icon/package.json
@@ -24,7 +24,7 @@
"@mdi/font": "^6.5.95",
"terser-webpack-plugin": "^5.2.5",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1"
},
"publishConfig": {
diff --git a/ui/demo/demo-core/package.json b/ui/demo/demo-core/package.json
index 2727d6fea..dc9698271 100644
--- a/ui/demo/demo-core/package.json
+++ b/ui/demo/demo-core/package.json
@@ -23,7 +23,7 @@
"devDependencies": {
"@openremote/util": "workspace:*",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/ui/demo/demo-or-asset-tree/package.json b/ui/demo/demo-or-asset-tree/package.json
index db55755db..42d1428dd 100644
--- a/ui/demo/demo-or-asset-tree/package.json
+++ b/ui/demo/demo-or-asset-tree/package.json
@@ -19,7 +19,7 @@
"devDependencies": {
"@openremote/util": "workspace:*",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/ui/demo/demo-or-map/package.json b/ui/demo/demo-or-map/package.json
index f0acba7b0..05eca99b9 100644
--- a/ui/demo/demo-or-map/package.json
+++ b/ui/demo/demo-or-map/package.json
@@ -19,7 +19,7 @@
"devDependencies": {
"@openremote/util": "workspace:*",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/ui/demo/demo-or-thermostat/package.json b/ui/demo/demo-or-thermostat/package.json
index cbc533918..4d9bd50d4 100644
--- a/ui/demo/demo-or-thermostat/package.json
+++ b/ui/demo/demo-or-thermostat/package.json
@@ -19,7 +19,7 @@
"devDependencies": {
"@openremote/util": "workspace:*",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/ui/demo/demo-rest/package.json b/ui/demo/demo-rest/package.json
index 63241825d..d00061867 100644
--- a/ui/demo/demo-rest/package.json
+++ b/ui/demo/demo-rest/package.json
@@ -20,7 +20,7 @@
"devDependencies": {
"@openremote/util": "workspace:*",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/ui/util/package.json b/ui/util/package.json
index b339e66f1..c902aa433 100644
--- a/ui/util/package.json
+++ b/ui/util/package.json
@@ -40,7 +40,7 @@
"style-loader": "^3.3.1",
"ts-loader": "^9.2.6",
"typescript": ">=4.5.2",
- "webpack": "^5.64.3",
+ "webpack": "^5.76.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^4.5.0"
}
diff --git a/yarn.lock b/yarn.lock
index 86087fad9..7cbf699ef 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2196,7 +2196,7 @@ __metadata:
i18next: ^21.5.3
lit: ^2.0.2
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
languageName: unknown
@@ -2210,7 +2210,7 @@ __metadata:
"@openremote/or-asset-tree": ^1.1.0
"@openremote/util": "workspace:*"
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
languageName: unknown
@@ -2224,7 +2224,7 @@ __metadata:
"@openremote/or-map": ^1.1.0
"@openremote/util": "workspace:*"
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
languageName: unknown
@@ -2238,7 +2238,7 @@ __metadata:
"@openremote/util": "workspace:*"
source-map-loader: ^3.0.0
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
languageName: unknown
@@ -2252,7 +2252,7 @@ __metadata:
"@openremote/util": "workspace:*"
lit: ^2.0.2
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
languageName: unknown
@@ -2506,7 +2506,7 @@ __metadata:
lit: ^2.0.2
terser-webpack-plugin: ^5.2.5
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
languageName: unknown
linkType: soft
@@ -2733,7 +2733,7 @@ __metadata:
style-loader: ^3.3.1
ts-loader: ^9.2.6
typescript: ">=4.5.2"
- webpack: ^5.64.3
+ webpack: ^5.76.0
webpack-cli: ^4.9.1
webpack-dev-server: ^4.5.0
bin:
@@ -3439,7 +3439,7 @@ __metadata:
languageName: node
linkType: hard
-"acorn@npm:^8.4.1, acorn@npm:^8.5.0, acorn@npm:^8.7.0":
+"acorn@npm:^8.5.0, acorn@npm:^8.7.0":
version: 8.7.1
resolution: "acorn@npm:8.7.1"
bin:
@@ -3448,6 +3448,15 @@ __metadata:
languageName: node
linkType: hard
+"acorn@npm:^8.7.1":
+ version: 8.8.2
+ resolution: "acorn@npm:8.8.2"
+ bin:
+ acorn: bin/acorn
+ checksum: f790b99a1bf63ef160c967e23c46feea7787e531292bb827126334612c234ed489a0dc2c7ba33156416f0ffa8d25bf2b0fdb7f35c2ba60eb3e960572bece4001
+ languageName: node
+ linkType: hard
+
"agent-base@npm:6, agent-base@npm:^6.0.2":
version: 6.0.2
resolution: "agent-base@npm:6.0.2"
@@ -4477,11 +4486,11 @@ __metadata:
linkType: hard
"dns-packet@npm:^5.2.2":
- version: 5.3.1
- resolution: "dns-packet@npm:5.3.1"
+ version: 5.4.0
+ resolution: "dns-packet@npm:5.4.0"
dependencies:
"@leichtgewicht/ip-codec": ^2.0.1
- checksum: 196ff74a0669126cf5fc901a5849b72f625bd7a4cb163b3f4d41fbe19ed0b017cf7674daef5b0acbd448c094fcd795e501d7066f301be428e4acecfcf3c5f336
+ checksum: a169963848e8539dfd8a19058562f9e1c15c0f82cbf76fa98942f11c46f3c74e7e7c82e3a8a5182d4c9e6ff19e21be738dbd098a876dde755d3aedd2cc730880
languageName: node
linkType: hard
@@ -4611,7 +4620,7 @@ __metadata:
languageName: node
linkType: hard
-"enhanced-resolve@npm:^5.0.0, enhanced-resolve@npm:^5.9.2":
+"enhanced-resolve@npm:^5.0.0":
version: 5.9.3
resolution: "enhanced-resolve@npm:5.9.3"
dependencies:
@@ -4621,6 +4630,16 @@ __metadata:
languageName: node
linkType: hard
+"enhanced-resolve@npm:^5.10.0":
+ version: 5.12.0
+ resolution: "enhanced-resolve@npm:5.12.0"
+ dependencies:
+ graceful-fs: ^4.2.4
+ tapable: ^2.2.0
+ checksum: bf3f787facaf4ce3439bef59d148646344e372bef5557f0d37ea8aa02c51f50a925cd1f07b8d338f18992c29f544ec235a8c64bcdb56030196c48832a5494174
+ languageName: node
+ linkType: hard
+
"entities@npm:^2.0.0":
version: 2.2.0
resolution: "entities@npm:2.2.0"
@@ -6278,14 +6297,7 @@ __metadata:
languageName: node
linkType: hard
-"json-parse-better-errors@npm:^1.0.2":
- version: 1.0.2
- resolution: "json-parse-better-errors@npm:1.0.2"
- checksum: ff2b5ba2a70e88fd97a3cb28c1840144c5ce8fae9cbeeddba15afa333a5c407cf0e42300cd0a2885dbb055227fe68d405070faad941beeffbfde9cf3b2c78c5d
- languageName: node
- linkType: hard
-
-"json-parse-even-better-errors@npm:^2.3.0":
+"json-parse-even-better-errors@npm:^2.3.0, json-parse-even-better-errors@npm:^2.3.1":
version: 2.3.1
resolution: "json-parse-even-better-errors@npm:2.3.1"
checksum: 798ed4cf3354a2d9ccd78e86d2169515a0097a5c133337807cdf7f1fc32e1391d207ccfc276518cc1d7d8d4db93288b8a50ba4293d212ad1336e52a8ec0a941f
@@ -8899,13 +8911,13 @@ __metadata:
languageName: node
linkType: hard
-"watchpack@npm:^2.3.1":
- version: 2.3.1
- resolution: "watchpack@npm:2.3.1"
+"watchpack@npm:^2.4.0":
+ version: 2.4.0
+ resolution: "watchpack@npm:2.4.0"
dependencies:
glob-to-regexp: ^0.4.1
graceful-fs: ^4.1.2
- checksum: 70a34f92842d94b5d842980f866d568d7a467de667c96ae5759c759f46587e49265863171f4650bdbafc5f3870a28f2b4453e9e847098ec4b718b38926d47d22
+ checksum: 23d4bc58634dbe13b86093e01c6a68d8096028b664ab7139d58f0c37d962d549a940e98f2f201cecdabd6f9c340338dc73ef8bf094a2249ef582f35183d1a131
languageName: node
linkType: hard
@@ -9034,40 +9046,40 @@ __metadata:
languageName: node
linkType: hard
-"webpack@npm:^5.64.3":
- version: 5.72.0
- resolution: "webpack@npm:5.72.0"
+"webpack@npm:^5.76.0":
+ version: 5.76.1
+ resolution: "webpack@npm:5.76.1"
dependencies:
"@types/eslint-scope": ^3.7.3
"@types/estree": ^0.0.51
"@webassemblyjs/ast": 1.11.1
"@webassemblyjs/wasm-edit": 1.11.1
"@webassemblyjs/wasm-parser": 1.11.1
- acorn: ^8.4.1
+ acorn: ^8.7.1
acorn-import-assertions: ^1.7.6
browserslist: ^4.14.5
chrome-trace-event: ^1.0.2
- enhanced-resolve: ^5.9.2
+ enhanced-resolve: ^5.10.0
es-module-lexer: ^0.9.0
eslint-scope: 5.1.1
events: ^3.2.0
glob-to-regexp: ^0.4.1
graceful-fs: ^4.2.9
- json-parse-better-errors: ^1.0.2
+ json-parse-even-better-errors: ^2.3.1
loader-runner: ^4.2.0
mime-types: ^2.1.27
neo-async: ^2.6.2
schema-utils: ^3.1.0
tapable: ^2.1.1
terser-webpack-plugin: ^5.1.3
- watchpack: ^2.3.1
+ watchpack: ^2.4.0
webpack-sources: ^3.2.3
peerDependenciesMeta:
webpack-cli:
optional: true
bin:
webpack: bin/webpack.js
- checksum: 8365f1466d0f7adbf80ebc9b780f263a28eeeabcd5fb515249bfd9a56ab7fe8d29ea53df3d9364d0732ab39ae774445eb28abce694ed375b13882a6b2fe93ffc
+ checksum: b01fe0bc2dbca0e10d290ddb0bf81e807a031de48028176e2b21afd696b4d3f25ab9accdad888ef4a1f7c7f4d41f13d5bf2395b7653fdf3e5e3dafa54e56dab2
languageName: node
linkType: hard