100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Manager persistence updates (standardised storage volume and Flyway env variable) (#1004)
Rich Turnerb217d555845b
diff --git a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.javaindex 9d894e279..6abb304e4 100644--- a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java+++ b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/BluetoothMeshProtocol.java@@ -27,6 +27,7 @@ import com.welie.blessed.ScanResult;import org.openremote.agent.protocol.AbstractProtocol;import org.openremote.agent.protocol.bluetooth.mesh.models.SigModelParser;import org.openremote.agent.protocol.bluetooth.mesh.utils.MeshParserUtils;+import org.openremote.container.persistence.PersistenceService;import org.openremote.model.Container;import org.openremote.model.asset.Asset;import org.openremote.model.asset.AssetTreeNode;@@ -43,6 +44,7 @@ import org.openremote.model.value.MetaItemType;import org.openremote.model.value.ValueFormat;import org.openremote.model.value.ValueType;+import java.nio.file.Path;import java.util.HashMap;import java.util.LinkedList;import java.util.List;@@ -74,9 +76,9 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,public static final Logger LOG = SyslogCategory.getLogger(SyslogCategory.PROTOCOL, BluetoothMeshProtocol.class.getName());- private static MainThreadManager mainThread = new MainThreadManager();+ private static final MainThreadManager mainThread = new MainThreadManager();private static ScheduledFuture<?> mainThreadFuture = null;- private static BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {+ private static final BluetoothCentralManagerCallback bluetoothManagerCallback = new BluetoothCentralManagerCallback() {@Overridepublic void onConnectedPeripheral(BluetoothPeripheral peripheral) {LOG.info("BluetoothCentralManager::onConnectedPeripheral: [Name=" + peripheral.getName() + ", Address=" + peripheral.getAddress() + "]");@@ -132,10 +134,10 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,}}};- private static BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);- private static List<BluetoothMeshNetwork> networkList = new LinkedList<>();- private static SequenceNumberPersistencyManager sequenceNumberManager = new SequenceNumberPersistencyManager();-+ private static final BluetoothCentralManager bluetoothCentral = new BluetoothCentralManager(bluetoothManagerCallback);+ private static final List<BluetoothMeshNetwork> networkList = new LinkedList<>();+ // Not ideal this but will do for now+ private static SequenceNumberPersistencyManager sequenceNumberManager;public synchronized static void initMainThread(ScheduledExecutorService executorService) {if (mainThreadFuture == null) {@@ -180,6 +182,15 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,@Overrideprotected synchronized void doStart(Container container) throws Exception {++ synchronized (BluetoothMeshProtocol.mainThread) {+ if (BluetoothMeshProtocol.sequenceNumberManager == null) {+ Path storagePath = container.getService(PersistenceService.class).getStorageDir();+ BluetoothMeshProtocol.sequenceNumberManager = new SequenceNumberPersistencyManager(storagePath.resolve("bluetoothmesh"));+ BluetoothMeshProtocol.sequenceNumberManager.load();+ }+ }+LOG.info("Starting Bluetooth Mesh protocol.");String meshNetKeyParam = agent.getNetworkKey().orElseThrow(() -> {String msg = "No Bluetooth Mesh network key provided for protocol: " + this;@@ -243,7 +254,7 @@ public class BluetoothMeshProtocol extends AbstractProtocol<BluetoothMeshAgent,}}};- BluetoothMeshProtocol.sequenceNumberManager.load();+Integer oldSequenceNumber = BluetoothMeshProtocol.sequenceNumberManager.getSequenceNumber(networkKey, sourceAddress);if (oldSequenceNumber == null) {oldSequenceNumber = sequenceNumberParam;diff --git a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.javaindex 364d92923..71c9c9b49 100644--- a/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java+++ b/agent/src/main/java/org/openremote/agent/protocol/bluetooth/mesh/SequenceNumberPersistencyManager.java@@ -31,10 +31,11 @@ import org.openremote.model.syslog.SyslogCategory;import java.io.File;import java.io.FileWriter;import java.io.IOException;-import java.net.URI;+import java.nio.file.Path;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.logging.Level;import java.util.logging.Logger;public class SequenceNumberPersistencyManager {@@ -48,7 +49,11 @@ public class SequenceNumberPersistencyManager {public static String XML_TAG_SEQUENCE_NUMBER = "SequenceNumber";private final Map<String, Map<String, Integer>> map = new HashMap<>();- private final URI filePath = new File("/btmesh/sequencenumber.xml").toURI();+ private final Path filePath;++ public SequenceNumberPersistencyManager(Path storagePath) {+ filePath = storagePath.resolve("sequencenumber.xml");+ }public synchronized void save(NetworkKey networkKey, int unicastAddress, int sequenceNumber) {String key = MeshParserUtils.bytesToHex(networkKey.key, false);@@ -65,7 +70,7 @@ public class SequenceNumberPersistencyManager {save();}- public synchronized void save() {+ protected synchronized void save() {Document doc = new Document();Element root = new Element(XML_TAG_SEQUENCE_NUMBER_LIST);@@ -96,8 +101,8 @@ public class SequenceNumberPersistencyManager {FileWriter writer = null;File file = null;try {- file = new File(filePath);- writer = new FileWriter(new File(filePath));+ file = filePath.toFile();+ writer = new FileWriter(file);outputter.output(doc, writer);writer.flush();} catch (IOException e) {@@ -114,21 +119,16 @@ public class SequenceNumberPersistencyManager {}public synchronized void load() {- if (!fileExists()) {+ if (!filePath.toFile().exists()) {return;}- File file = new File(filePath);+ File file = filePath.toFile();SAXBuilder builder = new SAXBuilder();- Document doc = null;+ Document doc;try {doc = builder.build(file);- } catch (JDOMException e) {- LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());- e.printStackTrace();- return;- } catch (IOException e) {- LOG.severe("Failed to load sequence number file: '" + file.getPath() + "' because: " + e.getMessage());- e.printStackTrace();+ } catch (JDOMException | IOException e) {+ LOG.log(Level.SEVERE, "Failed to load sequence number file: '" + file.getPath() + "'", e);return;}map.clear();@@ -147,14 +147,13 @@ public class SequenceNumberPersistencyManager {String number = numberElement.getText();if (!map.containsKey(key)) {- map.put(key, new HashMap<String, Integer>());+ map.put(key, new HashMap<>());}try {map.get(key).put(address, Integer.valueOf(number));} catch (NumberFormatException e) {- LOG.severe("Error while loading sequence number file: '" + file.getPath() + "' because: " + e.getMessage());+ LOG.log(Level.SEVERE, "Error while loading sequence number file: '" + file.getPath() + "'", e);e.printStackTrace();- continue;}}}@@ -168,10 +167,4 @@ public class SequenceNumberPersistencyManager {}return number;}-- // Private Instance Methods --------------------------------------------------------------------- private boolean fileExists() {- return new File(filePath).exists();- }}diff --git a/build.gradle b/build.gradleindex 16f113e86..eeb41e13f 100644--- a/build.gradle+++ b/build.gradle@@ -35,3 +35,9 @@ task checkFilesGitIgnoredNew(type: Exec) {commandLine args}++task clean() {+ doLast {+ delete "tmp"+ }+}diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.javaindex 1a0e77311..2c2bb22b9 100644--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java@@ -63,8 +63,13 @@ import javax.persistence.spi.ClassTransformer;import javax.persistence.spi.PersistenceUnitTransactionType;import javax.sql.DataSource;import javax.ws.rs.core.UriBuilder;+import java.io.File;import java.lang.reflect.Field;import java.net.URL;+import java.nio.file.FileSystemNotFoundException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.Paths;import java.util.*;import java.util.function.Consumer;import java.util.function.Function;@@ -225,6 +230,9 @@ public class PersistenceService implements ContainerService, Consumer<Persistencpublic static final int OR_DB_MAX_POOL_SIZE_DEFAULT = 20;public static final String OR_DB_CONNECTION_TIMEOUT_SECONDS = "OR_DB_CONNECTION_TIMEOUT_SECONDS";public static final int OR_DB_CONNECTION_TIMEOUT_SECONDS_DEFAULT = 300;+ public static final String OR_STORAGE_DIR = "OR_STORAGE_DIR";+ public static final String OR_STORAGE_DIR_DEFAULT = "tmp";+ public static final String OR_DB_FLYWAY_OUT_OF_ORDER = "OR_DB_FLYWAY_OUT_OF_ORDER";public static final int PRIORITY = Integer.MIN_VALUE + 100;protected MessageBrokerService messageBrokerService;@@ -236,6 +244,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistencprotected boolean forceClean;protected Set<String> defaultSchemaLocations = new HashSet<>();protected Set<String> schemas = new HashSet<>();+ protected Path storageDir;public static Predicate isPersistenceEventForEntityType(Class<?> type) {return exchange -> {@@ -292,8 +301,24 @@ public class PersistenceService implements ContainerService, Consumer<PersistencforceClean = getBoolean(container.getConfig(), OR_SETUP_RUN_ON_RESTART, container.isDevMode());+ storageDir = Paths.get(getString(container.getConfig(), OR_STORAGE_DIR, OR_STORAGE_DIR_DEFAULT));+ LOG.log(Level.INFO, "Setting storage directory to '" + storageDir.toAbsolutePath() + "'");++ if (!Files.exists(storageDir) || !Files.isDirectory(storageDir)) {+ String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' doesn't exist or is not a folder";+ LOG.log(Level.SEVERE, msg);+ throw new FileSystemNotFoundException(msg);+ } else {+ File testFile = storageDir.toFile();+ if (!testFile.canRead() || !testFile.canWrite()) {+ String msg = "Specified OR_STORAGE_DIR '" + storageDir.toAbsolutePath() + "' is not writable";+ LOG.log(Level.SEVERE, msg);+ throw new FileSystemNotFoundException(msg);+ }+ }+openDatabase(container, database, dbUsername, dbPassword, connectionUrl);- prepareSchema(connectionUrl, dbUsername, dbPassword, dbSchema);+ prepareSchema(container, connectionUrl, dbUsername, dbPassword, dbSchema);// Register standard entity classes and also any Entity ClassProvidersList<String> entityClasses = new ArrayList<>(50);@@ -480,7 +505,10 @@ public class PersistenceService implements ContainerService, Consumer<Persistencdatabase.open(persistenceUnitProperties, connectionUrl, username, password, connectionTimeoutSeconds, databaseMinPoolSize, databaseMaxPoolSize);}- protected void prepareSchema(String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {+ protected void prepareSchema(Container container, String connectionUrl, String databaseUsername, String databasePassword, String schemaName) {++ boolean outOfOrder = getBoolean(container.getConfig(), OR_DB_FLYWAY_OUT_OF_ORDER, false);+LOG.fine("Preparing database schema");List<String> locations = new ArrayList<>();List<String> schemas = new ArrayList<>();@@ -503,6 +531,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc.locations(locations.toArray(new String[0])).initSql(initSql.toString()).baselineOnMigrate(true)+ .outOfOrder(outOfOrder).load();MigrationInfo currentMigration;@@ -565,6 +594,10 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc'}';}+ public Path getStorageDir() {+ return storageDir;+ }+public static Field[] getEntityPropertyFields(Class<?> clazz, List<String> includeFields, List<String> excludeFields) {return Arrays.stream(clazz.getDeclaredFields()).filter(field -> ((field.isAnnotationPresent(Column.class) || field.isAnnotationPresent(EmbeddedId.class) || field.isAnnotationPresent(JoinColumn.class) || field.isAnnotationPresent(Formula.class)) && (excludeFields == null || !excludeFields.contains(field.getName())))diff --git a/deployment/manager/app/images/.appignore b/deployment/manager/app/images/.appignorenew file mode 100644index 000000000..e69de29bbdiff --git a/docker-compose.yml b/docker-compose.ymlindex c9bfa90e0..61a972c81 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -9,9 +9,8 @@ version: '2.4'volumes:proxy-data:- temp-data:+ manager-data:postgresql-data:-# btmesh-data:services:@@ -39,7 +38,7 @@ services:image: openremote/postgresql:${POSTGRESQL_VERSION:-latest}volumes:- postgresql-data:/var/lib/postgresql/data- - temp-data:/tmp+ - manager-data:/storagekeycloak:restart: always@@ -83,9 +82,4 @@ services:OR_SETUP_IMPORT_DEMO_AGENT_KNX:OR_SETUP_IMPORT_DEMO_AGENT_VELBUS:volumes:- - temp-data:/tmp-# - /var/run/dbus:/var/run/dbus-# # Bluetooth mesh volume-# - btmesh-data:/btmesh-# devices:-# - /dev/ttyACM0:/dev/ttyS0+ - manager-data:/storagediff --git a/manager/Dockerfile b/manager/Dockerfileindex 92ca3f810..f564f9178 100644--- a/manager/Dockerfile+++ b/manager/Dockerfile@@ -35,6 +35,7 @@ ENV OR_DB_PASSWORD ${OR_DB_PASSWORD:-postgres}ENV OR_DB_MIN_POOL_SIZE ${OR_DB_MIN_POOL_SIZE:-5}ENV OR_DB_MAX_POOL_SIZE ${OR_DB_MAX_POOL_SIZE:-20}ENV OR_DB_CONNECTION_TIMEOUT_SECONDS ${OR_DB_CONNECTION_TIMEOUT_SECONDS:-300}+ENV OR_DB_FLYWAY_OUT_OF_ORDER ${OR_DB_FLYWAY_OUT_OF_ORDER}ENV OR_KEYCLOAK_HOST ${OR_KEYCLOAK_HOST:-keycloak}ENV OR_KEYCLOAK_PORT ${OR_KEYCLOAK_PORT:-8080}ENV OR_KEYCLOAK_GRANT_FILE ${OR_KEYCLOAK_GRANT_FILE:-/deployment/manager/keycloak.json}@@ -42,7 +43,6 @@ ENV OR_APP_DOCROOT ${OR_APP_DOCROOT:-/opt/web}ENV OR_CUSTOM_APP_DOCROOT ${OR_CUSTOM_APP_DOCROOT:-/deployment/manager/app}ENV OR_PROVISIONING_DOCROOT ${OR_PROVISIONING_DOCROOT:-/deployment/manager/provisioning}ENV OR_ROOT_REDIRECT_PATH ${OR_ROOT_REDIRECT_PATH:-/manager}-ENV OR_DATA_POINTS_EXPORT_DIR ${OR_DATA_POINTS_EXPORT_DIR:-/tmp}ENV OR_LOGGING_CONFIG_FILE ${OR_LOGGING_CONFIG_FILE}ENV OR_MAP_TILES_PATH ${OR_MAP_TILES_PATH:-/deployment.local/mapdata/mapdata.mbtiles}ENV OR_MAP_SETTINGS_PATH ${OR_MAP_SETTINGS_PATH:-/deployment/map/mapsettings.json}@@ -54,6 +54,7 @@ ENV OR_RULE_EVENT_EXPIRES ${OR_RULE_EVENT_EXPIRES:-PT1H}ENV OR_IDENTITY_PROVIDER ${OR_IDENTITY_PROVIDER:-keycloak}ENV OR_IDENTITY_SESSION_MAX_MINUTES ${OR_IDENTITY_SESSION_MAX_MINUTES:-1440}ENV OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES ${OR_IDENTITY_SESSION_OFFLINE_TIMEOUT_MINUTES:-2628000}+ENV OR_STORAGE_DIR ${OR_STORAGE_DIR:-/storage}ENV OR_JAVA_OPTS ${OR_JAVA_OPTS:--Xms500m -Xmx2g \-XX:NativeMemoryTracking=summary \-Xlog:all=warning:stdout:uptime,level,tags \diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 82ae6fd6b..2fc5e1974 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -23,8 +23,8 @@ import org.openremote.model.value.MetaItemType;import javax.persistence.EntityManager;import java.io.File;+import java.nio.file.Files;import java.nio.file.Path;-import java.nio.file.Paths;import java.sql.Date;import java.time.Duration;import java.time.Instant;@@ -43,7 +43,6 @@ import static java.time.temporal.ChronoUnit.DAYS;import static java.util.stream.Collectors.groupingBy;import static java.util.stream.Collectors.toList;import static org.openremote.container.util.MapAccess.getInteger;-import static org.openremote.container.util.MapAccess.getString;import static org.openremote.model.value.MetaItemType.STORE_DATA_POINTS;/**@@ -56,9 +55,8 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoipublic static final String OR_DATA_POINTS_MAX_AGE_DAYS = "OR_DATA_POINTS_MAX_AGE_DAYS";public static final int OR_DATA_POINTS_MAX_AGE_DAYS_DEFAULT = 31;- public static final String OR_DATA_POINTS_EXPORT_DIR = "OR_DATA_POINTS_EXPORT_DIR";- public static final String OR_DATA_POINTS_EXPORT_DIR_DEFAULT = "tmp";private static final Logger LOG = Logger.getLogger(AssetDatapointService.class.getName());+ protected static final String EXPORT_STORAGE_DIR_NAME = "datapoint";protected int maxDatapointAgeDays;protected Path exportPath;@@ -81,7 +79,13 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoiLOG.warning(OR_DATA_POINTS_MAX_AGE_DAYS + " value is not a valid value so data points won't be auto purged");}- exportPath = Paths.get(getString(container.getConfig(), OR_DATA_POINTS_EXPORT_DIR, OR_DATA_POINTS_EXPORT_DIR_DEFAULT));+ Path storageDir = persistenceService.getStorageDir();+ exportPath = storageDir.resolve(EXPORT_STORAGE_DIR_NAME);+ // Ensure export dir exists and is writable+ Files.createDirectories(exportPath);+ if (!exportPath.toFile().setWritable(true, false)) {+ LOG.log(Level.WARNING, "Failed to set export dir write flag; data export may not work");+ }}@Override@@ -183,6 +187,34 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi} catch (Exception e) {LOG.log(Level.WARNING, "Failed to run data points purge", e);}++ // Purge old exports+ try {+ long oneDayMillis = 24*60*60*1000;+ File[] obsoleteExports = exportPath.toFile()+ .listFiles(file ->+ file.isFile()+ && file.getName().endsWith("csv")+ && file.lastModified() < timerService.getCurrentTimeMillis() - oneDayMillis+ );++ if (obsoleteExports != null) {+ Arrays.stream(obsoleteExports)+ .forEach(file -> {+ boolean success = false;+ try {+ success = file.delete();+ } catch (SecurityException e) {+ LOG.log(Level.WARNING, "Cannot access the export file to delete it", e);+ }+ if (!success) {+ LOG.log(Level.WARNING, "Failed to delete obsolete export '" + file.getName() + "'");+ }+ });+ }+ } catch (Exception e) {+ LOG.log(Level.WARNING, "Failed to purge old exports", e);+ }}protected String buildWhereClause(List<Pair<String, Attribute<?>>> attributes, boolean negate) {@@ -198,21 +230,27 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoireturn " and (dp.assetId, dp.attributeName) " + (negate ? "not " : "") + "in (" + whereStr + ")";}+ /**+ * Exports datapoints as CSV using SQL; the export path used in the SQL query must also be mapped into the manager+ * container so it can be accessed by this process.+ */public ScheduledFuture<File> exportDatapoints(AttributeRef[] attributeRefs,long fromTimestamp,long toTimestamp) {return executorService.schedule(() -> {String fileName = UniqueIdentifierGenerator.generateId() + ".csv";- StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000));-- sb.append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")));-- sb.append(String.format(")) to '" + exportPath.toString() + "/%s' delimiter ',' CSV HEADER;", fileName));+ StringBuilder sb = new StringBuilder(String.format("copy (select ad.timestamp, a.name, ad.attribute_name, value from asset_datapoint ad, asset a where ad.entity_id = a.id and ad.timestamp >= to_timestamp(%d) and ad.timestamp <= to_timestamp(%d) and (", fromTimestamp / 1000, toTimestamp / 1000))+ .append(Arrays.stream(attributeRefs).map(attributeRef -> String.format("(ad.entity_id = '%s' and ad.attribute_name = '%s')", attributeRef.getId(), attributeRef.getName())).collect(Collectors.joining(" or ")))+ .append(")) to '/storage/")+ .append(EXPORT_STORAGE_DIR_NAME)+ .append("/")+ .append(fileName)+ .append("' delimiter ',' CSV HEADER;");persistenceService.doTransaction(em -> em.createNativeQuery(sb.toString()).executeUpdate());+ // The same path must resolve in both the postgresql container and the manager containerreturn exportPath.resolve(fileName).toFile();}, 0, TimeUnit.MILLISECONDS);}-}diff --git a/profile/basic-identity.yml b/profile/basic-identity.ymlindex 69881fcb4..23566203d 100644--- a/profile/basic-identity.yml+++ b/profile/basic-identity.yml@@ -12,7 +12,7 @@version: '2.4'volumes:- temp-data:+ manager-data:postgresql-data:services:@@ -31,7 +31,7 @@ services:service: postgresqlvolumes:- postgresql-data:/var/lib/postgresql/data- - temp-data:/tmp+ - manager-data:/storagemanager:extends:@@ -41,6 +41,6 @@ services:postgresql:condition: service_healthyvolumes:- - temp-data:/tmp+ - manager-data:/storageenvironment:OR_IDENTITY_PROVIDER: basicdiff --git a/profile/demo.yml b/profile/demo.ymlindex 74fbd45c9..d06a9a486 100644--- a/profile/demo.yml+++ b/profile/demo.yml@@ -16,7 +16,7 @@ version: '2.4'volumes:proxy-data:- temp-data:+ manager-data:postgresql-data:deployment-data:@@ -61,7 +61,7 @@ services:image: openremote/postgresql:${POSTGRESQL_VERSION:-latest}volumes:- postgresql-data:/var/lib/postgresql/data- - temp-data:/tmp+ - manager-data:/storage<<: *awslogskeycloak:@@ -106,7 +106,7 @@ services:OR_SETUP_IMPORT_DEMO_AGENT_KNX:OR_SETUP_IMPORT_DEMO_AGENT_VELBUS:volumes:- - temp-data:/tmp+ - manager-data:/storage- deployment-data:/deployment<<: *awslogsdiff --git a/profile/deploy.yml b/profile/deploy.ymlindex caa0b36a1..24ad04314 100644--- a/profile/deploy.yml+++ b/profile/deploy.yml@@ -23,9 +23,8 @@ version: '2.4'volumes:# postgresql-data: # Needed if you want to persist postgres data outside of container- temp-data: # Needed to share temp export data between postgres and manager containers+ manager-data: # A storage volume for file persistence in the manager (can be used by services and protocols)proxy-data: #Needed to persist SSL certificates to avoid excessive requests to Letsencrypt- btmesh-data: #Needed to persist Bluetooth mesh data (if using Bluetooth mesh)services:@@ -115,9 +114,10 @@ services:# /deployment/map/mapdata.mbtiles - map tile data# /deployment/map/mapsettings.json - map stylingvolumes:- - temp-data:/tmp+ - manager-data:/storage# - deployment-data:/deployment# - postgresql-data:/postgres+ # - /var/run/dbus:/var/run/dbus # EXPOSE DBUS# Expose any devices required by agents#devices:# - /dev/ttyACM0@@ -212,6 +212,8 @@ services:OR_DB_MIN_POOL_SIZE: ${OR_DB_MIN_POOL_SIZE:-5}OR_DB_MAX_POOL_SIZE: ${OR_DB_MAX_POOL_SIZE:-20}OR_DB_CONNECTION_TIMEOUT_SECONDS: ${OR_DB_CONNECTION_TIMEOUT_SECONDS:-300}+ # Allow migration scripts to be applied out of order (see flyway DB docs)+ OR_DB_FLYWAY_OUT_OF_ORDER: ${OR_DB_FLYWAY_OUT_OF_ORDER}# The service-internal host name and port of the keycloak identity provider; defaults:OR_KEYCLOAK_HOST: ${OR_KEYCLOAK_HOST:-keycloak}@@ -366,7 +368,8 @@ services:restart: always# Docker volume for persistent storage of DB; can alternatively map to hostvolumes:- - temp-data:/tmp+ # Needs to be available on the same path as the manager for data exports to work+ - manager-data:/storage# - postgresql-data:/var/lib/postgresql/data#ports:# - "5432:5432" # DB server@@ -382,6 +385,11 @@ services:POSTGRES_USER: ${OR_DB_USER:-postgres}POSTGRES_PASSWORD: ${OR_DB_PASSWORD:-postgres}+ # When a newer POSTGRESQL image is used with a higher OR_REINDEX_COUNTER and PGDATA is+ # preserved then the container will REINDEX the DB automatically unless the following+ # is set to false+ #OR_DISABLE_REINDEX=${OR_DISABLE_REINDEX:-false}+# Map tile servertileserver:image: openremote/tileserver:${TILESERVER_VERSION:-latest}diff --git a/profile/dev-proxy.yml b/profile/dev-proxy.ymlindex faf6cec45..68d634bd8 100644--- a/profile/dev-proxy.yml+++ b/profile/dev-proxy.yml@@ -43,7 +43,7 @@ services:file: deploy.ymlservice: postgresqlvolumes:- - ../tmp:/tmp+ - ../tmp:/storage# Access directly if needed on localhostports:- "5432:5432"diff --git a/profile/dev-testing.yml b/profile/dev-testing.ymlindex 3e892a3b6..990532d9a 100644--- a/profile/dev-testing.yml+++ b/profile/dev-testing.yml@@ -31,7 +31,7 @@ services:file: deploy.ymlservice: postgresqlvolumes:- - ../tmp:/tmp+ - ../tmp:/storage# Access directly if needed on localhostports:- "5432:5432"diff --git a/profile/dev-ui.yml b/profile/dev-ui.ymlindex 228338aaa..d484451e3 100644--- a/profile/dev-ui.yml+++ b/profile/dev-ui.yml@@ -7,7 +7,7 @@version: '2.4'volumes:- temp-data:+ manager-data:services:@@ -35,7 +35,7 @@ services:file: deploy.ymlservice: postgresqlvolumes:- - temp-data:/tmp+ - manager-data:/storage# Access directly if needed on localhostports:- "5432:5432"@@ -52,7 +52,7 @@ services:postgresql:condition: service_healthyvolumes:- - temp-data:/tmp+ - manager-data:/storageenvironment:OR_SETUP_RUN_ON_RESTART: ${OR_SETUP_RUN_ON_RESTART:-true}OR_DEV_MODE: ${OR_DEV_MODE:-true}diff --git a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovyindex b07feacb7..3621008a8 100644--- a/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy+++ b/test/src/test/groovy/org/openremote/test/energy/EnergyOptimisationAssetTest.groovy@@ -11,7 +11,6 @@ import org.openremote.manager.setup.SetupServiceimport org.openremote.model.asset.impl.*import org.openremote.model.attribute.AttributeEventimport org.openremote.model.attribute.AttributeRef-import org.openremote.model.datapoint.DatapointIntervalimport org.openremote.model.datapoint.query.AssetDatapointIntervalQueryimport org.openremote.model.util.ValueUtilimport org.openremote.test.ManagerContainerTraitdiff --git a/ui/component/or-icon/package.json b/ui/component/or-icon/package.jsonindex 5219f622b..f03b47d47 100644--- a/ui/component/or-icon/package.json+++ b/ui/component/or-icon/package.json@@ -24,7 +24,7 @@"@mdi/font": "^6.5.95","terser-webpack-plugin": "^5.2.5","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1"},"publishConfig": {diff --git a/ui/demo/demo-core/package.json b/ui/demo/demo-core/package.jsonindex 2727d6fea..dc9698271 100644--- a/ui/demo/demo-core/package.json+++ b/ui/demo/demo-core/package.json@@ -23,7 +23,7 @@"devDependencies": {"@openremote/util": "workspace:*","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/ui/demo/demo-or-asset-tree/package.json b/ui/demo/demo-or-asset-tree/package.jsonindex db55755db..42d1428dd 100644--- a/ui/demo/demo-or-asset-tree/package.json+++ b/ui/demo/demo-or-asset-tree/package.json@@ -19,7 +19,7 @@"devDependencies": {"@openremote/util": "workspace:*","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/ui/demo/demo-or-map/package.json b/ui/demo/demo-or-map/package.jsonindex f0acba7b0..05eca99b9 100644--- a/ui/demo/demo-or-map/package.json+++ b/ui/demo/demo-or-map/package.json@@ -19,7 +19,7 @@"devDependencies": {"@openremote/util": "workspace:*","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/ui/demo/demo-or-thermostat/package.json b/ui/demo/demo-or-thermostat/package.jsonindex cbc533918..4d9bd50d4 100644--- a/ui/demo/demo-or-thermostat/package.json+++ b/ui/demo/demo-or-thermostat/package.json@@ -19,7 +19,7 @@"devDependencies": {"@openremote/util": "workspace:*","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/ui/demo/demo-rest/package.json b/ui/demo/demo-rest/package.jsonindex 63241825d..d00061867 100644--- a/ui/demo/demo-rest/package.json+++ b/ui/demo/demo-rest/package.json@@ -20,7 +20,7 @@"devDependencies": {"@openremote/util": "workspace:*","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/ui/util/package.json b/ui/util/package.jsonindex b339e66f1..c902aa433 100644--- a/ui/util/package.json+++ b/ui/util/package.json@@ -40,7 +40,7 @@"style-loader": "^3.3.1","ts-loader": "^9.2.6","typescript": ">=4.5.2",- "webpack": "^5.64.3",+ "webpack": "^5.76.0","webpack-cli": "^4.9.1","webpack-dev-server": "^4.5.0"}diff --git a/yarn.lock b/yarn.lockindex 86087fad9..7cbf699ef 100644--- a/yarn.lock+++ b/yarn.lock@@ -2196,7 +2196,7 @@ __metadata:i18next: ^21.5.3lit: ^2.0.2typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0languageName: unknown@@ -2210,7 +2210,7 @@ __metadata:"@openremote/or-asset-tree": ^1.1.0"@openremote/util": "workspace:*"typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0languageName: unknown@@ -2224,7 +2224,7 @@ __metadata:"@openremote/or-map": ^1.1.0"@openremote/util": "workspace:*"typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0languageName: unknown@@ -2238,7 +2238,7 @@ __metadata:"@openremote/util": "workspace:*"source-map-loader: ^3.0.0typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0languageName: unknown@@ -2252,7 +2252,7 @@ __metadata:"@openremote/util": "workspace:*"lit: ^2.0.2typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0languageName: unknown@@ -2506,7 +2506,7 @@ __metadata:lit: ^2.0.2terser-webpack-plugin: ^5.2.5typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1languageName: unknownlinkType: soft@@ -2733,7 +2733,7 @@ __metadata:style-loader: ^3.3.1ts-loader: ^9.2.6typescript: ">=4.5.2"- webpack: ^5.64.3+ webpack: ^5.76.0webpack-cli: ^4.9.1webpack-dev-server: ^4.5.0bin:@@ -3439,7 +3439,7 @@ __metadata:languageName: nodelinkType: hard-"acorn@npm:^8.4.1, acorn@npm:^8.5.0, acorn@npm:^8.7.0":+"acorn@npm:^8.5.0, acorn@npm:^8.7.0":version: 8.7.1resolution: "acorn@npm:8.7.1"bin:@@ -3448,6 +3448,15 @@ __metadata:languageName: nodelinkType: hard+"acorn@npm:^8.7.1":+ version: 8.8.2+ resolution: "acorn@npm:8.8.2"+ bin:+ acorn: bin/acorn+ checksum: f790b99a1bf63ef160c967e23c46feea7787e531292bb827126334612c234ed489a0dc2c7ba33156416f0ffa8d25bf2b0fdb7f35c2ba60eb3e960572bece4001+ languageName: node+ linkType: hard+"agent-base@npm:6, agent-base@npm:^6.0.2":version: 6.0.2resolution: "agent-base@npm:6.0.2"@@ -4477,11 +4486,11 @@ __metadata:linkType: hard"dns-packet@npm:^5.2.2":- version: 5.3.1- resolution: "dns-packet@npm:5.3.1"+ version: 5.4.0+ resolution: "dns-packet@npm:5.4.0"dependencies:"@leichtgewicht/ip-codec": ^2.0.1- checksum: 196ff74a0669126cf5fc901a5849b72f625bd7a4cb163b3f4d41fbe19ed0b017cf7674daef5b0acbd448c094fcd795e501d7066f301be428e4acecfcf3c5f336+ checksum: a169963848e8539dfd8a19058562f9e1c15c0f82cbf76fa98942f11c46f3c74e7e7c82e3a8a5182d4c9e6ff19e21be738dbd098a876dde755d3aedd2cc730880languageName: nodelinkType: hard@@ -4611,7 +4620,7 @@ __metadata:languageName: nodelinkType: hard-"enhanced-resolve@npm:^5.0.0, enhanced-resolve@npm:^5.9.2":+"enhanced-resolve@npm:^5.0.0":version: 5.9.3resolution: "enhanced-resolve@npm:5.9.3"dependencies:@@ -4621,6 +4630,16 @@ __metadata:languageName: nodelinkType: hard+"enhanced-resolve@npm:^5.10.0":+ version: 5.12.0+ resolution: "enhanced-resolve@npm:5.12.0"+ dependencies:+ graceful-fs: ^4.2.4+ tapable: ^2.2.0+ checksum: bf3f787facaf4ce3439bef59d148646344e372bef5557f0d37ea8aa02c51f50a925cd1f07b8d338f18992c29f544ec235a8c64bcdb56030196c48832a5494174+ languageName: node+ linkType: hard+"entities@npm:^2.0.0":version: 2.2.0resolution: "entities@npm:2.2.0"@@ -6278,14 +6297,7 @@ __metadata:languageName: nodelinkType: hard-"json-parse-better-errors@npm:^1.0.2":- version: 1.0.2- resolution: "json-parse-better-errors@npm:1.0.2"- checksum: ff2b5ba2a70e88fd97a3cb28c1840144c5ce8fae9cbeeddba15afa333a5c407cf0e42300cd0a2885dbb055227fe68d405070faad941beeffbfde9cf3b2c78c5d- languageName: node- linkType: hard--"json-parse-even-better-errors@npm:^2.3.0":+"json-parse-even-better-errors@npm:^2.3.0, json-parse-even-better-errors@npm:^2.3.1":version: 2.3.1resolution: "json-parse-even-better-errors@npm:2.3.1"checksum: 798ed4cf3354a2d9ccd78e86d2169515a0097a5c133337807cdf7f1fc32e1391d207ccfc276518cc1d7d8d4db93288b8a50ba4293d212ad1336e52a8ec0a941f@@ -8899,13 +8911,13 @@ __metadata:languageName: nodelinkType: hard-"watchpack@npm:^2.3.1":- version: 2.3.1- resolution: "watchpack@npm:2.3.1"+"watchpack@npm:^2.4.0":+ version: 2.4.0+ resolution: "watchpack@npm:2.4.0"dependencies:glob-to-regexp: ^0.4.1graceful-fs: ^4.1.2- checksum: 70a34f92842d94b5d842980f866d568d7a467de667c96ae5759c759f46587e49265863171f4650bdbafc5f3870a28f2b4453e9e847098ec4b718b38926d47d22+ checksum: 23d4bc58634dbe13b86093e01c6a68d8096028b664ab7139d58f0c37d962d549a940e98f2f201cecdabd6f9c340338dc73ef8bf094a2249ef582f35183d1a131languageName: nodelinkType: hard@@ -9034,40 +9046,40 @@ __metadata:languageName: nodelinkType: hard-"webpack@npm:^5.64.3":- version: 5.72.0- resolution: "webpack@npm:5.72.0"+"webpack@npm:^5.76.0":+ version: 5.76.1+ resolution: "webpack@npm:5.76.1"dependencies:"@types/eslint-scope": ^3.7.3"@types/estree": ^0.0.51"@webassemblyjs/ast": 1.11.1"@webassemblyjs/wasm-edit": 1.11.1"@webassemblyjs/wasm-parser": 1.11.1- acorn: ^8.4.1+ acorn: ^8.7.1acorn-import-assertions: ^1.7.6browserslist: ^4.14.5chrome-trace-event: ^1.0.2- enhanced-resolve: ^5.9.2+ enhanced-resolve: ^5.10.0es-module-lexer: ^0.9.0eslint-scope: 5.1.1events: ^3.2.0glob-to-regexp: ^0.4.1graceful-fs: ^4.2.9- json-parse-better-errors: ^1.0.2+ json-parse-even-better-errors: ^2.3.1loader-runner: ^4.2.0mime-types: ^2.1.27neo-async: ^2.6.2schema-utils: ^3.1.0tapable: ^2.1.1terser-webpack-plugin: ^5.1.3- watchpack: ^2.3.1+ watchpack: ^2.4.0webpack-sources: ^3.2.3peerDependenciesMeta:webpack-cli:optional: truebin:webpack: bin/webpack.js- checksum: 8365f1466d0f7adbf80ebc9b780f263a28eeeabcd5fb515249bfd9a56ab7fe8d29ea53df3d9364d0732ab39ae774445eb28abce694ed375b13882a6b2fe93ffc+ checksum: b01fe0bc2dbca0e10d290ddb0bf81e807a031de48028176e2b21afd696b4d3f25ab9accdad888ef4a1f7c7f4d41f13d5bf2395b7653fdf3e5e3dafa54e56dab2languageName: nodelinkType: hard