100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Asset soft delete and background purge task (#2916)
Rich Turner360b903da99f
diff --git a/container/src/main/java/org/openremote/container/Container.java b/container/src/main/java/org/openremote/container/Container.javaindex 19c982551..1570b07ce 100644--- a/container/src/main/java/org/openremote/container/Container.java+++ b/container/src/main/java/org/openremote/container/Container.java@@ -136,8 +136,10 @@ public class Container implements org.openremote.model.Container {new ContainerThreadFactory("ContainerExecutor"),new ThreadPoolExecutor.CallerRunsPolicy());- LOG.log(INFO, EXECUTOR);- LOG.log(INFO, SCHEDULED_EXECUTOR);+ LOG.log(+ INFO,+ "Container executor threads min=" + executorThreadsMin + ", max=" + executorThreadsMax);+ LOG.log(INFO, "Container Scheduled executor threads=" + scheduledExecutorThreads);if (meterRegistry != null) {SCHEDULED_EXECUTOR =diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java b/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.javaindex f46a5265e..ee4a50558 100644--- a/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java@@ -31,6 +31,7 @@ import org.hibernate.Interceptor;import org.hibernate.Transaction;import org.hibernate.type.Type;import org.openremote.model.PersistenceEvent;+import org.openremote.model.asset.Asset;/** Intercept Hibernate lifecycle events and publish a message. */public class PersistenceEventInterceptor implements Interceptor {@@ -61,17 +62,47 @@ public class PersistenceEventInterceptor implements Interceptor {String[] propertyNames,Type[] types)throws CallbackException {- persistenceEvents.add(- new PersistenceEvent<>(- PersistenceEvent.Cause.UPDATE, entity, propertyNames, currentState, previousState));+ if (isAssetDeletePendingUpdate(entity, currentState, previousState, propertyNames)) {+ // Assets are soft deleted but we still want a delete event to maintain existing behaviour+ persistenceEvents.add(+ new PersistenceEvent<>(PersistenceEvent.Cause.DELETE, entity, null, currentState));+ } else {+ persistenceEvents.add(+ new PersistenceEvent<>(+ PersistenceEvent.Cause.UPDATE, entity, propertyNames, currentState, previousState));+ }+ return false;+ }++ protected boolean isAssetDeletePendingUpdate(+ Object entity, Object[] currentState, Object[] previousState, String[] propertyNames) {+ if (!(entity instanceof Asset<?>)+ || currentState == null+ || previousState == null+ || propertyNames == null) {+ return false;+ }++ for (int i = 0; i < propertyNames.length; i++) {+ if (i < currentState.length+ && i < previousState.length+ && Asset.DELETE_PENDING_PROPERTY_NAME.equals(propertyNames[i])) {+ return Boolean.TRUE.equals(currentState[i]) && !Boolean.TRUE.equals(previousState[i]);+ }+ }+return false;}@Overridepublic void onRemove(Object entity, Object id, Object[] state, String[] propertyNames, Type[] types) {- persistenceEvents.add(- new PersistenceEvent<>(PersistenceEvent.Cause.DELETE, entity, propertyNames, state));+ PersistenceEvent.Cause cause = PersistenceEvent.Cause.DELETE;+ if (entity instanceof Asset<?>) {+ // Assets are soft deleted first; physical removal is signalled separately.+ cause = PersistenceEvent.Cause.DELETE_FINISHED;+ }+ persistenceEvents.add(new PersistenceEvent<>(cause, entity, propertyNames, state));}@Overridediff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.javaindex 8c0abcd66..5da32bc8c 100644--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java@@ -355,6 +355,9 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc}openDatabase(container, database, dbUsername, dbPassword, connectionUrl);++ verifyTimescaleDbVersion();+prepareSchema(container, connectionUrl, dbUsername, dbPassword, dbSchema, dbName);}@@ -369,8 +372,6 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc@Overridepublic void start(Container container) throws Exception {- verifyTimescaleDbVersion();-// Register standard entity classes and also any Entity ClassProvidersList<String> entityClasses = new ArrayList<>(50);entityClasses.add(Asset.class.getName());@@ -509,6 +510,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistencswitch (cause) {case CREATE -> publishPersistenceEvent(cause, currentEntity, null, null, null);case DELETE -> publishPersistenceEvent(cause, previousEntity, null, null, null);+ case DELETE_FINISHED -> publishPersistenceEvent(cause, currentEntity, null, null, null);case UPDATE -> {List<String> propertyNames = new ArrayList<>(propertyFields.length);List<Object> currentState = new ArrayList<>(propertyFields.length);diff --git a/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java b/manager/src/main/java/org/openremote/manager/alarm/AlarmService.javaindex 3329378a8..6cb90b129 100644--- a/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java+++ b/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java@@ -444,8 +444,9 @@ public class AlarmService extends RouteBuilder implements ContainerService {entityManager.createQuery("""- select aal from AlarmAssetLink aal- where aal.id.realm = :realm and aal.id.sentalarmId = :alarmId+ select aal from AlarmAssetLink aal, Asset a+ where a.id = aal.id.assetId and a.deletePending is false+ and aal.id.realm = :realm and aal.id.sentalarmId = :alarmIdorder by aal.createdOn desc""",AlarmAssetLink.class)diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.javaindex ca315d514..316f9baf1 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java@@ -619,6 +619,9 @@ public class AssetResourceImpl extends ManagerWebResource implements AssetResourpublic Asset<?>[] queryAssets(RequestParams requestParams, AssetQuery query) {if (query == null) {query = new AssetQuery();+ } else {+ // Don't allow deleted assets in query+ query.includeDeletePending(false);}if (!assetStorageService.authorizeAssetQuery(query, getAuthContext(), getRequestRealmName())) {@@ -637,6 +640,9 @@ public class AssetResourceImpl extends ManagerWebResource implements AssetResourpublic AssetTree queryAssetTree(RequestParams requestParams, AssetQuery query) {if (query == null) {query = new AssetQuery();+ } else {+ // Don't allow deleted assets in query+ query.includeDeletePending(false);}if (!assetStorageService.authorizeAssetQuery(query, getAuthContext(), getRequestRealmName())) {diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.javaindex 00f01dc23..f21a662a7 100644--- a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java+++ b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java@@ -26,6 +26,7 @@ import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttribuimport static org.openremote.model.query.AssetQuery.*;import static org.openremote.model.query.AssetQuery.Access.*;import static org.openremote.model.query.filter.ValuePredicate.asPredicateOrTrue;+import static org.openremote.model.util.MapAccess.getInteger;import static org.openremote.model.util.TextUtil.isNullOrEmpty;import static org.openremote.model.value.MetaItemType.ACCESS_PUBLIC_READ;import static org.openremote.model.value.MetaItemType.ACCESS_RESTRICTED_READ;@@ -37,9 +38,17 @@ import jakarta.persistence.TypedQuery;import jakarta.validation.ConstraintViolation;import jakarta.validation.ConstraintViolationException;import java.sql.*;+import java.time.LocalDateTime;+import java.time.ZoneId;+import java.time.temporal.ChronoUnit;import java.util.*;+import java.util.concurrent.CompletableFuture;import java.util.concurrent.ExecutorService;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ScheduledFuture;+import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;import java.util.function.BiConsumer;import java.util.function.Consumer;import java.util.function.Predicate;@@ -75,6 +84,7 @@ import org.openremote.model.asset.impl.UnknownAsset;import org.openremote.model.attribute.Attribute;import org.openremote.model.attribute.AttributeEvent;import org.openremote.model.attribute.AttributeMap;+import org.openremote.model.datapoint.AssetDatapoint;import org.openremote.model.event.Event;import org.openremote.model.event.RespondableEvent;import org.openremote.model.event.shared.EventSubscription;@@ -109,6 +119,17 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic}}+ public record DeleteResult(boolean accepted, CompletableFuture<Boolean> completion) {++ public static DeleteResult rejected() {+ return new DeleteResult(false, CompletableFuture.completedFuture(false));+ }++ public static DeleteResult accepted(CompletableFuture<Boolean> completion) {+ return new DeleteResult(true, completion);+ }+ }+public interface ParameterBinderextends BiConsumer<EntityManager, org.hibernate.query.Query<Object>> {@@ -127,6 +148,13 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicprivate static final Logger LOG = Logger.getLogger(AssetStorageService.class.getName());public static final int PRIORITY = MED_PRIORITY;+ public static final String OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD =+ "OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD";+ public static final int OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD_DEFAULT = 100000;+ public static final String OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS =+ "OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS";+ public static final int OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS_DEFAULT = 4;+ protected static final long FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS = 1L;// protected static final Field assetParentNameField;// protected static final Field assetParentTypeField;@@ -237,7 +265,18 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicprotected ClientEventService clientEventService;protected GatewayService gatewayService;protected ExecutorService executorService;+ protected ScheduledExecutorService scheduledExecutorService;protected final LockByKey assetLocks = new LockByKey();+ protected final AtomicReference<CompletableFuture<Void>> pendingAssetDeleteFuture =+ new AtomicReference<>(CompletableFuture.completedFuture(null));+ protected final AtomicReference<LocalDateTime> oldestAssetDatapointChunkStart =+ new AtomicReference<>();+ protected final Set<String> failedAssetDeleteIds =+ Collections.synchronizedSet(new LinkedHashSet<>());+ protected ScheduledFuture<?> failedAssetDeleteRetryFuture;+ protected int assetDeleteDatapointBatchThreshold;+ protected int assetDeleteDatapointBatchWeeks;+ protected volatile boolean pendingAssetDeleteStopping;/*** Will evaluate each {@link CalendarEventPredicate} and apply it depending on the {@link@@ -331,6 +370,19 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicclientEventService = container.getService(ClientEventService.class);gatewayService = container.getService(GatewayService.class);executorService = container.getExecutor();+ scheduledExecutorService = container.getScheduledExecutor();+ assetDeleteDatapointBatchThreshold =+ getInteger(+ container.getConfig(),+ OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD,+ OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD_DEFAULT);+ assetDeleteDatapointBatchWeeks =+ Math.max(+ 1,+ getInteger(+ container.getConfig(),+ OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS,+ OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS_DEFAULT));EventSubscriptionAuthorizer assetEventAuthorizer =AssetStorageService.assetInfoAuthorizer(identityService, this);@@ -392,10 +444,26 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic}@Override- public void start(Container container) throws Exception {}+ public void start(Container container) throws Exception {+ pendingAssetDeleteStopping = false;+ failedAssetDeleteRetryFuture =+ scheduledExecutorService.scheduleAtFixedRate(+ this::retryFailedAssetDeletes,+ FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS,+ FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS,+ TimeUnit.HOURS);+ queueAssetsDeletion();+ }@Override- public void stop(Container container) throws Exception {}+ public void stop(Container container) throws Exception {+ pendingAssetDeleteStopping = true;+ pendingAssetDeleteFuture.get().cancel(false);+ if (failedAssetDeleteRetryFuture != null) {+ failedAssetDeleteRetryFuture.cancel(false);+ failedAssetDeleteRetryFuture = null;+ }+ }@SuppressWarnings("unchecked")@Override@@ -565,11 +633,12 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicpublic List<String> findNames(String... ids) {if (ids == null || ids.length == 0) return new ArrayList<>();- // TODO: Do this in a loop in reasonably sized batchesreturn persistenceService.doReturningTransaction(em -> {List<Object[]> result =- em.createQuery("select a.id, a.name from Asset a where a.id in :ids", Object[].class)+ em.createQuery(+ "select a.id, a.name from Asset a where a.id in :ids and a.deletePending is false",+ Object[].class).setParameter("ids", Arrays.asList(ids)).getResultList();List<String> names = new ArrayList<>();@@ -639,12 +708,17 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicthrows IllegalStateException, ConstraintViolationException {if (LOG.isLoggable(Level.FINEST)) {- LOG.finest("Merging asset: " + asset);+ LOG.finest("Asset merge: " + asset.toStringAll());+ } else {+ LOG.fine("Asset merge: " + asset);}long startTime = System.currentTimeMillis();String assetId = asset.getId();+ // Ensure delete pending flag is not set for merges+ asset.setDeletePending(false);+// We skip all standard checks as asset is coming from a gateway and would be validated from// thereif (requestingGatewayAsset != null) {@@ -703,6 +777,16 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicif (existingAsset != null) {+ boolean overrideVersionTemp = overrideVersion;++ if (existingAsset.isDeletePending()) {+ LOG.info("Cancelling pending asset deletion: assetId=" + assetId);+ existingAsset.setDeletePending(false);+ failedAssetDeleteIds.remove(assetId);+ em.flush();+ overrideVersionTemp = true;+ }+// Verify type has not been changedif (!existingAsset.getType().equals(asset.getType())) {String msg = "Asset type cannot be changed: asset=" + asset;@@ -717,10 +801,11 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic}// Update timestamp on modified attributes this allows fast equality checking+ T finalExistingAsset = existingAsset;asset.getAttributes().stream().forEach(attr ->- existingAsset+ finalExistingAsset.getAttribute(attr.getName()).ifPresent(existingAttr -> {@@ -744,7 +829,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic// version, so the detached state always wins and this update will go through// and ignore// concurrent updates- if (overrideVersion) {+ if (overrideVersionTemp) {asset.setVersion(existingAsset.getVersion());}}@@ -770,13 +855,13 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicAsset<?> parent = find(em, asset.getParentId(), true);// The parent must exist- if (parent == null) {+ if (parent == null || parent.isDeletePending()) {String msg = "Asset parent not found: asset=" + asset;LOG.warning(msg);throw new IllegalStateException(msg);}- // The parent can not be a child of the asset+ // The parent cannot be a child of the assetif (parent.pathContains(asset.getId())) {String msg ="Asset parent cannot be a descendant of the asset: asset=" + asset;@@ -906,11 +991,16 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicupdatedAsset = em.merge(asset);}- if (LOG.isLoggable(FINE)) {- LOG.fine(- "Asset merge took: " + (System.currentTimeMillis() - startTime) + "ms");+ String msgPrefix = existingAsset != null ? "Asset updated: " : "Asset created: ";+ if (LOG.isLoggable(Level.FINEST)) {+ LOG.finest(msgPrefix + asset.toStringAll());+ } else {+ LOG.info(msgPrefix + asset);}+ LOG.finest(+ () -> "Asset merge took: " + (System.currentTimeMillis() - startTime) + "ms");+if (user != null) {createUserAssetLinks(em,@@ -940,17 +1030,50 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic}/**- * @return <code>true</code> if the assets were deleted, false if any of the assets still have- * children and can't be deleted.+ * @return <code>true</code> if all assets were accepted for deletion and marked as delete+ * pending, <code>false</code> if any asset still has children, doesn't exist, or can't be+ * marked delete pending.*/public boolean delete(List<String> assetIds) {return delete(assetIds, false);}+ public boolean isDeletePending(String assetId) {+ if (isNullOrEmpty(assetId)) {+ throw new IllegalArgumentException("Can't query null or empty asset identifier");+ }++ Asset<?> asset =+ find(+ new AssetQuery()+ .ids(assetId)+ .includeDeletePending(true)+ .select(new Select().excludeAttributes()));++ return asset != null && asset.isDeletePending();+ }+public boolean delete(List<String> assetIds, boolean skipGatewayCheck) {+ return deleteWithResult(assetIds, skipGatewayCheck).accepted();+ }++ public CompletableFuture<Boolean> deleteUntilFinished(List<String> assetIds) {+ return deleteUntilFinished(assetIds, false);+ }++ public CompletableFuture<Boolean> deleteUntilFinished(+ List<String> assetIds, boolean skipGatewayCheck) {+ DeleteResult result = deleteWithResult(assetIds, skipGatewayCheck);+ return result.accepted() ? result.completion() : CompletableFuture.completedFuture(false);+ }++ public DeleteResult deleteWithResult(List<String> assetIds, boolean skipGatewayCheck) {List<String> ids = new ArrayList<>(assetIds);+ LOG.info(+ () -> "Asset deletion: count=" + assetIds.size() + ", IDs=" + String.join(", ", assetIds));+if (!skipGatewayCheck) {// Don't allow deletion of gateway descendant assets (they must be deleted on the gateway@@ -970,68 +1093,469 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicif (!gatewayIds.isEmpty()) {// Handle gateway asset deletion in a special way+ LOG.fine("Deleting gateway assets: IDs=" + String.join(",", gatewayIds));ids.removeAll(gatewayIds);+ CompletableFuture<Boolean> gatewayDeleteFuture = CompletableFuture.completedFuture(true);for (String gatewayId : gatewayIds) {+ DeleteResult gatewayDeleteResult;try {- boolean deleted = gatewayService.deleteGateway(gatewayId);- if (!deleted) {- return false;- }+ gatewayDeleteResult = gatewayService.deleteGatewayUntilFinished(gatewayId);} catch (Exception e) {LOG.log(WARNING, "Failed to delete gateway asset: " + gatewayId, e);- return false;+ return DeleteResult.rejected();}+ if (!gatewayDeleteResult.accepted()) {+ return DeleteResult.rejected();+ }+ gatewayDeleteFuture =+ gatewayDeleteFuture.thenCombine(+ gatewayDeleteResult+ .completion()+ .exceptionally(+ ex -> {+ LOG.log(WARNING, "Failed to delete gateway asset: " + gatewayId, ex);+ return false;+ }),+ (deleted, gatewayDeleted) -> deleted && gatewayDeleted);+ }+ DeleteResult assetDeleteResult = markAssetsForDeletionAndQueue(ids, assetIds);+ if (!assetDeleteResult.accepted()) {+ return DeleteResult.rejected();}+ return DeleteResult.accepted(+ gatewayDeleteFuture.thenCombine(+ assetDeleteResult.completion(),+ (gatewaysDeleted, assetsDeleted) -> gatewaysDeleted && assetsDeleted));}}- if (!ids.isEmpty()) {- try {- // Get locks for each asset ID- ids.forEach(assetLocks::lock);+ return markAssetsForDeletionAndQueue(ids, assetIds);+ }- persistenceService.doTransaction(- em -> {- List<Asset<?>> assets =- em- .createQuery(- "select a from Asset a where not exists(select child.id from Asset child where child.parentId = a.id and not child.id in :ids) and a.id in :ids",- Asset.class)- .setParameter("ids", ids)- .getResultList()- .stream()- .map(asset -> (Asset<?>) asset)- .collect(Collectors.toList());-- if (ids.size() != assets.size()) {- throw new IllegalArgumentException(- "Cannot delete one or more requested assets as they either have children or don't exist");- }+ /**+ * Marks assets as pending deletion and queues them for actual deletion.+ *+ * @return DeleteResult indicating whether the deletion was accepted or rejected+ */+ protected DeleteResult markAssetsForDeletionAndQueue(List<String> ids, List<String> assetIds) {+ if (ids.isEmpty()) {+ return DeleteResult.accepted(CompletableFuture.completedFuture(true));+ }++ List<String> orderedAssetIdsToQueue;+ try {+ ids.forEach(assetLocks::lock);- assets.sort(- Comparator.comparingInt(- (Asset<?> asset) -> asset.getPath() == null ? 0 : asset.getPath().length)- .reversed());+ List<Asset<?>> assets = new ArrayList<>();+ persistenceService.doTransaction(+ em -> {+ assets.addAll(+ em+ .createQuery(+ "select a from Asset a where not exists(select child.id from Asset child where child.parentId = a.id and child.deletePending is false and not child.id in :ids) and a.id in :ids",+ Asset.class)+ .setParameter("ids", ids)+ .getResultList()+ .stream()+ .map(asset -> (Asset<?>) asset)+ .sorted(+ Comparator.comparingInt(+ (Asset<?> asset) ->+ asset.getPath() != null ? asset.getPath().length : 0)+ .reversed()+ .thenComparing(Asset::getCreatedOn))+ .toList());++ if (ids.size() != assets.size()) {+ throw new IllegalArgumentException(+ "Cannot delete one or more requested assets as they either have children or don't exist");+ }- // TODO: Remove when https://github.com/timescale/timescaledb/issues/9916 is fixed- // and the minimum supported TimescaleDB version includes that fix.- em.createNativeQuery("SET LOCAL plan_cache_mode = force_custom_plan").executeUpdate();+ assets.forEach(asset -> asset.setDeletePending(true));+ });++ orderedAssetIdsToQueue = assets.stream().map(Asset::getId).toList();+ } catch (Exception e) {+ LOG.log(+ SEVERE,+ "Failed to delete one or more requested assets: " + Arrays.toString(assetIds.toArray()),+ e);+ return DeleteResult.rejected();+ } finally {+ // Release batch locks BEFORE queueing background worker tasks+ ids.forEach(assetLocks::unlock);+ }++ return DeleteResult.accepted(queueAssetsDeletion(orderedAssetIdsToQueue));+ }++ /** Queue deletion of all assets marked as pending deletion */+ protected void queueAssetsDeletion() {+ queueAssetsDeletion(findPendingDeleteAssetIds());+ }++ /**+ * Queue deletion of the specified ordered asset IDs+ *+ * @return CompletableFuture indicating whether the etire deletion batch succeeded+ */+ protected CompletableFuture<Boolean> queueAssetsDeletion(List<String> orderedAssetIds) {+ if (pendingAssetDeleteStopping) {+ return CompletableFuture.completedFuture(false);+ }++ if (orderedAssetIds.isEmpty()) {+ return CompletableFuture.completedFuture(true);+ }++ CompletableFuture<Boolean> deleteResult = new CompletableFuture<>();+ Map<String, Long> datapointCounts = findAssetDatapointCounts(orderedAssetIds);+ LOG.fine(+ "Scheduling asset deletion: count="+ + orderedAssetIds.size()+ + ", IDs="+ + String.join(", ", orderedAssetIds));++ pendingAssetDeleteFuture.updateAndGet(+ existingFuture ->+ existingFuture+ .exceptionally(ex -> null)+ .thenRunAsync(+ () -> {+ boolean allDeleted = true;+ for (String assetId : orderedAssetIds) {+ if (pendingAssetDeleteStopping) {+ deleteResult.complete(false);+ return;+ }+ allDeleted =+ doAssetDeletion(assetId, datapointCounts.getOrDefault(assetId, 0L))+ && allDeleted;+ }+ deleteResult.complete(allDeleted);+ },+ executorService)+ .exceptionally(+ ex -> {+ LOG.log(SEVERE, "Failed to process queued asset deletions", ex);+ deleteResult.complete(false);+ return null;+ }));++ return deleteResult;+ }++ /** Re-attempts batch deletion of all assets that previously failed deletion. */+ protected void retryFailedAssetDeletes() {+ if (pendingAssetDeleteStopping) {+ return;+ }++ List<String> failedAssetIds;+ synchronized (failedAssetDeleteIds) {+ if (failedAssetDeleteIds.isEmpty()) {+ return;+ }+ failedAssetIds = new ArrayList<>(failedAssetDeleteIds);+ failedAssetDeleteIds.clear();+ }++ LOG.fine("Retrying failed pending asset deletes: count=" + failedAssetIds.size());+ requestPendingAssetDeletionRetry(failedAssetIds);+ }++ /** Re-attempts batch deletion of the specified assets. */+ protected void requestPendingAssetDeletionRetry(List<String> assetIds) {+ if (pendingAssetDeleteStopping || assetIds.isEmpty()) {+ return;+ }++ pendingAssetDeleteFuture.updateAndGet(+ existingFuture ->+ existingFuture+ .exceptionally(ex -> null)+ .thenRunAsync(+ () -> {+ for (String assetId : assetIds) {+ if (pendingAssetDeleteStopping) {+ return;+ }+ deletePendingAsset(assetId);+ }+ },+ executorService));+ }++ /**+ * Get all assets from the database that are marked as pending deletion+ *+ * @return List of asset IDs that are pending deletion+ */+ @SuppressWarnings("unchecked")+ protected List<String> findPendingDeleteAssetIds() {+ return persistenceService.doReturningTransaction(+ em ->+ (List<String>)+ em.createNativeQuery(+ "select id from asset where delete_pending is true order by nlevel(path) desc, created_on asc")+ .getResultList());+ }++ protected boolean deleteAssetDatapointsBeforeAsset(long datapointCount) {+ return datapointCount > assetDeleteDatapointBatchThreshold;+ }++ /**+ * Get the datapoint counts for all the provided asset IDs+ *+ * @return map of asset ID and datapoint count+ */+ protected Map<String, Long> findAssetDatapointCounts(Collection<String> assetIds) {+ if (assetIds.isEmpty()) {+ return Collections.emptyMap();+ }- assets.forEach(em::remove);- em.flush();+ long start = System.currentTimeMillis();+ Map<String, Long> datapointCounts =+ persistenceService.doReturningTransaction(+ em -> {+ @SuppressWarnings("unchecked")+ List<Object[]> result =+ em.createNativeQuery(+ "select entity_id, count(*) from "+ + AssetDatapoint.TABLE_NAME+ + " where entity_id in (:assetIds) group by entity_id")+ .setParameter("assetIds", assetIds)+ .getResultList();++ return result.stream()+ .collect(+ Collectors.toMap(+ row -> (String) row[0], row -> ((Number) row[1]).longValue()));});++ LOG.finer(+ "Counted asset datapoints for pending asset delete: assetCount="+ + assetIds.size()+ + ", durationMillis="+ + (System.currentTimeMillis() - start));+ return datapointCounts;+ }++ /**+ * Handles the actual deletion of an asset by batching the datapoint deletion if count is above+ * threshold; otherwise the asset is just deleted and FK cascading ensures datapoints are+ * correctly deleted.+ *+ * @return True if the asset was successfully deleted+ */+ protected boolean doAssetDeletion(String assetId, long datapointCount) {+ long start = System.currentTimeMillis();++ if (deleteAssetDatapointsBeforeAsset(datapointCount)) {+ LOG.fine(+ "Purging asset datapoints: assetId="+ + assetId+ + ", datapoints="+ + datapointCount+ + ", threshold="+ + assetDeleteDatapointBatchThreshold);+ try {+ if (deleteAssetDatapoints(assetId)) {+ failedAssetDeleteIds.remove(assetId);+ return true;+ }+ LOG.fine(+ "Purged asset datapoints: assetId="+ + assetId+ + ", durationMillis="+ + (System.currentTimeMillis() - start));} catch (Exception e) {- LOG.log(- SEVERE,- "Failed to delete one or more requested assets: " + Arrays.toString(assetIds.toArray()),- e);- return false;- } finally {- // Release all of the locks- ids.forEach(assetLocks::unlock);+ LOG.log(WARNING, "Purge asset datapoints failed: assetId=" + assetId, e);}}- return true;++ if (pendingAssetDeleteStopping) {+ return !isDeletePending(assetId);+ }++ if (!isDeletePending(assetId)) {+ failedAssetDeleteIds.remove(assetId);+ return true;+ }++ return deletePendingAsset(assetId);+ }++ /**+ * Purges the Asset entity from the database only if no ancestor assets have failed deletion.+ *+ * @return True if the asset was successfully deleted+ */+ protected boolean deletePendingAsset(String assetId) {+ if (pendingAssetDeleteStopping) {+ return false;+ }++ return withAssetLock(+ assetId,+ () -> {+ if (pendingAssetDeleteStopping) {+ return false;+ }++ LOG.fine("Deleting asset: assetId=" + assetId);+ AtomicReference<Boolean> blockedByFailedPath = new AtomicReference<>(false);++ try {+ persistenceService.doTransaction(+ em -> {+ useCustomQueryPlansForTimescaleDelete(em);+ Asset<?> asset = em.find(Asset.class, assetId);++ if (asset == null || !asset.isDeletePending()) {+ return;+ }++ synchronized (failedAssetDeleteIds) {+ if (blockedByFailedPath.updateAndGet(+ blocked ->+ failedAssetDeleteIds.stream()+ .anyMatch(+ failedAssetId ->+ !failedAssetId.equals(assetId)+ && asset.pathContains(failedAssetId)))) {+ return;+ }+ }++ em.remove(asset);+ em.flush();+ });++ if (blockedByFailedPath.get()) {+ failedAssetDeleteIds.add(assetId);+ LOG.fine(+ "Asset delete skipped because a failed pending delete asset is in its path: assetId="+ + assetId);+ return false;+ }++ failedAssetDeleteIds.remove(assetId);+ return true;+ } catch (Exception e) {+ failedAssetDeleteIds.add(assetId);+ // TODO: Raise an alarm for asset deletion failure.+ LOG.log(+ SEVERE, "Failed to delete pending asset, queued for retry: assetId=" + assetId, e);+ return false;+ }+ });+ }++ // TODO: Remove when https://github.com/timescale/timescaledb/issues/9916 is fixed+ // and the minimum supported TimescaleDB version includes that fix.+ protected void useCustomQueryPlansForTimescaleDelete(EntityManager em) {+ em.createNativeQuery("SET LOCAL plan_cache_mode = force_custom_plan").executeUpdate();+ }++ /**+ * Purges the datapoints for the asset in batches; checks the asset pending deletion flag between+ * executions.+ *+ * @return True if the asset is no longer pending deletion+ */+ protected boolean deleteAssetDatapoints(String assetId) {+ LocalDateTime oldestChunkStart = findOldestAssetDatapointChunkStart();++ if (oldestChunkStart == null) {+ LOG.fine("No asset datapoint chunks found for pending asset delete: assetId=" + assetId);+ return !isDeletePending(assetId);+ }++ if (pendingAssetDeleteStopping) {+ return !isDeletePending(assetId);+ }++ LocalDateTime rangeStart = oldestChunkStart;+ LocalDateTime finalEnd = LocalDateTime.ofInstant(timerService.getNow(), ZoneId.systemDefault());++ while (rangeStart.isBefore(finalEnd) && !pendingAssetDeleteStopping) {+ LocalDateTime nextEnd =+ rangeStart.plus((long) assetDeleteDatapointBatchWeeks * 7L, ChronoUnit.DAYS);+ LocalDateTime rangeEnd = nextEnd.isBefore(finalEnd) ? nextEnd : finalEnd;+ LocalDateTime batchStart = rangeStart;+ LocalDateTime batchEnd = rangeEnd;+ long start = System.currentTimeMillis();++ int deleted =+ persistenceService.doReturningTransaction(+ em -> {+ useCustomQueryPlansForTimescaleDelete(em);+ return em.createNativeQuery(+ "delete from "+ + AssetDatapoint.TABLE_NAME+ + " where entity_id = :assetId and timestamp >= :rangeStart and timestamp < :rangeEnd")+ .setParameter("assetId", assetId)+ .setParameter("rangeStart", batchStart)+ .setParameter("rangeEnd", batchEnd)+ .executeUpdate();+ });++ if (LOG.isLoggable(FINE)) {+ LOG.fine(+ "Deleted asset datapoints for pending asset: assetId="+ + assetId+ + ", chunkRangeStart="+ + batchStart+ + ", chunkRangeEnd="+ + batchEnd+ + ", rows="+ + deleted+ + ", durationMillis="+ + (System.currentTimeMillis() - start));+ }++ if (!isDeletePending(assetId)) {+ failedAssetDeleteIds.remove(assetId);+ return true;+ }++ rangeStart = rangeEnd;+ }++ return !isDeletePending(assetId);+ }++ protected LocalDateTime findOldestAssetDatapointChunkStart() {+ LocalDateTime cachedStart = oldestAssetDatapointChunkStart.get();+ if (cachedStart != null) {+ return cachedStart;+ }++ LocalDateTime start =+ persistenceService.doReturningTransaction(+ em ->+ em.unwrap(Session.class)+ .createNativeQuery(+ """+ select min(range_start::timestamp)+ from timescaledb_information.chunks+ where hypertable_schema = current_schema()+ and hypertable_name = :hypertableName+ and range_start is not null+ """,+ LocalDateTime.class)+ .setParameter("hypertableName", AssetDatapoint.TABLE_NAME)+ .getSingleResult());++ if (start == null) {+ return null;+ }++ return oldestAssetDatapointChunkStart.compareAndSet(null, start)+ ? start+ : oldestAssetDatapointChunkStart.get();}public boolean isUserAsset(String assetId) {@@ -1121,7 +1645,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServictry {return entityManager.createQuery(- "select count(a) from Asset a where a.realm = :realm and a.id in :assetIds",+ "select count(a) from Asset a where a.realm = :realm and a.id in :assetIds and a.deletePending is false",Long.class).setParameter("realm", realm).setParameter("assetIds", assetIds)@@ -1133,7 +1657,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic});}- /** Returns the realm for each asset ID that exists. */+ /** Returns the realm for each asset ID that exists and not pending deletion. */public Map<String, String> getAssetRealms(Collection<String> assetIds) {if (assetIds == null || assetIds.isEmpty()) {return Map.of();@@ -1145,7 +1669,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic.createQuery("""select a.id, a.realm from Asset a- where a.id in :assetIds+ where a.id in :assetIds and a.deletePending is false""",Object[].class).setParameter("assetIds", assetIds)@@ -1171,7 +1695,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicpublic Boolean execute(Connection connection) throws SQLException {try (PreparedStatement st =connection.prepareStatement(- "select count(*) from Asset a where a.path ~ lquery(?) AND a.id = ANY(?)")) {+ "select count(*) from Asset a where a.path ~ lquery(?) AND a.id = ANY(?) AND a.delete_pending is false")) {st.setString(1, "*." + parentAssetId + ".*");st.setArray(2, st.getConnection().createArrayOf("text", assetIds.toArray()));@@ -1188,7 +1712,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic/*** Returns a map of asset IDs with the respective hasChildren flag.*- * <p>It check whether any of the assetIds is set as a parentId in the Asset table.+ * <p>It checks whether any of the assetIds is set as a parentId in the Asset table.** @param assetIds The list of asset IDs to check.* @return A map of asset IDs with the respective hasChildren flag.@@ -1200,7 +1724,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicList<String> parentsWithChildren =entityManager.createQuery(- "select distinct a.parentId from Asset a where a.parentId in :assetIds",+ "select distinct a.parentId from Asset a where a.parentId in :assetIds and a.deletePending is false",String.class).setParameter("assetIds", assetIds).getResultList();@@ -1269,7 +1793,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicEntityManager em, String realm, Collection<String> userIds, Collection<String> assetIds) {StringBuilder sb = new StringBuilder();Map<String, Object> parameters = new HashMap<>(3);- sb.append("select ua from UserAssetLink ua where 1=1");+ sb.append("select ua from UserAssetLink ua, Asset a where a.id = ua.id.assetId");+ sb.append(" and a.deletePending is false");if (!isNullOrEmpty(realm)) {sb.append(" and ua.id.realm in :realm");@@ -1646,7 +2171,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic// https://stackoverflow.com/questions/53847917/postgresql-throws-column-is-of-type-jsonb-but-expression-is-of-type-bytea-with)Query query =em.createNativeQuery(- "UPDATE asset SET attributes[?] = attributes[?] || ?\\:\\:jsonb where id = ?")+ "UPDATE asset SET attributes[?] = attributes[?] || ?\\:\\:jsonb where id = ? and delete_pending is false").setParameter(1, event.getName()).setParameter(2, event.getName()).setParameter(@@ -1701,11 +2226,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicif (loadedAsset == null) {return;}- if (LOG.isLoggable(Level.FINEST)) {- LOG.finest("Asset created: " + loadedAsset.toStringAll());- } else {- LOG.fine("Asset created: " + loadedAsset);- }clientEventService.publishEvent(new AssetEvent(AssetEvent.Cause.CREATE, loadedAsset, null));// Raise attribute event for each created attribute@@ -1729,7 +2249,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicpersistenceEvent.getPropertyNames().size() > 1|| !persistenceEvent.hasPropertyChanged("attributes");boolean attributesChanged = persistenceEvent.hasPropertyChanged("attributes");- LOG.finest(() -> "Asset updated: " + persistenceEvent);clientEventService.publishEvent(new AssetEvent(@@ -1801,11 +2320,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic});}case DELETE -> {- if (LOG.isLoggable(Level.FINEST)) {- LOG.finest("Asset deleted: " + asset.toStringAll());- } else {- LOG.fine("Asset deleted: " + asset);- }clientEventService.publishEvent(new AssetEvent(AssetEvent.Cause.DELETE, asset, null));// Raise attribute event with deleted flag for each attribute@@ -1882,7 +2396,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicStringBuilder sb = new StringBuilder();AssetQuery.Select select = query.select;- sb.append("select A.ID as ID, A.NAME as NAME, A.ACCESS_PUBLIC_READ as ACCESS_PUBLIC_READ");+ sb.append(+ "select A.ID as ID, A.NAME as NAME, A.ACCESS_PUBLIC_READ as ACCESS_PUBLIC_READ, A.DELETE_PENDING as DELETE_PENDING");sb.append(", A.CREATED_ON AS CREATED_ON, A.TYPE AS TYPE, A.PARENT_ID AS PARENT_ID");sb.append(", A.REALM AS REALM, A.VERSION as VERSION");@@ -2106,6 +2621,10 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicsb.append(" and A.ACCESS_PUBLIC_READ is true");}+ if (!query.isIncludeDeletePending()) {+ sb.append(" and A.DELETE_PENDING is false");+ }+if (query.types != null) {String[] resolvedTypes = getResolvedAssetTypes(query.types);final int pos = binders.size() + 1;@@ -2585,6 +3104,9 @@ public class AssetStorageService extends RouteBuilder implements ContainerServicprotected <T extends HasAssetQuery & RespondableEvent> void onReadRequest(T event) {AssetQuery assetQuery = event.getAssetQuery();+ // Ensure that deleted assets are not included in the result+ assetQuery.includeDeletePending(false);+Event response = null;if (event.getResponseConsumer() == null) {diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.javaindex 47182ba41..da1650bb7 100644--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java@@ -118,7 +118,7 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi}if (maxDatapointAgeWeeks <= 0) {- LOG.warning("Data point purge disabled");+ LOG.info("Data point purge disabled");} else {LOG.log(Level.INFO, "Data point purge retention = " + maxDatapointAgeWeeks + " weeks");}diff --git a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.javaindex 29cf729f4..81ae740cc 100644--- a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java+++ b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java@@ -544,7 +544,7 @@ public class GatewayService extends RouteBuilder implements ContainerService {return false;}- public boolean deleteGateway(String gatewayId) {+ public AssetStorageService.DeleteResult deleteGatewayUntilFinished(String gatewayId) {GatewayConnector connector = gatewayConnectorMap.get(gatewayId.toLowerCase(Locale.ROOT));if (connector == null) {@@ -565,8 +565,8 @@ public class GatewayService extends RouteBuilder implements ContainerService {.collect(Collectors.toList());gatewayAssetIds.add(gatewayId);-- return assetStorageService.delete(gatewayAssetIds, true);+ LOG.info("Deleting gateway asset and descendants: IDs=" + String.join(",", gatewayAssetIds));+ return assetStorageService.deleteWithResult(gatewayAssetIds, true);}public Collection<GatewayTunnelInfo> getTunnelInfos() {diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.javaindex 0089dfe5d..9dae772e6 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java@@ -608,6 +608,10 @@ public class RulesService extends RouteBuilder implements ContainerService {// Must reload from the database, the ruleset might not be completely hydrated on CREATE or// UPDATEAssetRuleset assetRuleset = rulesetStorageService.find(AssetRuleset.class, ruleset.getId());+ if (assetRuleset == null) {+ undeployAssetRuleset((AssetRuleset) ruleset);+ return;+ }RulesEngine<AssetRuleset> engine = deployAssetRuleset(assetRuleset);engine.start();}diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java b/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.javaindex a8e009244..46544e478 100644--- a/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java+++ b/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java@@ -191,6 +191,10 @@ public class RulesetStorageService implements ContainerService {StringBuilder sb, Class<T> rulesetType, RulesetQuery query, List<ParameterBinder> binders) {sb.append(" WHERE 1=1");+ if (rulesetType == AssetRuleset.class) {+ sb.append(" AND A.DELETE_PENDING IS FALSE");+ }+if (query.publicOnly) {sb.append(" AND R.ACCESS_PUBLIC_READ");}diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.javaindex fd02b007a..6cfa228fd 100644--- a/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java+++ b/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java@@ -136,7 +136,8 @@ public interface ManagerIdentityProvider extends IdentityProvider {// BUILD FROMsb.append(" FROM public.user_entity u join PUBLIC.REALM r on r.ID = u.REALM_ID");if (userQuery.assets != null || userQuery.pathPredicate != null) {- sb.append(" join user_asset_link ua on ua.user_id = u.id");+ sb.append(+ " join user_asset_link ua on ua.user_id = u.id join asset a on a.id = ua.asset_id and a.delete_pending is false");}// BUILD WHEREdiff --git a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.javaindex aab94d610..8308777fe 100644--- a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java+++ b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java@@ -66,7 +66,6 @@ import org.openremote.manager.event.ClientEventService;import org.openremote.model.Constants;import org.openremote.model.Container;import org.openremote.model.PersistenceEvent;-import org.openremote.model.asset.Asset;import org.openremote.model.auth.OAuthGrant;import org.openremote.model.auth.OAuthPasswordGrant;import org.openremote.model.gateway.GatewayConnection;@@ -1193,7 +1192,17 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProviderthrow new NotFoundException("Realm does not exist: " + realmName);}+ int assetCount =+ assetStorageService.count(+ new AssetQuery().realm(new RealmPredicate(realmName)).includeDeletePending(true));++ if (assetCount > 0) {+ throw new IllegalStateException(+ "Cannot delete realm '" + realmName + "' because it still contains assets");+ }+realmCache.invalidate(realmName);+persistenceService.doTransaction(entityManager -> {@@ -1228,18 +1237,6 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider+ "where rs.realm = ?1");query.setParameter(1, realmName);query.executeUpdate();-- // Delete Assets- List<String> assetIds =- assetStorageService- .findAll(- new AssetQuery()- .select(new AssetQuery.Select().excludeAttributes())- .realm(new RealmPredicate(realmName)))- .stream()- .map(Asset::getId)- .toList();- assetStorageService.delete(assetIds);});LOG.fine("Deleting realm: " + realmName);diff --git a/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java b/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.javaindex 73ba56283..5d023d3f4 100644--- a/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java+++ b/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java@@ -137,6 +137,8 @@ public class RealmResourceImpl extends ManagerWebResource implements RealmResouridentityService.getIdentityProvider().deleteRealm(realm);} catch (ClientErrorException ex) {throw new WebApplicationException(ex.getCause(), ex.getResponse().getStatus());+ } catch (IllegalStateException ex) {+ throw new WebApplicationException(ex, Response.Status.CONFLICT);} catch (Exception ex) {throw new WebApplicationException(ex);}diff --git a/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java b/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.javaindex ac0d8202a..a93660553 100644--- a/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java+++ b/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java@@ -22,17 +22,24 @@ import static org.openremote.container.security.keycloak.KeycloakIdentityProvideimport static org.openremote.model.Constants.*;import java.util.Arrays;+import java.util.List;+import java.util.concurrent.ExecutionException;import java.util.logging.Logger;+import org.openremote.manager.asset.AssetStorageService;import org.openremote.model.Container;+import org.openremote.model.asset.Asset;+import org.openremote.model.query.AssetQuery;import org.openremote.model.query.UserQuery;import org.openremote.model.query.filter.RealmPredicate;public class KeycloakCleanSetup extends AbstractKeycloakSetup {private static final Logger LOG = Logger.getLogger(KeycloakCleanSetup.class.getName());+ private final AssetStorageService assetStorageService;public KeycloakCleanSetup(Container container) {super(container);+ this.assetStorageService = container.getService(AssetStorageService.class);}@Override@@ -48,6 +55,7 @@ public class KeycloakCleanSetup extends AbstractKeycloakSetup {.forEach(realm -> {if (!realm.getName().equals(MASTER_REALM)) {+ deleteRealmAssets(realm.getName());keycloakProvider.deleteRealm(realm.getName());}});@@ -86,4 +94,37 @@ public class KeycloakCleanSetup extends AbstractKeycloakSetup {keycloakProvider.deleteIdentityProvider(MASTER_REALM, idp.getAlias());});}++ protected void deleteRealmAssets(String realm) {+ List<String> assetIds =+ assetStorageService+ .findAll(+ new AssetQuery()+ .select(new AssetQuery.Select().excludeAttributes())+ .realm(new RealmPredicate(realm))+ .includeDeletePending(true))+ .stream()+ .map(Asset::getId)+ .toList();++ if (assetIds.isEmpty()) {+ return;+ }++ LOG.info("Deleting all assets in realm '" + realm + "': count=" + assetIds.size());+ boolean deleted;+ try {+ deleted = assetStorageService.deleteUntilFinished(assetIds, true).get();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(+ "Interrupted while deleting all assets in realm '" + realm + "'", e);+ } catch (ExecutionException e) {+ throw new IllegalStateException("Failed to delete all assets in realm '" + realm + "'", e);+ }++ if (!deleted) {+ throw new IllegalStateException("Failed to delete all assets in realm '" + realm + "'");+ }+ }}diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sqlnew file mode 100644index 000000000..67872c055--- /dev/null+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sql@@ -0,0 +1,2 @@+ALTER TABLE ${schemaName}.asset+ ADD COLUMN IF NOT EXISTS delete_pending boolean NOT NULL DEFAULT false;diff --git a/model/src/main/java/org/openremote/model/PersistenceEvent.java b/model/src/main/java/org/openremote/model/PersistenceEvent.javaindex 8588291cf..55f705da9 100644--- a/model/src/main/java/org/openremote/model/PersistenceEvent.java+++ b/model/src/main/java/org/openremote/model/PersistenceEvent.java@@ -26,7 +26,8 @@ public class PersistenceEvent<T> {public enum Cause {CREATE,UPDATE,- DELETE+ DELETE,+ DELETE_FINISHED}protected final Cause cause;@@ -109,7 +110,7 @@ public class PersistenceEvent<T> {+ ", entity="+ entity+ ", propertyNames="- + String.join(",", propertyNames)+ + (propertyNames != null ? String.join(",", propertyNames) : "null")+ ", currentState="+ Arrays.toString(currentState)+ ", previousState="diff --git a/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java b/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.javaindex af173aa9c..9a2d60083 100644--- a/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java+++ b/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java@@ -104,7 +104,7 @@ public class AlarmAssetLink {protected String assetName;@Formula(- "(select pa.NAME from ASSET a left outer join ASSET pa on a.PARENT_ID = pa.ID where a.ID = ASSET_ID)")+ "(select pa.NAME from ASSET a left outer join ASSET pa on a.PARENT_ID = pa.ID and pa.DELETE_PENDING is false where a.ID = ASSET_ID)")protected String parentAssetName;protected AlarmAssetLink() {}diff --git a/model/src/main/java/org/openremote/model/asset/Asset.java b/model/src/main/java/org/openremote/model/asset/Asset.javaindex f9712517e..16eb98240 100644--- a/model/src/main/java/org/openremote/model/asset/Asset.java+++ b/model/src/main/java/org/openremote/model/asset/Asset.java@@ -20,6 +20,8 @@ package org.openremote.model.asset;import static jakarta.persistence.DiscriminatorType.STRING;+import com.fasterxml.jackson.annotation.JsonIgnore;+import com.fasterxml.jackson.annotation.JsonInclude;import com.fasterxml.jackson.annotation.JsonTypeInfo;import com.fasterxml.jackson.core.JsonParser;import com.fasterxml.jackson.databind.DeserializationContext;@@ -262,6 +264,7 @@ import org.openremote.model.value.ValueType;"Polymorphic OpenRemote asset. The type discriminator selects a registered asset subtype; attributes are keyed by attribute name.")public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>, AssetInfo {+ public static final String DELETE_PENDING_PROPERTY_NAME = "deletePending";private static final java.util.regex.Pattern ASSET_ID_PATTERN =java.util.regex.Pattern.compile(Constants.ASSET_ID_REGEXP);@@ -364,6 +367,9 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>example = "false")protected boolean accessPublicRead;+ @Column(name = "DELETE_PENDING", nullable = false)+ protected boolean deletePending;+@Column(name = "PARENT_ID", length = 22, columnDefinition = "char(22)")@Pattern(regexp = Constants.ASSET_ID_REGEXP, message = "{Asset.parentId.Pattern}") @Schema(description = "Parent asset identifier. Omit for a root asset.",@@ -460,6 +466,17 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>return (T) this;}+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)+ public boolean isDeletePending() {+ return deletePending;+ }++ @JsonIgnore+ public T setDeletePending(boolean deletePending) {+ this.deletePending = deletePending;+ return (T) this;+ }+public T setParent(Asset<?> parent) {if (parent == null) {parentId = null;@@ -736,6 +753,7 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>Asset<?> asset = (Asset<?>) o;return version == asset.version&& accessPublicRead == asset.accessPublicRead+ && deletePending == asset.deletePending&& Objects.equals(id, asset.id)&& Objects.equals(createdOn, asset.createdOn)&& Objects.equals(name, asset.name)@@ -754,6 +772,7 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>createdOn,name,accessPublicRead,+ deletePending,parentId,realm,type,diff --git a/model/src/main/java/org/openremote/model/asset/AssetResource.java b/model/src/main/java/org/openremote/model/asset/AssetResource.javaindex 0abeb7a0c..db616548b 100644--- a/model/src/main/java/org/openremote/model/asset/AssetResource.java+++ b/model/src/main/java/org/openremote/model/asset/AssetResource.java@@ -661,7 +661,7 @@ public interface AssetResource {@Produces(APPLICATION_JSON)@Operation(operationId = "queryCount",- summary = "Count assets using a query",+ summary = "Count assets using a query (can also be used to count assets pending deletion)",description ="Returns only the number of assets matching an AssetQuery after applying the caller's realm, public, and linked-asset access constraints.")@OpenApiResponses.Okdiff --git a/model/src/main/java/org/openremote/model/query/AssetQuery.java b/model/src/main/java/org/openremote/model/query/AssetQuery.javaindex db2136b0d..e7047f467 100644--- a/model/src/main/java/org/openremote/model/query/AssetQuery.java+++ b/model/src/main/java/org/openremote/model/query/AssetQuery.java@@ -209,6 +209,7 @@ public class AssetQuery implements Serializable {public String[] types;public LogicGroup<AttributePredicate> attributes;+ public Boolean includeDeletePending;// Orderingpublic OrderBy orderBy;@@ -425,6 +426,15 @@ public class AssetQuery implements Serializable {return attributeValue(name, new NumberPredicate(d, operator));}+ public AssetQuery includeDeletePending(boolean includeDeletePending) {+ this.includeDeletePending = includeDeletePending;+ return this;+ }++ public boolean isIncludeDeletePending() {+ return includeDeletePending != null && includeDeletePending;+ }+public AssetQuery orderBy(OrderBy orderBy) {this.orderBy = orderBy;return this;@@ -463,6 +473,8 @@ public class AssetQuery implements Serializable {+ Arrays.toString(types)+ ", attribute="+ (attributes != null ? attributes.toString() : "null")+ + ", includeDeletePending="+ + includeDeletePending+ ", orderBy="+ orderBy+ ", limit="diff --git a/model/src/main/java/org/openremote/model/security/RealmResource.java b/model/src/main/java/org/openremote/model/security/RealmResource.javaindex 187879c77..7084a9070 100644--- a/model/src/main/java/org/openremote/model/security/RealmResource.java+++ b/model/src/main/java/org/openremote/model/security/RealmResource.java@@ -151,6 +151,9 @@ public interface RealmResource {@OpenApiResponses.Authenticated@OpenApiResponses.NotFound@ApiResponse(responseCode = "405", description = "The master realm cannot be deleted")+ @ApiResponse(+ responseCode = "409",+ description = "Realm cannot be deleted while it still contains assets")void delete(@BeanParam RequestParams requestParams,@Parameter(description = REALM, example = EXAMPLE_REALM) @PathParam("name") String realm);diff --git a/model/src/main/java/org/openremote/model/util/LockByKey.java b/model/src/main/java/org/openremote/model/util/LockByKey.javaindex 07f70d3ad..a1c6b9fce 100644--- a/model/src/main/java/org/openremote/model/util/LockByKey.java+++ b/model/src/main/java/org/openremote/model/util/LockByKey.java@@ -52,6 +52,7 @@ public class LockByKey {}public void lock(String key) {+ LOG.log(System.Logger.Level.TRACE, () -> "Lock acquiring: key=" + key);LockWrapper lockWrapper =locks.compute(key, (k, v) -> v == null ? createLockWrapper() : v.addThreadInQueue());try {diff --git a/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy b/test/src/main/groovy/org/openremote/test/ContainerTrait.groovyindex da09e262c..95e6f942c 100644--- a/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy+++ b/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy@@ -64,6 +64,7 @@ import org.openremote.model.util.MapAccessimport org.slf4j.Loggerimport org.slf4j.LoggerFactoryimport spock.lang.Shared+import spock.lang.Timeoutimport java.util.concurrent.TimeUnitimport java.util.concurrent.atomic.AtomicReference@@ -86,6 +87,7 @@ trait ContainerTrait {Logger LOG = LoggerFactory.getLogger(ContainerTrait.class)+ @Timeout(120)Container startContainer(Map<String, String> config, Iterable<ContainerService> services) {// Reset and start clock in case any previous tests stopped/modified it (pseudo clock is static so shared between tests)@@ -222,7 +224,7 @@ trait ContainerTrait {if (!currentAssets.isEmpty()) {LOG.info("Purging ${currentAssets.size()} asset(s)")- assetStorageService.delete(currentAssets, true)+ assetStorageService.deleteUntilFinished(currentAssets, true).get(60, TimeUnit.SECONDS)}// Wait for all assets to be unlinked from protocolsdiff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovyindex f27de9197..fc1956340 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy@@ -18,15 +18,20 @@*/package org.openremote.test.assets+import org.apache.camel.builder.RouteBuilderimport org.hibernate.cfg.AvailableSettings+import org.openremote.container.message.MessageBrokerServiceimport org.openremote.container.persistence.PersistenceServiceimport org.openremote.manager.asset.AssetStorageServiceimport org.openremote.manager.datapoint.AssetDatapointServiceimport org.openremote.manager.setup.SetupService+import org.openremote.model.PersistenceEvent+import org.openremote.model.asset.Assetimport org.openremote.model.asset.impl.ThingAssetimport org.openremote.model.attribute.Attributeimport org.openremote.model.attribute.AttributeRefimport org.openremote.model.datapoint.ValueDatapoint+import org.openremote.model.query.AssetQueryimport org.openremote.model.value.ValueTypeimport org.openremote.setup.integration.KeycloakTestSetupimport org.openremote.test.ManagerContainerTrait@@ -35,8 +40,11 @@ import spock.util.concurrent.PollingConditionsimport java.time.Instantimport java.time.temporal.ChronoUnit+import java.util.concurrent.CopyOnWriteArrayListimport java.util.concurrent.TimeUnit+import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC+import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityTypeimport static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_WEEKSclass AssetDatapointPurgeTest extends Specification implements ManagerContainerTrait {@@ -51,10 +59,43 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerTconfig.put(OR_DATA_POINTS_MAX_AGE_WEEKS, "4")def container = startContainer(config, defaultServices())def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)- def assetStorageService = container.getService(AssetStorageService.class)+ def originalAssetStorageService = container.getService(AssetStorageService.class)+ def assetStorageService = Spy(originalAssetStorageService)+ [email protected](AssetStorageService.class, assetStorageService)def assetDatapointService = container.getService(AssetDatapointService.class)+ def messageBrokerService = container.getService(MessageBrokerService.class)def persistenceService = container.getService(PersistenceService.class)def originalMaxTuplesDecompressedPerDmlTransaction = getMaxTuplesDecompressedPerDmlTransaction(persistenceService)+ def originalAssetDeleteDatapointBatchThreshold = assetStorageService.assetDeleteDatapointBatchThreshold+ def originalAssetDeleteDatapointBatchWeeks = assetStorageService.assetDeleteDatapointBatchWeeks+ assetStorageService.assetDeleteDatapointBatchThreshold = 100+ assetStorageService.assetDeleteDatapointBatchWeeks = 2++ def failedDeleteAssetIds = new CopyOnWriteArrayList<String>()+ assetStorageService.deletePendingAsset(_) >> { String assetId ->+ if (failedDeleteAssetIds.contains(assetId)) {+ assetStorageService.failedAssetDeleteIds.add(assetId)+ return+ }+ callRealMethod()+ }++ List<PersistenceEvent<Asset<?>>> assetPersistenceEvents = new CopyOnWriteArrayList<>()+ def assetPersistenceRouteId = "Test-AssetPersistenceEvents-${System.nanoTime()}"+ messageBrokerService.context.addRoutes(new RouteBuilder() {+ @Override+ void configure() {+ from(PERSISTENCE_TOPIC)+ .routeId(assetPersistenceRouteId)+ .filter(isPersistenceEventForEntityType(Asset.class))+ .process { exchange ->+ def event = exchange.in.getBody(PersistenceEvent.class) as PersistenceEvent<Asset<?>>+ if (event.cause in [PersistenceEvent.Cause.DELETE, PersistenceEvent.Cause.DELETE_FINISHED]) {+ assetPersistenceEvents.add(event)+ }+ }+ }+ })and: "the schema name is retrieved"def schemaName = persistenceService.persistenceUnitProperties.getProperty(AvailableSettings.DEFAULT_SCHEMA)@@ -207,12 +248,45 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerTwhen: "the segment-delete assets are deleted from compressed chunks with a low decompression limit"setMaxTuplesDecompressedPerDmlTransaction(persistenceService, 100)+ def failedSegmentDeleteAssetId = segmentDeleteAssetIds.first()+ failedDeleteAssetIds.add(failedSegmentDeleteAssetId)def segmentDeleteSucceeded = assetStorageService.delete(segmentDeleteAssetIds)- def segmentDeleteDatapointsAfterDelete = countDatapoints(persistenceService, segmentDeleteAssetIds, attributeNames)then: "TimescaleDB should delete full segments without decompressing chunks"segmentDeleteSucceeded- segmentDeleteDatapointsAfterDelete == 0++ and: "asset DELETE events should be published for all assets and DELETE_FINISHED for all but the failed asset"+ conditions.eventually {+ assert countDatapoints(persistenceService, segmentDeleteAssetIds, attributeNames) == 0++ def deleteIds = assetPersistenceEvents+ .findAll { it.cause == PersistenceEvent.Cause.DELETE }+ .collect { it.entity.id }+ def deleteFinishedIds = assetPersistenceEvents+ .findAll { it.cause == PersistenceEvent.Cause.DELETE_FINISHED }+ .collect { it.entity.id }++ assert segmentDeleteAssetIds.every { deleteIds.contains(it) }+ assert (segmentDeleteAssetIds - failedSegmentDeleteAssetId).every {+ deleteFinishedIds.contains(it)+ }+ assert !deleteFinishedIds.contains(failedSegmentDeleteAssetId)+ }++ and: "the failed asset should be hidden from asset queries"+ assetStorageService.find(new AssetQuery().ids(failedSegmentDeleteAssetId)) == null++ when: "the failed asset deletion is retried"+ failedDeleteAssetIds.remove(failedSegmentDeleteAssetId)+ assetStorageService.retryFailedAssetDeletes()++ then: "the failed asset should be physically deleted"+ conditions.eventually {+ assert assetPersistenceEvents.any {+ it.cause == PersistenceEvent.Cause.DELETE_FINISHED && it.entity.id == failedSegmentDeleteAssetId+ }+ assert countAssets(persistenceService, failedSegmentDeleteAssetId) == 0L+ }when: "Purging will be called, count datapoints before purging"def countBeforePurge = 0@@ -292,7 +366,28 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerTthen: "No exception should have occurred"true- cleanup: "restore TimescaleDB decompression limit"+ cleanup: "restore settings and remove the temporary route"+ if (messageBrokerService != null && assetPersistenceRouteId != null) {+ try {+ messageBrokerService.context.routeController.stopRoute(assetPersistenceRouteId)+ } catch (Exception ignored) {+ // Route may not have been added if setup failed early.+ }+ try {+ messageBrokerService.context.removeRoute(assetPersistenceRouteId)+ } catch (Exception ignored) {+ // Route may already be gone during container shutdown.+ }+ }+ if (assetStorageService != null && originalAssetDeleteDatapointBatchThreshold != null) {+ assetStorageService.assetDeleteDatapointBatchThreshold = originalAssetDeleteDatapointBatchThreshold+ }+ if (assetStorageService != null && originalAssetDeleteDatapointBatchWeeks != null) {+ assetStorageService.assetDeleteDatapointBatchWeeks = originalAssetDeleteDatapointBatchWeeks+ }+ if (container != null && originalAssetStorageService != null) {+ [email protected](AssetStorageService.class, originalAssetStorageService)+ }if (persistenceService != null && originalMaxTuplesDecompressedPerDmlTransaction != null) {setMaxTuplesDecompressedPerDmlTransaction(persistenceService, originalMaxTuplesDecompressedPerDmlTransaction)}@@ -316,6 +411,14 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerT}}+ private static long countAssets(PersistenceService persistenceService, String assetId) {+ return persistenceService.doReturningTransaction { em ->+ def query = em.createNativeQuery("SELECT count(*) FROM asset WHERE id = :assetId")+ query.setParameter("assetId", assetId)+ return (query.getSingleResult() as Number).longValue()+ }+ }+private static String getMaxTuplesDecompressedPerDmlTransaction(PersistenceService persistenceService) {return persistenceService.doReturningTransaction { em ->em.createNativeQuery("SHOW timescaledb.max_tuples_decompressed_per_dml_transaction").getSingleResult() as Stringdiff --git a/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovyindex 31d72689d..d03b0f8d3 100644--- a/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy+++ b/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy@@ -34,6 +34,7 @@ import org.openremote.model.asset.impl.ThingAssetimport org.openremote.model.attribute.Attributeimport org.openremote.model.attribute.AttributeStateimport org.openremote.model.attribute.AttributeEvent+import org.openremote.model.query.AssetQueryimport org.openremote.model.util.UniqueIdentifierGeneratorimport org.openremote.model.value.ValueTypeimport org.openremote.setup.integration.KeycloakTestSetup@@ -505,6 +506,81 @@ class AssetIntegrityTest extends Specification implements ManagerContainerTrait(assetStorageService.find(gatewayDescendant.id, true) as RoomAsset).parentId == gateway.id}+ def "Asset deletion allows children that are already pending deletion"() {+ given: "the server container is started"+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)+ def container = startContainer(defaultConfig(), defaultServices())+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)+ def originalAssetStorageService = container.getService(AssetStorageService.class)+ def assetStorageService = Spy(originalAssetStorageService)+ [email protected](AssetStorageService.class, assetStorageService)++ and: "physical deletion is held back for selected assets"+ def failedDeleteAssetIds = Collections.synchronizedSet(new HashSet<String>())+ assetStorageService.deletePendingAsset(_) >> { String assetId ->+ if (failedDeleteAssetIds.contains(assetId)) {+ assetStorageService.failedAssetDeleteIds.add(assetId)+ return false+ }+ callRealMethod()+ }++ and: "there is a parent with a child"+ def parentAsset = assetStorageService.merge(new RoomAsset("Pending delete parent")+ .setRealm(keycloakTestSetup.realmMaster.name))+ def childAsset = assetStorageService.merge(new RoomAsset("Pending delete child")+ .setRealm(keycloakTestSetup.realmMaster.name)+ .setParentId(parentAsset.id))++ when: "the child is marked for deletion but physical deletion has not completed"+ failedDeleteAssetIds.add(childAsset.id)+ def childAccepted = assetStorageService.delete([childAsset.id])++ then: "the child is pending deletion so can only be queried through the count include delete pending endpoint"+ childAccepted+ conditions.eventually {+ assert assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 0+ assert assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(childAsset.id)) == 1+ }++ when: "the parent is deleted while the child is still pending deletion"+ failedDeleteAssetIds.add(parentAsset.id)+ def parentAccepted = assetStorageService.delete([parentAsset.id])++ then: "the parent is accepted for deletion"+ parentAccepted+ conditions.eventually {+ assert assetStorageService.count(new AssetQuery().ids(parentAsset.id)) == 0+ assert assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(parentAsset.id)) == 1+ }++ when: "the child asset is re-merged while it is still pending deletion"+ failedDeleteAssetIds.remove(childAsset.id)+ childAsset.setName("Restored pending delete child")+ def restoredChildAsset = assetStorageService.merge(childAsset)++ then: "the child deletion is cancelled and the asset is visible again"+ restoredChildAsset.id == childAsset.id+ restoredChildAsset.name == "Restored pending delete child"+ !restoredChildAsset.isDeletePending()+ conditions.eventually {+ assert assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 1+ assert !assetStorageService.failedAssetDeleteIds.contains(childAsset.id)+ }++ when: "pending asset deletion is retried"+ assetStorageService.requestPendingAssetDeletionRetry([childAsset.id])+ TimeUnit.MILLISECONDS.sleep(500)++ then: "the restored child is not physically deleted by the background deletion task"+ assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 1+ assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(childAsset.id)) == 1++ and: "the parent remains pending deletion"+ assetStorageService.count(new AssetQuery().ids(parentAsset.id)) == 0+ assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(parentAsset.id)) == 1+ }+def "Test writing attributes with timestamps"() {given: "the server container is started"def container = startContainer(defaultConfig(), defaultServices())diff --git a/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy b/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovyindex 87d6d394c..1afab8827 100644--- a/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy+++ b/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy@@ -18,26 +18,34 @@*/package org.openremote.test.rules+import jakarta.ws.rs.WebApplicationException+import org.openremote.manager.asset.AssetStorageServiceimport org.openremote.manager.rules.RulesEngineimport org.openremote.manager.rules.RulesServiceimport org.openremote.manager.rules.RulesetStorageServiceimport org.openremote.manager.security.ManagerIdentityServiceimport org.openremote.manager.setup.SetupService+import org.openremote.model.query.AssetQuery+import org.openremote.model.query.filter.RealmPredicateimport org.openremote.setup.integration.KeycloakTestSetupimport org.openremote.setup.integration.ManagerTestSetupimport org.openremote.model.rules.AssetRulesetimport org.openremote.model.rules.GlobalRulesetimport org.openremote.model.rules.RealmRuleset+import org.openremote.model.security.RealmResourceimport org.openremote.test.ManagerContainerTraitimport spock.lang.Specificationimport spock.util.concurrent.PollingConditions+import java.util.concurrent.TimeUnit+import static org.openremote.model.util.MapAccess.getStringimport static org.openremote.manager.security.ManagerIdentityProvider.OR_ADMIN_PASSWORDimport static org.openremote.manager.security.ManagerIdentityProvider.OR_ADMIN_PASSWORD_DEFAULTimport static org.openremote.model.Constants.*import static org.openremote.model.rules.Ruleset.Lang.GROOVYimport static org.openremote.model.rules.RulesetStatus.*+import static jakarta.ws.rs.core.Response.Status.CONFLICTclass BasicRulesDeploymentTest extends Specification implements ManagerContainerTrait {@@ -53,6 +61,7 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainerdef keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)def rulesService = container.getService(RulesService.class)def identityService = container.getService(ManagerIdentityService.class)+ def assetStorageService = container.getService(AssetStorageService.class)def rulesetStorageService = container.getService(RulesetStorageService.class)and: "some test rulesets have been imported"@@ -66,6 +75,7 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainerMASTER_REALM_ADMIN_USER,getString(container.getConfig(), OR_ADMIN_PASSWORD, OR_ADMIN_PASSWORD_DEFAULT))+ def realmResource = getClientApiTarget(serverUri(serverPort), MASTER_REALM, accessToken).proxy(RealmResource.class)expect: "the rules engines to be ready"conditions.eventually {@@ -297,8 +307,25 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainerassert !realmBuildingEngine.isError()}- when: "a realm is deleted"- identityService.getIdentityProvider().deleteRealm(realmBuilding.getName())+ when: "a realm containing assets is deleted"+ realmResource.delete(null, realmBuilding.getName())++ then: "the delete is rejected with a conflict"+ WebApplicationException ex = thrown()+ try {+ assert ex.response.status == CONFLICT.statusCode+ } finally {+ ex.response.close()+ }++ when: "the realm assets are deleted and the realm delete is retried"+ def buildingAssetIds = assetStorageService.findAll(new AssetQuery()+ .select(new AssetQuery.Select().excludeAttributes())+ .realm(new RealmPredicate(realmBuilding.getName())))+ .collect { it.id }++ assert assetStorageService.deleteUntilFinished(buildingAssetIds).get(10, TimeUnit.SECONDS)+ realmResource.delete(null, realmBuilding.getName())then: "the realms rule engine should stop and all asset rule engines in this realm should also stop"conditions.eventually {diff --git a/ui/app/manager/src/pages/page-realms.ts b/ui/app/manager/src/pages/page-realms.tsindex 7bd9bc70e..f5c02710c 100644--- a/ui/app/manager/src/pages/page-realms.ts+++ b/ui/app/manager/src/pages/page-realms.ts@@ -529,7 +529,8 @@ export class PageRealms extends Page<AppStateKeyed> {this._realms = this._realms.filter((r) => r !== realm);this.dispatchEvent(new CustomEvent("realms-changed", { bubbles: true, composed: true }));} catch (e) {- showSnackbar(undefined, "realmDeleteFailed", "dismiss");+ const snackbarKey = (e as any)?.response?.status === 409 ? "realmDeleteFailedHasAssets" : "realmDeleteFailed";+ showSnackbar(undefined, snackbarKey, "dismiss");}};diff --git a/ui/app/manager/test/fixtures/manager.ts b/ui/app/manager/test/fixtures/manager.tsindex 37ba1ea33..33426ea91 100644--- a/ui/app/manager/test/fixtures/manager.ts+++ b/ui/app/manager/test/fixtures/manager.ts@@ -565,6 +565,52 @@ export class Manager {}}+ /**+ * Deletes all assets in a realm+ * @param realm The realm+ * @param config The axios request config+ */+ async deleteAssetsInRealm(realm: string, config?: AxiosRequestConfig<any>) {+ config = config ?? (await this.adminConfig());++ const assetIdRetreiver = async () => {+ const assetResponse = await rest.api.AssetResource.queryAssets(+ {+ select: { attributes: [] },+ realm: { name: realm },+ },+ config+ );+ expect(assetResponse.status).toBe(200);+ const assets = assetResponse.data;+ return assets.map(({ id }) => id!);+ };++ let assetIds = await assetIdRetreiver();++ while (assetIds.length > 0) {+ const response = await rest.api.AssetResource.delete({ assetId: assetIds }, config);+ expect(response.status).toBe(204);+ // Wait for these assets to be deleted+ await expect+ .poll(async () => {+ const countResponse = await rest.api.AssetResource.queryCount(+ {+ select: { attributes: [] },+ ids: assetIds,+ realm: { name: realm },+ includeDeletePending: true,+ },+ config+ );+ expect(countResponse.status).toBe(200);+ return countResponse.data;+ })+ .toBe(0);+ assetIds = await assetIdRetreiver();+ }+ }+/*** Delete role*diff --git a/ui/app/manager/test/test.cleanup.ts b/ui/app/manager/test/test.cleanup.tsindex ea2f4c38a..1584d93d8 100644--- a/ui/app/manager/test/test.cleanup.ts+++ b/ui/app/manager/test/test.cleanup.ts@@ -48,6 +48,7 @@ cleanup(`Delete the "smartcity" user`, async ({ page, manager }) => {* @and The realm picker should no longer be visible*/cleanup(`Delete the "smartcity" realm`, async ({ page, manager, realmsPage }) => {+ await manager.deleteAssetsInRealm("smartcity");await manager.goToRealmStartPage("master");await manager.navigateToMenuItem("Realms");await realmsPage.deleteRealm("smartcity");diff --git a/ui/app/shared/locales/cs/or.json b/ui/app/shared/locales/cs/or.jsonindex ba0b7a5f4..1f78bcfff 100644--- a/ui/app/shared/locales/cs/or.json+++ b/ui/app/shared/locales/cs/or.json@@ -830,7 +830,7 @@"attributeValueLabel": "Hodnota atributu","assetTypeDoesNotExist": "Filtr: Typ assetu neexistuje"},- "realmDeleteConfirm": "Varování! Tímto smažete vše v tomto realmu, pro potvrzení zadejte název realmu „{{realmName}}“.",+ "realmDeleteConfirm": "Varování! Realm nesmí obsahovat žádná aktiva, aby jej bylo možné smazat, tímto automaticky smažete vše ostatní v tomto realmu, pro potvrzení zadejte název realmu „{{realmName}}“.","realmDeleteFailed": "Nepodařilo se smazat realm","addRule": "Přidat pravidlo","rulesetStatus": {diff --git a/ui/app/shared/locales/en/or.json b/ui/app/shared/locales/en/or.jsonindex c869cc36a..0be4b945e 100644--- a/ui/app/shared/locales/en/or.json+++ b/ui/app/shared/locales/en/or.json@@ -830,8 +830,9 @@"attributeValueLabel": "Attribute value","assetTypeDoesNotExist": "Filter : Asset type does not exist"},- "realmDeleteConfirm": "Warning! this will delete everything in this realm, type the realm name '{{realmName}}' to confirm.",+ "realmDeleteConfirm": "Warning! Realm must contain no assets to allow deletion, this will delete everything else in this realm automatically, type the realm name '{{realmName}}' to confirm.","realmDeleteFailed": "Failed to delete realm",+ "realmDeleteFailedHasAssets": "Failed to delete realm because it still contains assets","addRule": "Add rule","rulesetStatus": {"DEPLOYED": "Successfully deployed",diff --git a/ui/app/shared/locales/nl/or.json b/ui/app/shared/locales/nl/or.jsonindex 904e309ff..01f9b9a7f 100644--- a/ui/app/shared/locales/nl/or.json+++ b/ui/app/shared/locales/nl/or.json@@ -820,8 +820,9 @@"attributeValueLabel": "Attribuutwaarde","assetTypeDoesNotExist": "Filter: Asset type bestaat niet"},- "realmDeleteConfirm": "Let op! Hiermee verwijder je alles in deze realm, typ de realm naam '{{realmName}}' om te bevestigen.",+ "realmDeleteConfirm": "Let op! De realm mag geen assets bevatten om te kunnen worden verwijderd, hiermee verwijder je automatisch al het andere in deze realm, typ de realm naam '{{realmName}}' om te bevestigen.","realmDeleteFailed": "Realm verwijderen mislukt",+ "realmDeleteFailedHasAssets": "Realm verwijderen mislukt omdat deze nog assets bevat","addRule": "Regel toevoegen","rulesetStatus": {"DEPLOYED": "Succesvol aangemaakt",diff --git a/ui/app/shared/locales/pl/or.json b/ui/app/shared/locales/pl/or.jsonindex 3cb1b9efc..ef42a3ced 100644--- a/ui/app/shared/locales/pl/or.json+++ b/ui/app/shared/locales/pl/or.json@@ -1102,7 +1102,7 @@"attributeValueLabel": "Wartość atrybutu","assetTypeDoesNotExist": "Filtr: typ zasobu nie istnieje"},- "realmDeleteConfirm": "Uwaga! Spowoduje to usunięcie wszystkich danych w tym obszarze. Wpisz nazwę obszaru '{{realmName}}', aby potwierdzić.",+ "realmDeleteConfirm": "Uwaga! Obszar nie może zawierać żadnych zasobów, aby można go było usunąć. Spowoduje to automatyczne usunięcie wszystkich pozostałych elementów w tym obszarze, wpisz nazwę obszaru '{{realmName}}', aby potwierdzić.","realmDeleteFailed": "Nie udało się usunąć obszaru","addRule": "Dodaj regułę","rulesetStatus": {diff --git a/ui/app/shared/locales/ro/or.json b/ui/app/shared/locales/ro/or.jsonindex 6a5ca0555..0d6e8949f 100644--- a/ui/app/shared/locales/ro/or.json+++ b/ui/app/shared/locales/ro/or.json@@ -574,7 +574,7 @@"attributeValueLabel": "Valoarea atributului","assetTypeDoesNotExist": "Filtrare : Tipul de obiect nu exista"},- "realmDeleteConfirm": "Avertisment! se va sterge totul în acest domeniu, scrie numele domeniului '{{realmName}}' pentru a confirma.",+ "realmDeleteConfirm": "Avertisment! Domeniul nu trebuie să conțină active pentru a putea fi șters, această acțiune va șterge automat tot ce se mai află în acest domeniu, scrie numele domeniului '{{realmName}}' pentru a confirma.","realmDeleteFailed": "Stergerea domeniului esuata","rulesetStatus": {"DEPLOYED": "Instalat cu succes",diff --git a/ui/app/shared/locales/uk/or.json b/ui/app/shared/locales/uk/or.jsonindex a17e5ddeb..a605e5b9f 100644--- a/ui/app/shared/locales/uk/or.json+++ b/ui/app/shared/locales/uk/or.json@@ -847,7 +847,7 @@"attributeValueLabel": "Значення атрибута","assetTypeDoesNotExist": "Фільтр: Тип активу не існує"},- "realmDeleteConfirm": "Попередження! Це видалить усе в цьому реалмі, введіть назву реалму '{{realmName}}', щоб підтвердити.",+ "realmDeleteConfirm": "Попередження! Реалм не повинен містити активів, щоб його можна було видалити, це автоматично видалить усе інше в цьому реалмі, введіть назву реалму '{{realmName}}', щоб підтвердити.","realmDeleteFailed": "Не вдалося видалити реалм","addRule": "Додати правило","rulesetStatus": {diff --git a/ui/test/app.config.cts b/ui/test/app.config.ctsindex 4378bf555..62f640c0b 100644--- a/ui/test/app.config.cts+++ b/ui/test/app.config.cts@@ -84,6 +84,7 @@ export const defineAppConfig = (path: string) => {video: "on",locale: "en",},+ timeout: 60_000,webServer: {command: `node ${join(__dirname, "manager.cjs")}`,url: "http://127.0.0.1:8080",