Skip to content

armgit / core

Public Imported from GitHub
0

100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps

Asset soft delete and background purge task (#2916)

Rich Turner360b903da99f
diff --git a/container/src/main/java/org/openremote/container/Container.java b/container/src/main/java/org/openremote/container/Container.java
index 19c982551..1570b07ce 100644
--- a/container/src/main/java/org/openremote/container/Container.java
+++ b/container/src/main/java/org/openremote/container/Container.java
@@ -136,8 +136,10 @@ public class Container implements org.openremote.model.Container {
new ContainerThreadFactory("ContainerExecutor"),
new ThreadPoolExecutor.CallerRunsPolicy());
- LOG.log(INFO, EXECUTOR);
- LOG.log(INFO, SCHEDULED_EXECUTOR);
+ LOG.log(
+ INFO,
+ "Container executor threads min=" + executorThreadsMin + ", max=" + executorThreadsMax);
+ LOG.log(INFO, "Container Scheduled executor threads=" + scheduledExecutorThreads);
if (meterRegistry != null) {
SCHEDULED_EXECUTOR =
diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java b/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java
index f46a5265e..ee4a50558 100644
--- a/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java
+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceEventInterceptor.java
@@ -31,6 +31,7 @@ import org.hibernate.Interceptor;
import org.hibernate.Transaction;
import org.hibernate.type.Type;
import org.openremote.model.PersistenceEvent;
+import org.openremote.model.asset.Asset;
/** Intercept Hibernate lifecycle events and publish a message. */
public class PersistenceEventInterceptor implements Interceptor {
@@ -61,17 +62,47 @@ public class PersistenceEventInterceptor implements Interceptor {
String[] propertyNames,
Type[] types)
throws CallbackException {
- persistenceEvents.add(
- new PersistenceEvent<>(
- PersistenceEvent.Cause.UPDATE, entity, propertyNames, currentState, previousState));
+ if (isAssetDeletePendingUpdate(entity, currentState, previousState, propertyNames)) {
+ // Assets are soft deleted but we still want a delete event to maintain existing behaviour
+ persistenceEvents.add(
+ new PersistenceEvent<>(PersistenceEvent.Cause.DELETE, entity, null, currentState));
+ } else {
+ persistenceEvents.add(
+ new PersistenceEvent<>(
+ PersistenceEvent.Cause.UPDATE, entity, propertyNames, currentState, previousState));
+ }
+ return false;
+ }
+
+ protected boolean isAssetDeletePendingUpdate(
+ Object entity, Object[] currentState, Object[] previousState, String[] propertyNames) {
+ if (!(entity instanceof Asset<?>)
+ || currentState == null
+ || previousState == null
+ || propertyNames == null) {
+ return false;
+ }
+
+ for (int i = 0; i < propertyNames.length; i++) {
+ if (i < currentState.length
+ && i < previousState.length
+ && Asset.DELETE_PENDING_PROPERTY_NAME.equals(propertyNames[i])) {
+ return Boolean.TRUE.equals(currentState[i]) && !Boolean.TRUE.equals(previousState[i]);
+ }
+ }
+
return false;
}
@Override
public void onRemove(
Object entity, Object id, Object[] state, String[] propertyNames, Type[] types) {
- persistenceEvents.add(
- new PersistenceEvent<>(PersistenceEvent.Cause.DELETE, entity, propertyNames, state));
+ PersistenceEvent.Cause cause = PersistenceEvent.Cause.DELETE;
+ if (entity instanceof Asset<?>) {
+ // Assets are soft deleted first; physical removal is signalled separately.
+ cause = PersistenceEvent.Cause.DELETE_FINISHED;
+ }
+ persistenceEvents.add(new PersistenceEvent<>(cause, entity, propertyNames, state));
}
@Override
diff --git a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
index 8c0abcd66..5da32bc8c 100644
--- a/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
+++ b/container/src/main/java/org/openremote/container/persistence/PersistenceService.java
@@ -355,6 +355,9 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
}
openDatabase(container, database, dbUsername, dbPassword, connectionUrl);
+
+ verifyTimescaleDbVersion();
+
prepareSchema(container, connectionUrl, dbUsername, dbPassword, dbSchema, dbName);
}
@@ -369,8 +372,6 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
@Override
public void start(Container container) throws Exception {
- verifyTimescaleDbVersion();
-
// Register standard entity classes and also any Entity ClassProviders
List<String> entityClasses = new ArrayList<>(50);
entityClasses.add(Asset.class.getName());
@@ -509,6 +510,7 @@ public class PersistenceService implements ContainerService, Consumer<Persistenc
switch (cause) {
case CREATE -> publishPersistenceEvent(cause, currentEntity, null, null, null);
case DELETE -> publishPersistenceEvent(cause, previousEntity, null, null, null);
+ case DELETE_FINISHED -> publishPersistenceEvent(cause, currentEntity, null, null, null);
case UPDATE -> {
List<String> propertyNames = new ArrayList<>(propertyFields.length);
List<Object> currentState = new ArrayList<>(propertyFields.length);
diff --git a/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java b/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java
index 3329378a8..6cb90b129 100644
--- a/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java
+++ b/manager/src/main/java/org/openremote/manager/alarm/AlarmService.java
@@ -444,8 +444,9 @@ public class AlarmService extends RouteBuilder implements ContainerService {
entityManager
.createQuery(
"""
- select aal from AlarmAssetLink aal
- where aal.id.realm = :realm and aal.id.sentalarmId = :alarmId
+ select aal from AlarmAssetLink aal, Asset a
+ where a.id = aal.id.assetId and a.deletePending is false
+ and aal.id.realm = :realm and aal.id.sentalarmId = :alarmId
order by aal.createdOn desc
""",
AlarmAssetLink.class)
diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java
index ca315d514..316f9baf1 100644
--- a/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/asset/AssetResourceImpl.java
@@ -619,6 +619,9 @@ public class AssetResourceImpl extends ManagerWebResource implements AssetResour
public Asset<?>[] queryAssets(RequestParams requestParams, AssetQuery query) {
if (query == null) {
query = new AssetQuery();
+ } else {
+ // Don't allow deleted assets in query
+ query.includeDeletePending(false);
}
if (!assetStorageService.authorizeAssetQuery(query, getAuthContext(), getRequestRealmName())) {
@@ -637,6 +640,9 @@ public class AssetResourceImpl extends ManagerWebResource implements AssetResour
public AssetTree queryAssetTree(RequestParams requestParams, AssetQuery query) {
if (query == null) {
query = new AssetQuery();
+ } else {
+ // Don't allow deleted assets in query
+ query.includeDeletePending(false);
}
if (!assetStorageService.authorizeAssetQuery(query, getAuthContext(), getRequestRealmName())) {
diff --git a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
index 00f01dc23..f21a662a7 100644
--- a/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
+++ b/manager/src/main/java/org/openremote/manager/asset/AssetStorageService.java
@@ -26,6 +26,7 @@ import static org.openremote.model.attribute.Attribute.getAddedOrModifiedAttribu
import static org.openremote.model.query.AssetQuery.*;
import static org.openremote.model.query.AssetQuery.Access.*;
import static org.openremote.model.query.filter.ValuePredicate.asPredicateOrTrue;
+import static org.openremote.model.util.MapAccess.getInteger;
import static org.openremote.model.util.TextUtil.isNullOrEmpty;
import static org.openremote.model.value.MetaItemType.ACCESS_PUBLIC_READ;
import static org.openremote.model.value.MetaItemType.ACCESS_RESTRICTED_READ;
@@ -37,9 +38,17 @@ import jakarta.persistence.TypedQuery;
import jakarta.validation.ConstraintViolation;
import jakarta.validation.ConstraintViolationException;
import java.sql.*;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.temporal.ChronoUnit;
import java.util.*;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -75,6 +84,7 @@ import org.openremote.model.asset.impl.UnknownAsset;
import org.openremote.model.attribute.Attribute;
import org.openremote.model.attribute.AttributeEvent;
import org.openremote.model.attribute.AttributeMap;
+import org.openremote.model.datapoint.AssetDatapoint;
import org.openremote.model.event.Event;
import org.openremote.model.event.RespondableEvent;
import org.openremote.model.event.shared.EventSubscription;
@@ -109,6 +119,17 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
}
}
+ public record DeleteResult(boolean accepted, CompletableFuture<Boolean> completion) {
+
+ public static DeleteResult rejected() {
+ return new DeleteResult(false, CompletableFuture.completedFuture(false));
+ }
+
+ public static DeleteResult accepted(CompletableFuture<Boolean> completion) {
+ return new DeleteResult(true, completion);
+ }
+ }
+
public interface ParameterBinder
extends BiConsumer<EntityManager, org.hibernate.query.Query<Object>> {
@@ -127,6 +148,13 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
private static final Logger LOG = Logger.getLogger(AssetStorageService.class.getName());
public static final int PRIORITY = MED_PRIORITY;
+ public static final String OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD =
+ "OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD";
+ public static final int OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD_DEFAULT = 100000;
+ public static final String OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS =
+ "OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS";
+ public static final int OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS_DEFAULT = 4;
+ protected static final long FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS = 1L;
// protected static final Field assetParentNameField;
// protected static final Field assetParentTypeField;
@@ -237,7 +265,18 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
protected ClientEventService clientEventService;
protected GatewayService gatewayService;
protected ExecutorService executorService;
+ protected ScheduledExecutorService scheduledExecutorService;
protected final LockByKey assetLocks = new LockByKey();
+ protected final AtomicReference<CompletableFuture<Void>> pendingAssetDeleteFuture =
+ new AtomicReference<>(CompletableFuture.completedFuture(null));
+ protected final AtomicReference<LocalDateTime> oldestAssetDatapointChunkStart =
+ new AtomicReference<>();
+ protected final Set<String> failedAssetDeleteIds =
+ Collections.synchronizedSet(new LinkedHashSet<>());
+ protected ScheduledFuture<?> failedAssetDeleteRetryFuture;
+ protected int assetDeleteDatapointBatchThreshold;
+ protected int assetDeleteDatapointBatchWeeks;
+ protected volatile boolean pendingAssetDeleteStopping;
/**
* Will evaluate each {@link CalendarEventPredicate} and apply it depending on the {@link
@@ -331,6 +370,19 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
clientEventService = container.getService(ClientEventService.class);
gatewayService = container.getService(GatewayService.class);
executorService = container.getExecutor();
+ scheduledExecutorService = container.getScheduledExecutor();
+ assetDeleteDatapointBatchThreshold =
+ getInteger(
+ container.getConfig(),
+ OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD,
+ OR_ASSET_DELETE_DATAPOINT_BATCH_THRESHOLD_DEFAULT);
+ assetDeleteDatapointBatchWeeks =
+ Math.max(
+ 1,
+ getInteger(
+ container.getConfig(),
+ OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS,
+ OR_ASSET_DELETE_DATAPOINT_BATCH_WEEKS_DEFAULT));
EventSubscriptionAuthorizer assetEventAuthorizer =
AssetStorageService.assetInfoAuthorizer(identityService, this);
@@ -392,10 +444,26 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
}
@Override
- public void start(Container container) throws Exception {}
+ public void start(Container container) throws Exception {
+ pendingAssetDeleteStopping = false;
+ failedAssetDeleteRetryFuture =
+ scheduledExecutorService.scheduleAtFixedRate(
+ this::retryFailedAssetDeletes,
+ FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS,
+ FAILED_ASSET_DELETE_RETRY_PERIOD_HOURS,
+ TimeUnit.HOURS);
+ queueAssetsDeletion();
+ }
@Override
- public void stop(Container container) throws Exception {}
+ public void stop(Container container) throws Exception {
+ pendingAssetDeleteStopping = true;
+ pendingAssetDeleteFuture.get().cancel(false);
+ if (failedAssetDeleteRetryFuture != null) {
+ failedAssetDeleteRetryFuture.cancel(false);
+ failedAssetDeleteRetryFuture = null;
+ }
+ }
@SuppressWarnings("unchecked")
@Override
@@ -565,11 +633,12 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
public List<String> findNames(String... ids) {
if (ids == null || ids.length == 0) return new ArrayList<>();
- // TODO: Do this in a loop in reasonably sized batches
return persistenceService.doReturningTransaction(
em -> {
List<Object[]> result =
- em.createQuery("select a.id, a.name from Asset a where a.id in :ids", Object[].class)
+ em.createQuery(
+ "select a.id, a.name from Asset a where a.id in :ids and a.deletePending is false",
+ Object[].class)
.setParameter("ids", Arrays.asList(ids))
.getResultList();
List<String> names = new ArrayList<>();
@@ -639,12 +708,17 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
throws IllegalStateException, ConstraintViolationException {
if (LOG.isLoggable(Level.FINEST)) {
- LOG.finest("Merging asset: " + asset);
+ LOG.finest("Asset merge: " + asset.toStringAll());
+ } else {
+ LOG.fine("Asset merge: " + asset);
}
long startTime = System.currentTimeMillis();
String assetId = asset.getId();
+ // Ensure delete pending flag is not set for merges
+ asset.setDeletePending(false);
+
// We skip all standard checks as asset is coming from a gateway and would be validated from
// there
if (requestingGatewayAsset != null) {
@@ -703,6 +777,16 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
if (existingAsset != null) {
+ boolean overrideVersionTemp = overrideVersion;
+
+ if (existingAsset.isDeletePending()) {
+ LOG.info("Cancelling pending asset deletion: assetId=" + assetId);
+ existingAsset.setDeletePending(false);
+ failedAssetDeleteIds.remove(assetId);
+ em.flush();
+ overrideVersionTemp = true;
+ }
+
// Verify type has not been changed
if (!existingAsset.getType().equals(asset.getType())) {
String msg = "Asset type cannot be changed: asset=" + asset;
@@ -717,10 +801,11 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
}
// Update timestamp on modified attributes this allows fast equality checking
+ T finalExistingAsset = existingAsset;
asset.getAttributes().stream()
.forEach(
attr ->
- existingAsset
+ finalExistingAsset
.getAttribute(attr.getName())
.ifPresent(
existingAttr -> {
@@ -744,7 +829,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
// version, so the detached state always wins and this update will go through
// and ignore
// concurrent updates
- if (overrideVersion) {
+ if (overrideVersionTemp) {
asset.setVersion(existingAsset.getVersion());
}
}
@@ -770,13 +855,13 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
Asset<?> parent = find(em, asset.getParentId(), true);
// The parent must exist
- if (parent == null) {
+ if (parent == null || parent.isDeletePending()) {
String msg = "Asset parent not found: asset=" + asset;
LOG.warning(msg);
throw new IllegalStateException(msg);
}
- // The parent can not be a child of the asset
+ // The parent cannot be a child of the asset
if (parent.pathContains(asset.getId())) {
String msg =
"Asset parent cannot be a descendant of the asset: asset=" + asset;
@@ -906,11 +991,16 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
updatedAsset = em.merge(asset);
}
- if (LOG.isLoggable(FINE)) {
- LOG.fine(
- "Asset merge took: " + (System.currentTimeMillis() - startTime) + "ms");
+ String msgPrefix = existingAsset != null ? "Asset updated: " : "Asset created: ";
+ if (LOG.isLoggable(Level.FINEST)) {
+ LOG.finest(msgPrefix + asset.toStringAll());
+ } else {
+ LOG.info(msgPrefix + asset);
}
+ LOG.finest(
+ () -> "Asset merge took: " + (System.currentTimeMillis() - startTime) + "ms");
+
if (user != null) {
createUserAssetLinks(
em,
@@ -940,17 +1030,50 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
}
/**
- * @return <code>true</code> if the assets were deleted, false if any of the assets still have
- * children and can't be deleted.
+ * @return <code>true</code> if all assets were accepted for deletion and marked as delete
+ * pending, <code>false</code> if any asset still has children, doesn't exist, or can't be
+ * marked delete pending.
*/
public boolean delete(List<String> assetIds) {
return delete(assetIds, false);
}
+ public boolean isDeletePending(String assetId) {
+ if (isNullOrEmpty(assetId)) {
+ throw new IllegalArgumentException("Can't query null or empty asset identifier");
+ }
+
+ Asset<?> asset =
+ find(
+ new AssetQuery()
+ .ids(assetId)
+ .includeDeletePending(true)
+ .select(new Select().excludeAttributes()));
+
+ return asset != null && asset.isDeletePending();
+ }
+
public boolean delete(List<String> assetIds, boolean skipGatewayCheck) {
+ return deleteWithResult(assetIds, skipGatewayCheck).accepted();
+ }
+
+ public CompletableFuture<Boolean> deleteUntilFinished(List<String> assetIds) {
+ return deleteUntilFinished(assetIds, false);
+ }
+
+ public CompletableFuture<Boolean> deleteUntilFinished(
+ List<String> assetIds, boolean skipGatewayCheck) {
+ DeleteResult result = deleteWithResult(assetIds, skipGatewayCheck);
+ return result.accepted() ? result.completion() : CompletableFuture.completedFuture(false);
+ }
+
+ public DeleteResult deleteWithResult(List<String> assetIds, boolean skipGatewayCheck) {
List<String> ids = new ArrayList<>(assetIds);
+ LOG.info(
+ () -> "Asset deletion: count=" + assetIds.size() + ", IDs=" + String.join(", ", assetIds));
+
if (!skipGatewayCheck) {
// Don't allow deletion of gateway descendant assets (they must be deleted on the gateway
@@ -970,68 +1093,469 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
if (!gatewayIds.isEmpty()) {
// Handle gateway asset deletion in a special way
+ LOG.fine("Deleting gateway assets: IDs=" + String.join(",", gatewayIds));
ids.removeAll(gatewayIds);
+ CompletableFuture<Boolean> gatewayDeleteFuture = CompletableFuture.completedFuture(true);
for (String gatewayId : gatewayIds) {
+ DeleteResult gatewayDeleteResult;
try {
- boolean deleted = gatewayService.deleteGateway(gatewayId);
- if (!deleted) {
- return false;
- }
+ gatewayDeleteResult = gatewayService.deleteGatewayUntilFinished(gatewayId);
} catch (Exception e) {
LOG.log(WARNING, "Failed to delete gateway asset: " + gatewayId, e);
- return false;
+ return DeleteResult.rejected();
}
+ if (!gatewayDeleteResult.accepted()) {
+ return DeleteResult.rejected();
+ }
+ gatewayDeleteFuture =
+ gatewayDeleteFuture.thenCombine(
+ gatewayDeleteResult
+ .completion()
+ .exceptionally(
+ ex -> {
+ LOG.log(WARNING, "Failed to delete gateway asset: " + gatewayId, ex);
+ return false;
+ }),
+ (deleted, gatewayDeleted) -> deleted && gatewayDeleted);
+ }
+ DeleteResult assetDeleteResult = markAssetsForDeletionAndQueue(ids, assetIds);
+ if (!assetDeleteResult.accepted()) {
+ return DeleteResult.rejected();
}
+ return DeleteResult.accepted(
+ gatewayDeleteFuture.thenCombine(
+ assetDeleteResult.completion(),
+ (gatewaysDeleted, assetsDeleted) -> gatewaysDeleted && assetsDeleted));
}
}
- if (!ids.isEmpty()) {
- try {
- // Get locks for each asset ID
- ids.forEach(assetLocks::lock);
+ return markAssetsForDeletionAndQueue(ids, assetIds);
+ }
- persistenceService.doTransaction(
- em -> {
- List<Asset<?>> assets =
- em
- .createQuery(
- "select a from Asset a where not exists(select child.id from Asset child where child.parentId = a.id and not child.id in :ids) and a.id in :ids",
- Asset.class)
- .setParameter("ids", ids)
- .getResultList()
- .stream()
- .map(asset -> (Asset<?>) asset)
- .collect(Collectors.toList());
-
- if (ids.size() != assets.size()) {
- throw new IllegalArgumentException(
- "Cannot delete one or more requested assets as they either have children or don't exist");
- }
+ /**
+ * Marks assets as pending deletion and queues them for actual deletion.
+ *
+ * @return DeleteResult indicating whether the deletion was accepted or rejected
+ */
+ protected DeleteResult markAssetsForDeletionAndQueue(List<String> ids, List<String> assetIds) {
+ if (ids.isEmpty()) {
+ return DeleteResult.accepted(CompletableFuture.completedFuture(true));
+ }
+
+ List<String> orderedAssetIdsToQueue;
+ try {
+ ids.forEach(assetLocks::lock);
- assets.sort(
- Comparator.comparingInt(
- (Asset<?> asset) -> asset.getPath() == null ? 0 : asset.getPath().length)
- .reversed());
+ List<Asset<?>> assets = new ArrayList<>();
+ persistenceService.doTransaction(
+ em -> {
+ assets.addAll(
+ em
+ .createQuery(
+ "select a from Asset a where not exists(select child.id from Asset child where child.parentId = a.id and child.deletePending is false and not child.id in :ids) and a.id in :ids",
+ Asset.class)
+ .setParameter("ids", ids)
+ .getResultList()
+ .stream()
+ .map(asset -> (Asset<?>) asset)
+ .sorted(
+ Comparator.comparingInt(
+ (Asset<?> asset) ->
+ asset.getPath() != null ? asset.getPath().length : 0)
+ .reversed()
+ .thenComparing(Asset::getCreatedOn))
+ .toList());
+
+ if (ids.size() != assets.size()) {
+ throw new IllegalArgumentException(
+ "Cannot delete one or more requested assets as they either have children or don't exist");
+ }
- // TODO: Remove when https://github.com/timescale/timescaledb/issues/9916 is fixed
- // and the minimum supported TimescaleDB version includes that fix.
- em.createNativeQuery("SET LOCAL plan_cache_mode = force_custom_plan").executeUpdate();
+ assets.forEach(asset -> asset.setDeletePending(true));
+ });
+
+ orderedAssetIdsToQueue = assets.stream().map(Asset::getId).toList();
+ } catch (Exception e) {
+ LOG.log(
+ SEVERE,
+ "Failed to delete one or more requested assets: " + Arrays.toString(assetIds.toArray()),
+ e);
+ return DeleteResult.rejected();
+ } finally {
+ // Release batch locks BEFORE queueing background worker tasks
+ ids.forEach(assetLocks::unlock);
+ }
+
+ return DeleteResult.accepted(queueAssetsDeletion(orderedAssetIdsToQueue));
+ }
+
+ /** Queue deletion of all assets marked as pending deletion */
+ protected void queueAssetsDeletion() {
+ queueAssetsDeletion(findPendingDeleteAssetIds());
+ }
+
+ /**
+ * Queue deletion of the specified ordered asset IDs
+ *
+ * @return CompletableFuture indicating whether the etire deletion batch succeeded
+ */
+ protected CompletableFuture<Boolean> queueAssetsDeletion(List<String> orderedAssetIds) {
+ if (pendingAssetDeleteStopping) {
+ return CompletableFuture.completedFuture(false);
+ }
+
+ if (orderedAssetIds.isEmpty()) {
+ return CompletableFuture.completedFuture(true);
+ }
+
+ CompletableFuture<Boolean> deleteResult = new CompletableFuture<>();
+ Map<String, Long> datapointCounts = findAssetDatapointCounts(orderedAssetIds);
+ LOG.fine(
+ "Scheduling asset deletion: count="
+ + orderedAssetIds.size()
+ + ", IDs="
+ + String.join(", ", orderedAssetIds));
+
+ pendingAssetDeleteFuture.updateAndGet(
+ existingFuture ->
+ existingFuture
+ .exceptionally(ex -> null)
+ .thenRunAsync(
+ () -> {
+ boolean allDeleted = true;
+ for (String assetId : orderedAssetIds) {
+ if (pendingAssetDeleteStopping) {
+ deleteResult.complete(false);
+ return;
+ }
+ allDeleted =
+ doAssetDeletion(assetId, datapointCounts.getOrDefault(assetId, 0L))
+ && allDeleted;
+ }
+ deleteResult.complete(allDeleted);
+ },
+ executorService)
+ .exceptionally(
+ ex -> {
+ LOG.log(SEVERE, "Failed to process queued asset deletions", ex);
+ deleteResult.complete(false);
+ return null;
+ }));
+
+ return deleteResult;
+ }
+
+ /** Re-attempts batch deletion of all assets that previously failed deletion. */
+ protected void retryFailedAssetDeletes() {
+ if (pendingAssetDeleteStopping) {
+ return;
+ }
+
+ List<String> failedAssetIds;
+ synchronized (failedAssetDeleteIds) {
+ if (failedAssetDeleteIds.isEmpty()) {
+ return;
+ }
+ failedAssetIds = new ArrayList<>(failedAssetDeleteIds);
+ failedAssetDeleteIds.clear();
+ }
+
+ LOG.fine("Retrying failed pending asset deletes: count=" + failedAssetIds.size());
+ requestPendingAssetDeletionRetry(failedAssetIds);
+ }
+
+ /** Re-attempts batch deletion of the specified assets. */
+ protected void requestPendingAssetDeletionRetry(List<String> assetIds) {
+ if (pendingAssetDeleteStopping || assetIds.isEmpty()) {
+ return;
+ }
+
+ pendingAssetDeleteFuture.updateAndGet(
+ existingFuture ->
+ existingFuture
+ .exceptionally(ex -> null)
+ .thenRunAsync(
+ () -> {
+ for (String assetId : assetIds) {
+ if (pendingAssetDeleteStopping) {
+ return;
+ }
+ deletePendingAsset(assetId);
+ }
+ },
+ executorService));
+ }
+
+ /**
+ * Get all assets from the database that are marked as pending deletion
+ *
+ * @return List of asset IDs that are pending deletion
+ */
+ @SuppressWarnings("unchecked")
+ protected List<String> findPendingDeleteAssetIds() {
+ return persistenceService.doReturningTransaction(
+ em ->
+ (List<String>)
+ em.createNativeQuery(
+ "select id from asset where delete_pending is true order by nlevel(path) desc, created_on asc")
+ .getResultList());
+ }
+
+ protected boolean deleteAssetDatapointsBeforeAsset(long datapointCount) {
+ return datapointCount > assetDeleteDatapointBatchThreshold;
+ }
+
+ /**
+ * Get the datapoint counts for all the provided asset IDs
+ *
+ * @return map of asset ID and datapoint count
+ */
+ protected Map<String, Long> findAssetDatapointCounts(Collection<String> assetIds) {
+ if (assetIds.isEmpty()) {
+ return Collections.emptyMap();
+ }
- assets.forEach(em::remove);
- em.flush();
+ long start = System.currentTimeMillis();
+ Map<String, Long> datapointCounts =
+ persistenceService.doReturningTransaction(
+ em -> {
+ @SuppressWarnings("unchecked")
+ List<Object[]> result =
+ em.createNativeQuery(
+ "select entity_id, count(*) from "
+ + AssetDatapoint.TABLE_NAME
+ + " where entity_id in (:assetIds) group by entity_id")
+ .setParameter("assetIds", assetIds)
+ .getResultList();
+
+ return result.stream()
+ .collect(
+ Collectors.toMap(
+ row -> (String) row[0], row -> ((Number) row[1]).longValue()));
});
+
+ LOG.finer(
+ "Counted asset datapoints for pending asset delete: assetCount="
+ + assetIds.size()
+ + ", durationMillis="
+ + (System.currentTimeMillis() - start));
+ return datapointCounts;
+ }
+
+ /**
+ * Handles the actual deletion of an asset by batching the datapoint deletion if count is above
+ * threshold; otherwise the asset is just deleted and FK cascading ensures datapoints are
+ * correctly deleted.
+ *
+ * @return True if the asset was successfully deleted
+ */
+ protected boolean doAssetDeletion(String assetId, long datapointCount) {
+ long start = System.currentTimeMillis();
+
+ if (deleteAssetDatapointsBeforeAsset(datapointCount)) {
+ LOG.fine(
+ "Purging asset datapoints: assetId="
+ + assetId
+ + ", datapoints="
+ + datapointCount
+ + ", threshold="
+ + assetDeleteDatapointBatchThreshold);
+ try {
+ if (deleteAssetDatapoints(assetId)) {
+ failedAssetDeleteIds.remove(assetId);
+ return true;
+ }
+ LOG.fine(
+ "Purged asset datapoints: assetId="
+ + assetId
+ + ", durationMillis="
+ + (System.currentTimeMillis() - start));
} catch (Exception e) {
- LOG.log(
- SEVERE,
- "Failed to delete one or more requested assets: " + Arrays.toString(assetIds.toArray()),
- e);
- return false;
- } finally {
- // Release all of the locks
- ids.forEach(assetLocks::unlock);
+ LOG.log(WARNING, "Purge asset datapoints failed: assetId=" + assetId, e);
}
}
- return true;
+
+ if (pendingAssetDeleteStopping) {
+ return !isDeletePending(assetId);
+ }
+
+ if (!isDeletePending(assetId)) {
+ failedAssetDeleteIds.remove(assetId);
+ return true;
+ }
+
+ return deletePendingAsset(assetId);
+ }
+
+ /**
+ * Purges the Asset entity from the database only if no ancestor assets have failed deletion.
+ *
+ * @return True if the asset was successfully deleted
+ */
+ protected boolean deletePendingAsset(String assetId) {
+ if (pendingAssetDeleteStopping) {
+ return false;
+ }
+
+ return withAssetLock(
+ assetId,
+ () -> {
+ if (pendingAssetDeleteStopping) {
+ return false;
+ }
+
+ LOG.fine("Deleting asset: assetId=" + assetId);
+ AtomicReference<Boolean> blockedByFailedPath = new AtomicReference<>(false);
+
+ try {
+ persistenceService.doTransaction(
+ em -> {
+ useCustomQueryPlansForTimescaleDelete(em);
+ Asset<?> asset = em.find(Asset.class, assetId);
+
+ if (asset == null || !asset.isDeletePending()) {
+ return;
+ }
+
+ synchronized (failedAssetDeleteIds) {
+ if (blockedByFailedPath.updateAndGet(
+ blocked ->
+ failedAssetDeleteIds.stream()
+ .anyMatch(
+ failedAssetId ->
+ !failedAssetId.equals(assetId)
+ && asset.pathContains(failedAssetId)))) {
+ return;
+ }
+ }
+
+ em.remove(asset);
+ em.flush();
+ });
+
+ if (blockedByFailedPath.get()) {
+ failedAssetDeleteIds.add(assetId);
+ LOG.fine(
+ "Asset delete skipped because a failed pending delete asset is in its path: assetId="
+ + assetId);
+ return false;
+ }
+
+ failedAssetDeleteIds.remove(assetId);
+ return true;
+ } catch (Exception e) {
+ failedAssetDeleteIds.add(assetId);
+ // TODO: Raise an alarm for asset deletion failure.
+ LOG.log(
+ SEVERE, "Failed to delete pending asset, queued for retry: assetId=" + assetId, e);
+ return false;
+ }
+ });
+ }
+
+ // TODO: Remove when https://github.com/timescale/timescaledb/issues/9916 is fixed
+ // and the minimum supported TimescaleDB version includes that fix.
+ protected void useCustomQueryPlansForTimescaleDelete(EntityManager em) {
+ em.createNativeQuery("SET LOCAL plan_cache_mode = force_custom_plan").executeUpdate();
+ }
+
+ /**
+ * Purges the datapoints for the asset in batches; checks the asset pending deletion flag between
+ * executions.
+ *
+ * @return True if the asset is no longer pending deletion
+ */
+ protected boolean deleteAssetDatapoints(String assetId) {
+ LocalDateTime oldestChunkStart = findOldestAssetDatapointChunkStart();
+
+ if (oldestChunkStart == null) {
+ LOG.fine("No asset datapoint chunks found for pending asset delete: assetId=" + assetId);
+ return !isDeletePending(assetId);
+ }
+
+ if (pendingAssetDeleteStopping) {
+ return !isDeletePending(assetId);
+ }
+
+ LocalDateTime rangeStart = oldestChunkStart;
+ LocalDateTime finalEnd = LocalDateTime.ofInstant(timerService.getNow(), ZoneId.systemDefault());
+
+ while (rangeStart.isBefore(finalEnd) && !pendingAssetDeleteStopping) {
+ LocalDateTime nextEnd =
+ rangeStart.plus((long) assetDeleteDatapointBatchWeeks * 7L, ChronoUnit.DAYS);
+ LocalDateTime rangeEnd = nextEnd.isBefore(finalEnd) ? nextEnd : finalEnd;
+ LocalDateTime batchStart = rangeStart;
+ LocalDateTime batchEnd = rangeEnd;
+ long start = System.currentTimeMillis();
+
+ int deleted =
+ persistenceService.doReturningTransaction(
+ em -> {
+ useCustomQueryPlansForTimescaleDelete(em);
+ return em.createNativeQuery(
+ "delete from "
+ + AssetDatapoint.TABLE_NAME
+ + " where entity_id = :assetId and timestamp >= :rangeStart and timestamp < :rangeEnd")
+ .setParameter("assetId", assetId)
+ .setParameter("rangeStart", batchStart)
+ .setParameter("rangeEnd", batchEnd)
+ .executeUpdate();
+ });
+
+ if (LOG.isLoggable(FINE)) {
+ LOG.fine(
+ "Deleted asset datapoints for pending asset: assetId="
+ + assetId
+ + ", chunkRangeStart="
+ + batchStart
+ + ", chunkRangeEnd="
+ + batchEnd
+ + ", rows="
+ + deleted
+ + ", durationMillis="
+ + (System.currentTimeMillis() - start));
+ }
+
+ if (!isDeletePending(assetId)) {
+ failedAssetDeleteIds.remove(assetId);
+ return true;
+ }
+
+ rangeStart = rangeEnd;
+ }
+
+ return !isDeletePending(assetId);
+ }
+
+ protected LocalDateTime findOldestAssetDatapointChunkStart() {
+ LocalDateTime cachedStart = oldestAssetDatapointChunkStart.get();
+ if (cachedStart != null) {
+ return cachedStart;
+ }
+
+ LocalDateTime start =
+ persistenceService.doReturningTransaction(
+ em ->
+ em.unwrap(Session.class)
+ .createNativeQuery(
+ """
+ select min(range_start::timestamp)
+ from timescaledb_information.chunks
+ where hypertable_schema = current_schema()
+ and hypertable_name = :hypertableName
+ and range_start is not null
+ """,
+ LocalDateTime.class)
+ .setParameter("hypertableName", AssetDatapoint.TABLE_NAME)
+ .getSingleResult());
+
+ if (start == null) {
+ return null;
+ }
+
+ return oldestAssetDatapointChunkStart.compareAndSet(null, start)
+ ? start
+ : oldestAssetDatapointChunkStart.get();
}
public boolean isUserAsset(String assetId) {
@@ -1121,7 +1645,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
try {
return entityManager
.createQuery(
- "select count(a) from Asset a where a.realm = :realm and a.id in :assetIds",
+ "select count(a) from Asset a where a.realm = :realm and a.id in :assetIds and a.deletePending is false",
Long.class)
.setParameter("realm", realm)
.setParameter("assetIds", assetIds)
@@ -1133,7 +1657,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
});
}
- /** Returns the realm for each asset ID that exists. */
+ /** Returns the realm for each asset ID that exists and not pending deletion. */
public Map<String, String> getAssetRealms(Collection<String> assetIds) {
if (assetIds == null || assetIds.isEmpty()) {
return Map.of();
@@ -1145,7 +1669,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
.createQuery(
"""
select a.id, a.realm from Asset a
- where a.id in :assetIds
+ where a.id in :assetIds and a.deletePending is false
""",
Object[].class)
.setParameter("assetIds", assetIds)
@@ -1171,7 +1695,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
public Boolean execute(Connection connection) throws SQLException {
try (PreparedStatement st =
connection.prepareStatement(
- "select count(*) from Asset a where a.path ~ lquery(?) AND a.id = ANY(?)")) {
+ "select count(*) from Asset a where a.path ~ lquery(?) AND a.id = ANY(?) AND a.delete_pending is false")) {
st.setString(1, "*." + parentAssetId + ".*");
st.setArray(
2, st.getConnection().createArrayOf("text", assetIds.toArray()));
@@ -1188,7 +1712,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
/**
* Returns a map of asset IDs with the respective hasChildren flag.
*
- * <p>It check whether any of the assetIds is set as a parentId in the Asset table.
+ * <p>It checks whether any of the assetIds is set as a parentId in the Asset table.
*
* @param assetIds The list of asset IDs to check.
* @return A map of asset IDs with the respective hasChildren flag.
@@ -1200,7 +1724,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
List<String> parentsWithChildren =
entityManager
.createQuery(
- "select distinct a.parentId from Asset a where a.parentId in :assetIds",
+ "select distinct a.parentId from Asset a where a.parentId in :assetIds and a.deletePending is false",
String.class)
.setParameter("assetIds", assetIds)
.getResultList();
@@ -1269,7 +1793,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
EntityManager em, String realm, Collection<String> userIds, Collection<String> assetIds) {
StringBuilder sb = new StringBuilder();
Map<String, Object> parameters = new HashMap<>(3);
- sb.append("select ua from UserAssetLink ua where 1=1");
+ sb.append("select ua from UserAssetLink ua, Asset a where a.id = ua.id.assetId");
+ sb.append(" and a.deletePending is false");
if (!isNullOrEmpty(realm)) {
sb.append(" and ua.id.realm in :realm");
@@ -1646,7 +2171,7 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
// https://stackoverflow.com/questions/53847917/postgresql-throws-column-is-of-type-jsonb-but-expression-is-of-type-bytea-with)
Query query =
em.createNativeQuery(
- "UPDATE asset SET attributes[?] = attributes[?] || ?\\:\\:jsonb where id = ?")
+ "UPDATE asset SET attributes[?] = attributes[?] || ?\\:\\:jsonb where id = ? and delete_pending is false")
.setParameter(1, event.getName())
.setParameter(2, event.getName())
.setParameter(
@@ -1701,11 +2226,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
if (loadedAsset == null) {
return;
}
- if (LOG.isLoggable(Level.FINEST)) {
- LOG.finest("Asset created: " + loadedAsset.toStringAll());
- } else {
- LOG.fine("Asset created: " + loadedAsset);
- }
clientEventService.publishEvent(new AssetEvent(AssetEvent.Cause.CREATE, loadedAsset, null));
// Raise attribute event for each created attribute
@@ -1729,7 +2249,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
persistenceEvent.getPropertyNames().size() > 1
|| !persistenceEvent.hasPropertyChanged("attributes");
boolean attributesChanged = persistenceEvent.hasPropertyChanged("attributes");
- LOG.finest(() -> "Asset updated: " + persistenceEvent);
clientEventService.publishEvent(
new AssetEvent(
@@ -1801,11 +2320,6 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
});
}
case DELETE -> {
- if (LOG.isLoggable(Level.FINEST)) {
- LOG.finest("Asset deleted: " + asset.toStringAll());
- } else {
- LOG.fine("Asset deleted: " + asset);
- }
clientEventService.publishEvent(new AssetEvent(AssetEvent.Cause.DELETE, asset, null));
// Raise attribute event with deleted flag for each attribute
@@ -1882,7 +2396,8 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
StringBuilder sb = new StringBuilder();
AssetQuery.Select select = query.select;
- sb.append("select A.ID as ID, A.NAME as NAME, A.ACCESS_PUBLIC_READ as ACCESS_PUBLIC_READ");
+ sb.append(
+ "select A.ID as ID, A.NAME as NAME, A.ACCESS_PUBLIC_READ as ACCESS_PUBLIC_READ, A.DELETE_PENDING as DELETE_PENDING");
sb.append(", A.CREATED_ON AS CREATED_ON, A.TYPE AS TYPE, A.PARENT_ID AS PARENT_ID");
sb.append(", A.REALM AS REALM, A.VERSION as VERSION");
@@ -2106,6 +2621,10 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
sb.append(" and A.ACCESS_PUBLIC_READ is true");
}
+ if (!query.isIncludeDeletePending()) {
+ sb.append(" and A.DELETE_PENDING is false");
+ }
+
if (query.types != null) {
String[] resolvedTypes = getResolvedAssetTypes(query.types);
final int pos = binders.size() + 1;
@@ -2585,6 +3104,9 @@ public class AssetStorageService extends RouteBuilder implements ContainerServic
protected <T extends HasAssetQuery & RespondableEvent> void onReadRequest(T event) {
AssetQuery assetQuery = event.getAssetQuery();
+ // Ensure that deleted assets are not included in the result
+ assetQuery.includeDeletePending(false);
+
Event response = null;
if (event.getResponseConsumer() == null) {
diff --git a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
index 47182ba41..da1650bb7 100644
--- a/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
+++ b/manager/src/main/java/org/openremote/manager/datapoint/AssetDatapointService.java
@@ -118,7 +118,7 @@ public class AssetDatapointService extends AbstractDatapointService<AssetDatapoi
}
if (maxDatapointAgeWeeks <= 0) {
- LOG.warning("Data point purge disabled");
+ LOG.info("Data point purge disabled");
} else {
LOG.log(Level.INFO, "Data point purge retention = " + maxDatapointAgeWeeks + " weeks");
}
diff --git a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java
index 29cf729f4..81ae740cc 100644
--- a/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java
+++ b/manager/src/main/java/org/openremote/manager/gateway/GatewayService.java
@@ -544,7 +544,7 @@ public class GatewayService extends RouteBuilder implements ContainerService {
return false;
}
- public boolean deleteGateway(String gatewayId) {
+ public AssetStorageService.DeleteResult deleteGatewayUntilFinished(String gatewayId) {
GatewayConnector connector = gatewayConnectorMap.get(gatewayId.toLowerCase(Locale.ROOT));
if (connector == null) {
@@ -565,8 +565,8 @@ public class GatewayService extends RouteBuilder implements ContainerService {
.collect(Collectors.toList());
gatewayAssetIds.add(gatewayId);
-
- return assetStorageService.delete(gatewayAssetIds, true);
+ LOG.info("Deleting gateway asset and descendants: IDs=" + String.join(",", gatewayAssetIds));
+ return assetStorageService.deleteWithResult(gatewayAssetIds, true);
}
public Collection<GatewayTunnelInfo> getTunnelInfos() {
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesService.java b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
index 0089dfe5d..9dae772e6 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesService.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesService.java
@@ -608,6 +608,10 @@ public class RulesService extends RouteBuilder implements ContainerService {
// Must reload from the database, the ruleset might not be completely hydrated on CREATE or
// UPDATE
AssetRuleset assetRuleset = rulesetStorageService.find(AssetRuleset.class, ruleset.getId());
+ if (assetRuleset == null) {
+ undeployAssetRuleset((AssetRuleset) ruleset);
+ return;
+ }
RulesEngine<AssetRuleset> engine = deployAssetRuleset(assetRuleset);
engine.start();
}
diff --git a/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java b/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java
index a8e009244..46544e478 100644
--- a/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java
+++ b/manager/src/main/java/org/openremote/manager/rules/RulesetStorageService.java
@@ -191,6 +191,10 @@ public class RulesetStorageService implements ContainerService {
StringBuilder sb, Class<T> rulesetType, RulesetQuery query, List<ParameterBinder> binders) {
sb.append(" WHERE 1=1");
+ if (rulesetType == AssetRuleset.class) {
+ sb.append(" AND A.DELETE_PENDING IS FALSE");
+ }
+
if (query.publicOnly) {
sb.append(" AND R.ACCESS_PUBLIC_READ");
}
diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java
index fd02b007a..6cfa228fd 100644
--- a/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java
+++ b/manager/src/main/java/org/openremote/manager/security/ManagerIdentityProvider.java
@@ -136,7 +136,8 @@ public interface ManagerIdentityProvider extends IdentityProvider {
// BUILD FROM
sb.append(" FROM public.user_entity u join PUBLIC.REALM r on r.ID = u.REALM_ID");
if (userQuery.assets != null || userQuery.pathPredicate != null) {
- sb.append(" join user_asset_link ua on ua.user_id = u.id");
+ sb.append(
+ " join user_asset_link ua on ua.user_id = u.id join asset a on a.id = ua.asset_id and a.delete_pending is false");
}
// BUILD WHERE
diff --git a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
index aab94d610..8308777fe 100644
--- a/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
+++ b/manager/src/main/java/org/openremote/manager/security/ManagerKeycloakIdentityProvider.java
@@ -66,7 +66,6 @@ import org.openremote.manager.event.ClientEventService;
import org.openremote.model.Constants;
import org.openremote.model.Container;
import org.openremote.model.PersistenceEvent;
-import org.openremote.model.asset.Asset;
import org.openremote.model.auth.OAuthGrant;
import org.openremote.model.auth.OAuthPasswordGrant;
import org.openremote.model.gateway.GatewayConnection;
@@ -1193,7 +1192,17 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider
throw new NotFoundException("Realm does not exist: " + realmName);
}
+ int assetCount =
+ assetStorageService.count(
+ new AssetQuery().realm(new RealmPredicate(realmName)).includeDeletePending(true));
+
+ if (assetCount > 0) {
+ throw new IllegalStateException(
+ "Cannot delete realm '" + realmName + "' because it still contains assets");
+ }
+
realmCache.invalidate(realmName);
+
persistenceService.doTransaction(
entityManager -> {
@@ -1228,18 +1237,6 @@ public class ManagerKeycloakIdentityProvider extends KeycloakIdentityProvider
+ "where rs.realm = ?1");
query.setParameter(1, realmName);
query.executeUpdate();
-
- // Delete Assets
- List<String> assetIds =
- assetStorageService
- .findAll(
- new AssetQuery()
- .select(new AssetQuery.Select().excludeAttributes())
- .realm(new RealmPredicate(realmName)))
- .stream()
- .map(Asset::getId)
- .toList();
- assetStorageService.delete(assetIds);
});
LOG.fine("Deleting realm: " + realmName);
diff --git a/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java b/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java
index 73ba56283..5d023d3f4 100644
--- a/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java
+++ b/manager/src/main/java/org/openremote/manager/security/RealmResourceImpl.java
@@ -137,6 +137,8 @@ public class RealmResourceImpl extends ManagerWebResource implements RealmResour
identityService.getIdentityProvider().deleteRealm(realm);
} catch (ClientErrorException ex) {
throw new WebApplicationException(ex.getCause(), ex.getResponse().getStatus());
+ } catch (IllegalStateException ex) {
+ throw new WebApplicationException(ex, Response.Status.CONFLICT);
} catch (Exception ex) {
throw new WebApplicationException(ex);
}
diff --git a/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java b/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java
index ac0d8202a..a93660553 100644
--- a/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java
+++ b/manager/src/main/java/org/openremote/manager/setup/KeycloakCleanSetup.java
@@ -22,17 +22,24 @@ import static org.openremote.container.security.keycloak.KeycloakIdentityProvide
import static org.openremote.model.Constants.*;
import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
+import org.openremote.manager.asset.AssetStorageService;
import org.openremote.model.Container;
+import org.openremote.model.asset.Asset;
+import org.openremote.model.query.AssetQuery;
import org.openremote.model.query.UserQuery;
import org.openremote.model.query.filter.RealmPredicate;
public class KeycloakCleanSetup extends AbstractKeycloakSetup {
private static final Logger LOG = Logger.getLogger(KeycloakCleanSetup.class.getName());
+ private final AssetStorageService assetStorageService;
public KeycloakCleanSetup(Container container) {
super(container);
+ this.assetStorageService = container.getService(AssetStorageService.class);
}
@Override
@@ -48,6 +55,7 @@ public class KeycloakCleanSetup extends AbstractKeycloakSetup {
.forEach(
realm -> {
if (!realm.getName().equals(MASTER_REALM)) {
+ deleteRealmAssets(realm.getName());
keycloakProvider.deleteRealm(realm.getName());
}
});
@@ -86,4 +94,37 @@ public class KeycloakCleanSetup extends AbstractKeycloakSetup {
keycloakProvider.deleteIdentityProvider(MASTER_REALM, idp.getAlias());
});
}
+
+ protected void deleteRealmAssets(String realm) {
+ List<String> assetIds =
+ assetStorageService
+ .findAll(
+ new AssetQuery()
+ .select(new AssetQuery.Select().excludeAttributes())
+ .realm(new RealmPredicate(realm))
+ .includeDeletePending(true))
+ .stream()
+ .map(Asset::getId)
+ .toList();
+
+ if (assetIds.isEmpty()) {
+ return;
+ }
+
+ LOG.info("Deleting all assets in realm '" + realm + "': count=" + assetIds.size());
+ boolean deleted;
+ try {
+ deleted = assetStorageService.deleteUntilFinished(assetIds, true).get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(
+ "Interrupted while deleting all assets in realm '" + realm + "'", e);
+ } catch (ExecutionException e) {
+ throw new IllegalStateException("Failed to delete all assets in realm '" + realm + "'", e);
+ }
+
+ if (!deleted) {
+ throw new IllegalStateException("Failed to delete all assets in realm '" + realm + "'");
+ }
+ }
}
diff --git a/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sql b/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sql
new file mode 100644
index 000000000..67872c055
--- /dev/null
+++ b/manager/src/main/resources/org/openremote/manager/setup/database/V20260923_1100__AssetDeletePending.sql
@@ -0,0 +1,2 @@
+ALTER TABLE ${schemaName}.asset
+ ADD COLUMN IF NOT EXISTS delete_pending boolean NOT NULL DEFAULT false;
diff --git a/model/src/main/java/org/openremote/model/PersistenceEvent.java b/model/src/main/java/org/openremote/model/PersistenceEvent.java
index 8588291cf..55f705da9 100644
--- a/model/src/main/java/org/openremote/model/PersistenceEvent.java
+++ b/model/src/main/java/org/openremote/model/PersistenceEvent.java
@@ -26,7 +26,8 @@ public class PersistenceEvent<T> {
public enum Cause {
CREATE,
UPDATE,
- DELETE
+ DELETE,
+ DELETE_FINISHED
}
protected final Cause cause;
@@ -109,7 +110,7 @@ public class PersistenceEvent<T> {
+ ", entity="
+ entity
+ ", propertyNames="
- + String.join(",", propertyNames)
+ + (propertyNames != null ? String.join(",", propertyNames) : "null")
+ ", currentState="
+ Arrays.toString(currentState)
+ ", previousState="
diff --git a/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java b/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java
index af173aa9c..9a2d60083 100644
--- a/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java
+++ b/model/src/main/java/org/openremote/model/alarm/AlarmAssetLink.java
@@ -104,7 +104,7 @@ public class AlarmAssetLink {
protected String assetName;
@Formula(
- "(select pa.NAME from ASSET a left outer join ASSET pa on a.PARENT_ID = pa.ID where a.ID = ASSET_ID)")
+ "(select pa.NAME from ASSET a left outer join ASSET pa on a.PARENT_ID = pa.ID and pa.DELETE_PENDING is false where a.ID = ASSET_ID)")
protected String parentAssetName;
protected AlarmAssetLink() {}
diff --git a/model/src/main/java/org/openremote/model/asset/Asset.java b/model/src/main/java/org/openremote/model/asset/Asset.java
index f9712517e..16eb98240 100644
--- a/model/src/main/java/org/openremote/model/asset/Asset.java
+++ b/model/src/main/java/org/openremote/model/asset/Asset.java
@@ -20,6 +20,8 @@ package org.openremote.model.asset;
import static jakarta.persistence.DiscriminatorType.STRING;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
@@ -262,6 +264,7 @@ import org.openremote.model.value.ValueType;
"Polymorphic OpenRemote asset. The type discriminator selects a registered asset subtype; attributes are keyed by attribute name.")
public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>, AssetInfo {
+ public static final String DELETE_PENDING_PROPERTY_NAME = "deletePending";
private static final java.util.regex.Pattern ASSET_ID_PATTERN =
java.util.regex.Pattern.compile(Constants.ASSET_ID_REGEXP);
@@ -364,6 +367,9 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>
example = "false")
protected boolean accessPublicRead;
+ @Column(name = "DELETE_PENDING", nullable = false)
+ protected boolean deletePending;
+
@Column(name = "PARENT_ID", length = 22, columnDefinition = "char(22)")
@Pattern(regexp = Constants.ASSET_ID_REGEXP, message = "{Asset.parentId.Pattern}") @Schema(
description = "Parent asset identifier. Omit for a root asset.",
@@ -460,6 +466,17 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>
return (T) this;
}
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public boolean isDeletePending() {
+ return deletePending;
+ }
+
+ @JsonIgnore
+ public T setDeletePending(boolean deletePending) {
+ this.deletePending = deletePending;
+ return (T) this;
+ }
+
public T setParent(Asset<?> parent) {
if (parent == null) {
parentId = null;
@@ -736,6 +753,7 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>
Asset<?> asset = (Asset<?>) o;
return version == asset.version
&& accessPublicRead == asset.accessPublicRead
+ && deletePending == asset.deletePending
&& Objects.equals(id, asset.id)
&& Objects.equals(createdOn, asset.createdOn)
&& Objects.equals(name, asset.name)
@@ -754,6 +772,7 @@ public abstract class Asset<T extends Asset<?>> implements IdentifiableEntity<T>
createdOn,
name,
accessPublicRead,
+ deletePending,
parentId,
realm,
type,
diff --git a/model/src/main/java/org/openremote/model/asset/AssetResource.java b/model/src/main/java/org/openremote/model/asset/AssetResource.java
index 0abeb7a0c..db616548b 100644
--- a/model/src/main/java/org/openremote/model/asset/AssetResource.java
+++ b/model/src/main/java/org/openremote/model/asset/AssetResource.java
@@ -661,7 +661,7 @@ public interface AssetResource {
@Produces(APPLICATION_JSON)
@Operation(
operationId = "queryCount",
- summary = "Count assets using a query",
+ summary = "Count assets using a query (can also be used to count assets pending deletion)",
description =
"Returns only the number of assets matching an AssetQuery after applying the caller's realm, public, and linked-asset access constraints.")
@OpenApiResponses.Ok
diff --git a/model/src/main/java/org/openremote/model/query/AssetQuery.java b/model/src/main/java/org/openremote/model/query/AssetQuery.java
index db2136b0d..e7047f467 100644
--- a/model/src/main/java/org/openremote/model/query/AssetQuery.java
+++ b/model/src/main/java/org/openremote/model/query/AssetQuery.java
@@ -209,6 +209,7 @@ public class AssetQuery implements Serializable {
public String[] types;
public LogicGroup<AttributePredicate> attributes;
+ public Boolean includeDeletePending;
// Ordering
public OrderBy orderBy;
@@ -425,6 +426,15 @@ public class AssetQuery implements Serializable {
return attributeValue(name, new NumberPredicate(d, operator));
}
+ public AssetQuery includeDeletePending(boolean includeDeletePending) {
+ this.includeDeletePending = includeDeletePending;
+ return this;
+ }
+
+ public boolean isIncludeDeletePending() {
+ return includeDeletePending != null && includeDeletePending;
+ }
+
public AssetQuery orderBy(OrderBy orderBy) {
this.orderBy = orderBy;
return this;
@@ -463,6 +473,8 @@ public class AssetQuery implements Serializable {
+ Arrays.toString(types)
+ ", attribute="
+ (attributes != null ? attributes.toString() : "null")
+ + ", includeDeletePending="
+ + includeDeletePending
+ ", orderBy="
+ orderBy
+ ", limit="
diff --git a/model/src/main/java/org/openremote/model/security/RealmResource.java b/model/src/main/java/org/openremote/model/security/RealmResource.java
index 187879c77..7084a9070 100644
--- a/model/src/main/java/org/openremote/model/security/RealmResource.java
+++ b/model/src/main/java/org/openremote/model/security/RealmResource.java
@@ -151,6 +151,9 @@ public interface RealmResource {
@OpenApiResponses.Authenticated
@OpenApiResponses.NotFound
@ApiResponse(responseCode = "405", description = "The master realm cannot be deleted")
+ @ApiResponse(
+ responseCode = "409",
+ description = "Realm cannot be deleted while it still contains assets")
void delete(
@BeanParam RequestParams requestParams,
@Parameter(description = REALM, example = EXAMPLE_REALM) @PathParam("name") String realm);
diff --git a/model/src/main/java/org/openremote/model/util/LockByKey.java b/model/src/main/java/org/openremote/model/util/LockByKey.java
index 07f70d3ad..a1c6b9fce 100644
--- a/model/src/main/java/org/openremote/model/util/LockByKey.java
+++ b/model/src/main/java/org/openremote/model/util/LockByKey.java
@@ -52,6 +52,7 @@ public class LockByKey {
}
public void lock(String key) {
+ LOG.log(System.Logger.Level.TRACE, () -> "Lock acquiring: key=" + key);
LockWrapper lockWrapper =
locks.compute(key, (k, v) -> v == null ? createLockWrapper() : v.addThreadInQueue());
try {
diff --git a/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy b/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy
index da09e262c..95e6f942c 100644
--- a/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy
+++ b/test/src/main/groovy/org/openremote/test/ContainerTrait.groovy
@@ -64,6 +64,7 @@ import org.openremote.model.util.MapAccess
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import spock.lang.Shared
+import spock.lang.Timeout
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicReference
@@ -86,6 +87,7 @@ trait ContainerTrait {
Logger LOG = LoggerFactory.getLogger(ContainerTrait.class)
+ @Timeout(120)
Container startContainer(Map<String, String> config, Iterable<ContainerService> services) {
// Reset and start clock in case any previous tests stopped/modified it (pseudo clock is static so shared between tests)
@@ -222,7 +224,7 @@ trait ContainerTrait {
if (!currentAssets.isEmpty()) {
LOG.info("Purging ${currentAssets.size()} asset(s)")
- assetStorageService.delete(currentAssets, true)
+ assetStorageService.deleteUntilFinished(currentAssets, true).get(60, TimeUnit.SECONDS)
}
// Wait for all assets to be unlinked from protocols
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy
index f27de9197..fc1956340 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetDatapointPurgeTest.groovy
@@ -18,15 +18,20 @@
*/
package org.openremote.test.assets
+import org.apache.camel.builder.RouteBuilder
import org.hibernate.cfg.AvailableSettings
+import org.openremote.container.message.MessageBrokerService
import org.openremote.container.persistence.PersistenceService
import org.openremote.manager.asset.AssetStorageService
import org.openremote.manager.datapoint.AssetDatapointService
import org.openremote.manager.setup.SetupService
+import org.openremote.model.PersistenceEvent
+import org.openremote.model.asset.Asset
import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
import org.openremote.model.attribute.AttributeRef
import org.openremote.model.datapoint.ValueDatapoint
+import org.openremote.model.query.AssetQuery
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.KeycloakTestSetup
import org.openremote.test.ManagerContainerTrait
@@ -35,8 +40,11 @@ import spock.util.concurrent.PollingConditions
import java.time.Instant
import java.time.temporal.ChronoUnit
+import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.TimeUnit
+import static org.openremote.container.persistence.PersistenceService.PERSISTENCE_TOPIC
+import static org.openremote.container.persistence.PersistenceService.isPersistenceEventForEntityType
import static org.openremote.manager.datapoint.AssetDatapointService.OR_DATA_POINTS_MAX_AGE_WEEKS
class AssetDatapointPurgeTest extends Specification implements ManagerContainerTrait {
@@ -51,10 +59,43 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerT
config.put(OR_DATA_POINTS_MAX_AGE_WEEKS, "4")
def container = startContainer(config, defaultServices())
def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
- def assetStorageService = container.getService(AssetStorageService.class)
+ def originalAssetStorageService = container.getService(AssetStorageService.class)
+ def assetStorageService = Spy(originalAssetStorageService)
+ [email protected](AssetStorageService.class, assetStorageService)
def assetDatapointService = container.getService(AssetDatapointService.class)
+ def messageBrokerService = container.getService(MessageBrokerService.class)
def persistenceService = container.getService(PersistenceService.class)
def originalMaxTuplesDecompressedPerDmlTransaction = getMaxTuplesDecompressedPerDmlTransaction(persistenceService)
+ def originalAssetDeleteDatapointBatchThreshold = assetStorageService.assetDeleteDatapointBatchThreshold
+ def originalAssetDeleteDatapointBatchWeeks = assetStorageService.assetDeleteDatapointBatchWeeks
+ assetStorageService.assetDeleteDatapointBatchThreshold = 100
+ assetStorageService.assetDeleteDatapointBatchWeeks = 2
+
+ def failedDeleteAssetIds = new CopyOnWriteArrayList<String>()
+ assetStorageService.deletePendingAsset(_) >> { String assetId ->
+ if (failedDeleteAssetIds.contains(assetId)) {
+ assetStorageService.failedAssetDeleteIds.add(assetId)
+ return
+ }
+ callRealMethod()
+ }
+
+ List<PersistenceEvent<Asset<?>>> assetPersistenceEvents = new CopyOnWriteArrayList<>()
+ def assetPersistenceRouteId = "Test-AssetPersistenceEvents-${System.nanoTime()}"
+ messageBrokerService.context.addRoutes(new RouteBuilder() {
+ @Override
+ void configure() {
+ from(PERSISTENCE_TOPIC)
+ .routeId(assetPersistenceRouteId)
+ .filter(isPersistenceEventForEntityType(Asset.class))
+ .process { exchange ->
+ def event = exchange.in.getBody(PersistenceEvent.class) as PersistenceEvent<Asset<?>>
+ if (event.cause in [PersistenceEvent.Cause.DELETE, PersistenceEvent.Cause.DELETE_FINISHED]) {
+ assetPersistenceEvents.add(event)
+ }
+ }
+ }
+ })
and: "the schema name is retrieved"
def schemaName = persistenceService.persistenceUnitProperties.getProperty(AvailableSettings.DEFAULT_SCHEMA)
@@ -207,12 +248,45 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerT
when: "the segment-delete assets are deleted from compressed chunks with a low decompression limit"
setMaxTuplesDecompressedPerDmlTransaction(persistenceService, 100)
+ def failedSegmentDeleteAssetId = segmentDeleteAssetIds.first()
+ failedDeleteAssetIds.add(failedSegmentDeleteAssetId)
def segmentDeleteSucceeded = assetStorageService.delete(segmentDeleteAssetIds)
- def segmentDeleteDatapointsAfterDelete = countDatapoints(persistenceService, segmentDeleteAssetIds, attributeNames)
then: "TimescaleDB should delete full segments without decompressing chunks"
segmentDeleteSucceeded
- segmentDeleteDatapointsAfterDelete == 0
+
+ and: "asset DELETE events should be published for all assets and DELETE_FINISHED for all but the failed asset"
+ conditions.eventually {
+ assert countDatapoints(persistenceService, segmentDeleteAssetIds, attributeNames) == 0
+
+ def deleteIds = assetPersistenceEvents
+ .findAll { it.cause == PersistenceEvent.Cause.DELETE }
+ .collect { it.entity.id }
+ def deleteFinishedIds = assetPersistenceEvents
+ .findAll { it.cause == PersistenceEvent.Cause.DELETE_FINISHED }
+ .collect { it.entity.id }
+
+ assert segmentDeleteAssetIds.every { deleteIds.contains(it) }
+ assert (segmentDeleteAssetIds - failedSegmentDeleteAssetId).every {
+ deleteFinishedIds.contains(it)
+ }
+ assert !deleteFinishedIds.contains(failedSegmentDeleteAssetId)
+ }
+
+ and: "the failed asset should be hidden from asset queries"
+ assetStorageService.find(new AssetQuery().ids(failedSegmentDeleteAssetId)) == null
+
+ when: "the failed asset deletion is retried"
+ failedDeleteAssetIds.remove(failedSegmentDeleteAssetId)
+ assetStorageService.retryFailedAssetDeletes()
+
+ then: "the failed asset should be physically deleted"
+ conditions.eventually {
+ assert assetPersistenceEvents.any {
+ it.cause == PersistenceEvent.Cause.DELETE_FINISHED && it.entity.id == failedSegmentDeleteAssetId
+ }
+ assert countAssets(persistenceService, failedSegmentDeleteAssetId) == 0L
+ }
when: "Purging will be called, count datapoints before purging"
def countBeforePurge = 0
@@ -292,7 +366,28 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerT
then: "No exception should have occurred"
true
- cleanup: "restore TimescaleDB decompression limit"
+ cleanup: "restore settings and remove the temporary route"
+ if (messageBrokerService != null && assetPersistenceRouteId != null) {
+ try {
+ messageBrokerService.context.routeController.stopRoute(assetPersistenceRouteId)
+ } catch (Exception ignored) {
+ // Route may not have been added if setup failed early.
+ }
+ try {
+ messageBrokerService.context.removeRoute(assetPersistenceRouteId)
+ } catch (Exception ignored) {
+ // Route may already be gone during container shutdown.
+ }
+ }
+ if (assetStorageService != null && originalAssetDeleteDatapointBatchThreshold != null) {
+ assetStorageService.assetDeleteDatapointBatchThreshold = originalAssetDeleteDatapointBatchThreshold
+ }
+ if (assetStorageService != null && originalAssetDeleteDatapointBatchWeeks != null) {
+ assetStorageService.assetDeleteDatapointBatchWeeks = originalAssetDeleteDatapointBatchWeeks
+ }
+ if (container != null && originalAssetStorageService != null) {
+ [email protected](AssetStorageService.class, originalAssetStorageService)
+ }
if (persistenceService != null && originalMaxTuplesDecompressedPerDmlTransaction != null) {
setMaxTuplesDecompressedPerDmlTransaction(persistenceService, originalMaxTuplesDecompressedPerDmlTransaction)
}
@@ -316,6 +411,14 @@ class AssetDatapointPurgeTest extends Specification implements ManagerContainerT
}
}
+ private static long countAssets(PersistenceService persistenceService, String assetId) {
+ return persistenceService.doReturningTransaction { em ->
+ def query = em.createNativeQuery("SELECT count(*) FROM asset WHERE id = :assetId")
+ query.setParameter("assetId", assetId)
+ return (query.getSingleResult() as Number).longValue()
+ }
+ }
+
private static String getMaxTuplesDecompressedPerDmlTransaction(PersistenceService persistenceService) {
return persistenceService.doReturningTransaction { em ->
em.createNativeQuery("SHOW timescaledb.max_tuples_decompressed_per_dml_transaction").getSingleResult() as String
diff --git a/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy b/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy
index 31d72689d..d03b0f8d3 100644
--- a/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/assets/AssetIntegrityTest.groovy
@@ -34,6 +34,7 @@ import org.openremote.model.asset.impl.ThingAsset
import org.openremote.model.attribute.Attribute
import org.openremote.model.attribute.AttributeState
import org.openremote.model.attribute.AttributeEvent
+import org.openremote.model.query.AssetQuery
import org.openremote.model.util.UniqueIdentifierGenerator
import org.openremote.model.value.ValueType
import org.openremote.setup.integration.KeycloakTestSetup
@@ -505,6 +506,81 @@ class AssetIntegrityTest extends Specification implements ManagerContainerTrait
(assetStorageService.find(gatewayDescendant.id, true) as RoomAsset).parentId == gateway.id
}
+ def "Asset deletion allows children that are already pending deletion"() {
+ given: "the server container is started"
+ def conditions = new PollingConditions(timeout: 10, delay: 0.2)
+ def container = startContainer(defaultConfig(), defaultServices())
+ def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
+ def originalAssetStorageService = container.getService(AssetStorageService.class)
+ def assetStorageService = Spy(originalAssetStorageService)
+ [email protected](AssetStorageService.class, assetStorageService)
+
+ and: "physical deletion is held back for selected assets"
+ def failedDeleteAssetIds = Collections.synchronizedSet(new HashSet<String>())
+ assetStorageService.deletePendingAsset(_) >> { String assetId ->
+ if (failedDeleteAssetIds.contains(assetId)) {
+ assetStorageService.failedAssetDeleteIds.add(assetId)
+ return false
+ }
+ callRealMethod()
+ }
+
+ and: "there is a parent with a child"
+ def parentAsset = assetStorageService.merge(new RoomAsset("Pending delete parent")
+ .setRealm(keycloakTestSetup.realmMaster.name))
+ def childAsset = assetStorageService.merge(new RoomAsset("Pending delete child")
+ .setRealm(keycloakTestSetup.realmMaster.name)
+ .setParentId(parentAsset.id))
+
+ when: "the child is marked for deletion but physical deletion has not completed"
+ failedDeleteAssetIds.add(childAsset.id)
+ def childAccepted = assetStorageService.delete([childAsset.id])
+
+ then: "the child is pending deletion so can only be queried through the count include delete pending endpoint"
+ childAccepted
+ conditions.eventually {
+ assert assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 0
+ assert assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(childAsset.id)) == 1
+ }
+
+ when: "the parent is deleted while the child is still pending deletion"
+ failedDeleteAssetIds.add(parentAsset.id)
+ def parentAccepted = assetStorageService.delete([parentAsset.id])
+
+ then: "the parent is accepted for deletion"
+ parentAccepted
+ conditions.eventually {
+ assert assetStorageService.count(new AssetQuery().ids(parentAsset.id)) == 0
+ assert assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(parentAsset.id)) == 1
+ }
+
+ when: "the child asset is re-merged while it is still pending deletion"
+ failedDeleteAssetIds.remove(childAsset.id)
+ childAsset.setName("Restored pending delete child")
+ def restoredChildAsset = assetStorageService.merge(childAsset)
+
+ then: "the child deletion is cancelled and the asset is visible again"
+ restoredChildAsset.id == childAsset.id
+ restoredChildAsset.name == "Restored pending delete child"
+ !restoredChildAsset.isDeletePending()
+ conditions.eventually {
+ assert assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 1
+ assert !assetStorageService.failedAssetDeleteIds.contains(childAsset.id)
+ }
+
+ when: "pending asset deletion is retried"
+ assetStorageService.requestPendingAssetDeletionRetry([childAsset.id])
+ TimeUnit.MILLISECONDS.sleep(500)
+
+ then: "the restored child is not physically deleted by the background deletion task"
+ assetStorageService.count(new AssetQuery().ids(childAsset.id)) == 1
+ assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(childAsset.id)) == 1
+
+ and: "the parent remains pending deletion"
+ assetStorageService.count(new AssetQuery().ids(parentAsset.id)) == 0
+ assetStorageService.count(new AssetQuery().includeDeletePending(true).ids(parentAsset.id)) == 1
+ }
+
def "Test writing attributes with timestamps"() {
given: "the server container is started"
def container = startContainer(defaultConfig(), defaultServices())
diff --git a/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy b/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy
index 87d6d394c..1afab8827 100644
--- a/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy
+++ b/test/src/test/groovy/org/openremote/test/rules/BasicRulesDeploymentTest.groovy
@@ -18,26 +18,34 @@
*/
package org.openremote.test.rules
+import jakarta.ws.rs.WebApplicationException
+import org.openremote.manager.asset.AssetStorageService
import org.openremote.manager.rules.RulesEngine
import org.openremote.manager.rules.RulesService
import org.openremote.manager.rules.RulesetStorageService
import org.openremote.manager.security.ManagerIdentityService
import org.openremote.manager.setup.SetupService
+import org.openremote.model.query.AssetQuery
+import org.openremote.model.query.filter.RealmPredicate
import org.openremote.setup.integration.KeycloakTestSetup
import org.openremote.setup.integration.ManagerTestSetup
import org.openremote.model.rules.AssetRuleset
import org.openremote.model.rules.GlobalRuleset
import org.openremote.model.rules.RealmRuleset
+import org.openremote.model.security.RealmResource
import org.openremote.test.ManagerContainerTrait
import spock.lang.Specification
import spock.util.concurrent.PollingConditions
+import java.util.concurrent.TimeUnit
+
import static org.openremote.model.util.MapAccess.getString
import static org.openremote.manager.security.ManagerIdentityProvider.OR_ADMIN_PASSWORD
import static org.openremote.manager.security.ManagerIdentityProvider.OR_ADMIN_PASSWORD_DEFAULT
import static org.openremote.model.Constants.*
import static org.openremote.model.rules.Ruleset.Lang.GROOVY
import static org.openremote.model.rules.RulesetStatus.*
+import static jakarta.ws.rs.core.Response.Status.CONFLICT
class BasicRulesDeploymentTest extends Specification implements ManagerContainerTrait {
@@ -53,6 +61,7 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainer
def keycloakTestSetup = container.getService(SetupService.class).getTaskOfType(KeycloakTestSetup.class)
def rulesService = container.getService(RulesService.class)
def identityService = container.getService(ManagerIdentityService.class)
+ def assetStorageService = container.getService(AssetStorageService.class)
def rulesetStorageService = container.getService(RulesetStorageService.class)
and: "some test rulesets have been imported"
@@ -66,6 +75,7 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainer
MASTER_REALM_ADMIN_USER,
getString(container.getConfig(), OR_ADMIN_PASSWORD, OR_ADMIN_PASSWORD_DEFAULT)
)
+ def realmResource = getClientApiTarget(serverUri(serverPort), MASTER_REALM, accessToken).proxy(RealmResource.class)
expect: "the rules engines to be ready"
conditions.eventually {
@@ -297,8 +307,25 @@ class BasicRulesDeploymentTest extends Specification implements ManagerContainer
assert !realmBuildingEngine.isError()
}
- when: "a realm is deleted"
- identityService.getIdentityProvider().deleteRealm(realmBuilding.getName())
+ when: "a realm containing assets is deleted"
+ realmResource.delete(null, realmBuilding.getName())
+
+ then: "the delete is rejected with a conflict"
+ WebApplicationException ex = thrown()
+ try {
+ assert ex.response.status == CONFLICT.statusCode
+ } finally {
+ ex.response.close()
+ }
+
+ when: "the realm assets are deleted and the realm delete is retried"
+ def buildingAssetIds = assetStorageService.findAll(new AssetQuery()
+ .select(new AssetQuery.Select().excludeAttributes())
+ .realm(new RealmPredicate(realmBuilding.getName())))
+ .collect { it.id }
+
+ assert assetStorageService.deleteUntilFinished(buildingAssetIds).get(10, TimeUnit.SECONDS)
+ realmResource.delete(null, realmBuilding.getName())
then: "the realms rule engine should stop and all asset rule engines in this realm should also stop"
conditions.eventually {
diff --git a/ui/app/manager/src/pages/page-realms.ts b/ui/app/manager/src/pages/page-realms.ts
index 7bd9bc70e..f5c02710c 100644
--- a/ui/app/manager/src/pages/page-realms.ts
+++ b/ui/app/manager/src/pages/page-realms.ts
@@ -529,7 +529,8 @@ export class PageRealms extends Page<AppStateKeyed> {
this._realms = this._realms.filter((r) => r !== realm);
this.dispatchEvent(new CustomEvent("realms-changed", { bubbles: true, composed: true }));
} catch (e) {
- showSnackbar(undefined, "realmDeleteFailed", "dismiss");
+ const snackbarKey = (e as any)?.response?.status === 409 ? "realmDeleteFailedHasAssets" : "realmDeleteFailed";
+ showSnackbar(undefined, snackbarKey, "dismiss");
}
};
diff --git a/ui/app/manager/test/fixtures/manager.ts b/ui/app/manager/test/fixtures/manager.ts
index 37ba1ea33..33426ea91 100644
--- a/ui/app/manager/test/fixtures/manager.ts
+++ b/ui/app/manager/test/fixtures/manager.ts
@@ -565,6 +565,52 @@ export class Manager {
}
}
+ /**
+ * Deletes all assets in a realm
+ * @param realm The realm
+ * @param config The axios request config
+ */
+ async deleteAssetsInRealm(realm: string, config?: AxiosRequestConfig<any>) {
+ config = config ?? (await this.adminConfig());
+
+ const assetIdRetreiver = async () => {
+ const assetResponse = await rest.api.AssetResource.queryAssets(
+ {
+ select: { attributes: [] },
+ realm: { name: realm },
+ },
+ config
+ );
+ expect(assetResponse.status).toBe(200);
+ const assets = assetResponse.data;
+ return assets.map(({ id }) => id!);
+ };
+
+ let assetIds = await assetIdRetreiver();
+
+ while (assetIds.length > 0) {
+ const response = await rest.api.AssetResource.delete({ assetId: assetIds }, config);
+ expect(response.status).toBe(204);
+ // Wait for these assets to be deleted
+ await expect
+ .poll(async () => {
+ const countResponse = await rest.api.AssetResource.queryCount(
+ {
+ select: { attributes: [] },
+ ids: assetIds,
+ realm: { name: realm },
+ includeDeletePending: true,
+ },
+ config
+ );
+ expect(countResponse.status).toBe(200);
+ return countResponse.data;
+ })
+ .toBe(0);
+ assetIds = await assetIdRetreiver();
+ }
+ }
+
/**
* Delete role
*
diff --git a/ui/app/manager/test/test.cleanup.ts b/ui/app/manager/test/test.cleanup.ts
index ea2f4c38a..1584d93d8 100644
--- a/ui/app/manager/test/test.cleanup.ts
+++ b/ui/app/manager/test/test.cleanup.ts
@@ -48,6 +48,7 @@ cleanup(`Delete the "smartcity" user`, async ({ page, manager }) => {
* @and The realm picker should no longer be visible
*/
cleanup(`Delete the "smartcity" realm`, async ({ page, manager, realmsPage }) => {
+ await manager.deleteAssetsInRealm("smartcity");
await manager.goToRealmStartPage("master");
await manager.navigateToMenuItem("Realms");
await realmsPage.deleteRealm("smartcity");
diff --git a/ui/app/shared/locales/cs/or.json b/ui/app/shared/locales/cs/or.json
index ba0b7a5f4..1f78bcfff 100644
--- a/ui/app/shared/locales/cs/or.json
+++ b/ui/app/shared/locales/cs/or.json
@@ -830,7 +830,7 @@
"attributeValueLabel": "Hodnota atributu",
"assetTypeDoesNotExist": "Filtr: Typ assetu neexistuje"
},
- "realmDeleteConfirm": "Varování! Tímto smažete vše v tomto realmu, pro potvrzení zadejte název realmu „{{realmName}}“.",
+ "realmDeleteConfirm": "Varování! Realm nesmí obsahovat žádná aktiva, aby jej bylo možné smazat, tímto automaticky smažete vše ostatní v tomto realmu, pro potvrzení zadejte název realmu „{{realmName}}“.",
"realmDeleteFailed": "Nepodařilo se smazat realm",
"addRule": "Přidat pravidlo",
"rulesetStatus": {
diff --git a/ui/app/shared/locales/en/or.json b/ui/app/shared/locales/en/or.json
index c869cc36a..0be4b945e 100644
--- a/ui/app/shared/locales/en/or.json
+++ b/ui/app/shared/locales/en/or.json
@@ -830,8 +830,9 @@
"attributeValueLabel": "Attribute value",
"assetTypeDoesNotExist": "Filter : Asset type does not exist"
},
- "realmDeleteConfirm": "Warning! this will delete everything in this realm, type the realm name '{{realmName}}' to confirm.",
+ "realmDeleteConfirm": "Warning! Realm must contain no assets to allow deletion, this will delete everything else in this realm automatically, type the realm name '{{realmName}}' to confirm.",
"realmDeleteFailed": "Failed to delete realm",
+ "realmDeleteFailedHasAssets": "Failed to delete realm because it still contains assets",
"addRule": "Add rule",
"rulesetStatus": {
"DEPLOYED": "Successfully deployed",
diff --git a/ui/app/shared/locales/nl/or.json b/ui/app/shared/locales/nl/or.json
index 904e309ff..01f9b9a7f 100644
--- a/ui/app/shared/locales/nl/or.json
+++ b/ui/app/shared/locales/nl/or.json
@@ -820,8 +820,9 @@
"attributeValueLabel": "Attribuutwaarde",
"assetTypeDoesNotExist": "Filter: Asset type bestaat niet"
},
- "realmDeleteConfirm": "Let op! Hiermee verwijder je alles in deze realm, typ de realm naam '{{realmName}}' om te bevestigen.",
+ "realmDeleteConfirm": "Let op! De realm mag geen assets bevatten om te kunnen worden verwijderd, hiermee verwijder je automatisch al het andere in deze realm, typ de realm naam '{{realmName}}' om te bevestigen.",
"realmDeleteFailed": "Realm verwijderen mislukt",
+ "realmDeleteFailedHasAssets": "Realm verwijderen mislukt omdat deze nog assets bevat",
"addRule": "Regel toevoegen",
"rulesetStatus": {
"DEPLOYED": "Succesvol aangemaakt",
diff --git a/ui/app/shared/locales/pl/or.json b/ui/app/shared/locales/pl/or.json
index 3cb1b9efc..ef42a3ced 100644
--- a/ui/app/shared/locales/pl/or.json
+++ b/ui/app/shared/locales/pl/or.json
@@ -1102,7 +1102,7 @@
"attributeValueLabel": "Wartość atrybutu",
"assetTypeDoesNotExist": "Filtr: typ zasobu nie istnieje"
},
- "realmDeleteConfirm": "Uwaga! Spowoduje to usunięcie wszystkich danych w tym obszarze. Wpisz nazwę obszaru '{{realmName}}', aby potwierdzić.",
+ "realmDeleteConfirm": "Uwaga! Obszar nie może zawierać żadnych zasobów, aby można go było usunąć. Spowoduje to automatyczne usunięcie wszystkich pozostałych elementów w tym obszarze, wpisz nazwę obszaru '{{realmName}}', aby potwierdzić.",
"realmDeleteFailed": "Nie udało się usunąć obszaru",
"addRule": "Dodaj regułę",
"rulesetStatus": {
diff --git a/ui/app/shared/locales/ro/or.json b/ui/app/shared/locales/ro/or.json
index 6a5ca0555..0d6e8949f 100644
--- a/ui/app/shared/locales/ro/or.json
+++ b/ui/app/shared/locales/ro/or.json
@@ -574,7 +574,7 @@
"attributeValueLabel": "Valoarea atributului",
"assetTypeDoesNotExist": "Filtrare : Tipul de obiect nu exista"
},
- "realmDeleteConfirm": "Avertisment! se va sterge totul în acest domeniu, scrie numele domeniului '{{realmName}}' pentru a confirma.",
+ "realmDeleteConfirm": "Avertisment! Domeniul nu trebuie să conțină active pentru a putea fi șters, această acțiune va șterge automat tot ce se mai află în acest domeniu, scrie numele domeniului '{{realmName}}' pentru a confirma.",
"realmDeleteFailed": "Stergerea domeniului esuata",
"rulesetStatus": {
"DEPLOYED": "Instalat cu succes",
diff --git a/ui/app/shared/locales/uk/or.json b/ui/app/shared/locales/uk/or.json
index a17e5ddeb..a605e5b9f 100644
--- a/ui/app/shared/locales/uk/or.json
+++ b/ui/app/shared/locales/uk/or.json
@@ -847,7 +847,7 @@
"attributeValueLabel": "Значення атрибута",
"assetTypeDoesNotExist": "Фільтр: Тип активу не існує"
},
- "realmDeleteConfirm": "Попередження! Це видалить усе в цьому реалмі, введіть назву реалму '{{realmName}}', щоб підтвердити.",
+ "realmDeleteConfirm": "Попередження! Реалм не повинен містити активів, щоб його можна було видалити, це автоматично видалить усе інше в цьому реалмі, введіть назву реалму '{{realmName}}', щоб підтвердити.",
"realmDeleteFailed": "Не вдалося видалити реалм",
"addRule": "Додати правило",
"rulesetStatus": {
diff --git a/ui/test/app.config.cts b/ui/test/app.config.cts
index 4378bf555..62f640c0b 100644
--- a/ui/test/app.config.cts
+++ b/ui/test/app.config.cts
@@ -84,6 +84,7 @@ export const defineAppConfig = (path: string) => {
video: "on",
locale: "en",
},
+ timeout: 60_000,
webServer: {
command: `node ${join(__dirname, "manager.cjs")}`,
url: "http://127.0.0.1:8080",