100% open-source IoT Platform - Integrate your devices, add intelligence, build custom apps
Java asset-managementbuildingdashboarddataplatformdevice-managementedge-computingenergyenergy-managementenergy-monitorfleet-managementfleet-telematicsiotiot-applicationiot-platformmodbusmqttmqtt-brokerota-updaterules-enginesmartcity
Fix thread starvation issue with LockByKey (#2117)
Rich Turner7471a8d07f66
diff --git a/model/src/main/java/org/openremote/model/util/LockByKey.java b/model/src/main/java/org/openremote/model/util/LockByKey.javaindex aad01008f..1c3c67a76 100644--- a/model/src/main/java/org/openremote/model/util/LockByKey.java+++ b/model/src/main/java/org/openremote/model/util/LockByKey.java@@ -12,31 +12,45 @@ import java.util.concurrent.atomic.AtomicInteger;public class LockByKey {- private static final System.Logger LOG = System.getLogger(LockByKey.class.getName());+ protected static class LockWrapper {+ protected final Semaphore lock = new Semaphore(1); // Don't change to ReentrantLock as camel switches thread between try/finally blocks on route processors+ protected final AtomicInteger numberOfThreadsInQueue = new AtomicInteger(1);- private static class LockWrapper {- private final Semaphore lock = new Semaphore(1); // Don't change to ReentrantLock as camel switches thread between try/finally blocks on route processors- private final AtomicInteger numberOfThreadsInQueue = new AtomicInteger(1);-- private LockWrapper addThreadInQueue() {+ protected LockWrapper addThreadInQueue() {numberOfThreadsInQueue.incrementAndGet();return this;}- private int removeThreadFromQueue() {+ protected int removeThreadFromQueue() {return numberOfThreadsInQueue.decrementAndGet();}}- private final Map<String, LockWrapper> locks = new ConcurrentHashMap<>();+ protected static final System.Logger LOG = System.getLogger(LockByKey.class.getName());+ protected final Map<String, LockWrapper> locks = new ConcurrentHashMap<>();++ /**+ * This method is here for overriding purposes in tests+ */+ @SuppressWarnings("unused")+ protected LockWrapper createLockWrapper() {+ return new LockWrapper();+ }public void lock(String key) {- LockWrapper lockWrapper = locks.compute(key, (k, v) -> v == null ? new LockWrapper() : v.addThreadInQueue());+ LockWrapper lockWrapper = locks.compute(key, (k, v) -> v == null ? createLockWrapper() : v.addThreadInQueue());try {lockWrapper.lock.acquire();LOG.log(System.Logger.Level.TRACE, () -> "Lock acquired: key=" + key);} catch (InterruptedException e) {+ // Atomically check the queue count and remove the wrapper if it's the last one.+ locks.computeIfPresent(key, (k, v) -> {+ if (v.removeThreadFromQueue() == 0) {+ return null; // Atomically remove if the queue is empty+ }+ return v;+ });Thread.currentThread().interrupt(); // Restore interrupted statusthrow new RuntimeException("Interrupted while acquiring lock for key: " + key, e);}@@ -47,11 +61,12 @@ public class LockByKey {if (lockWrapper != null) {LOG.log(System.Logger.Level.TRACE, () -> "Lock release: key=" + key);lockWrapper.lock.release();- if (lockWrapper.removeThreadFromQueue() == 0) {- // NB : We pass in the specific value to remove to handle the case where another thread would queue right before the removal- locks.remove(key, lockWrapper);- }+ locks.computeIfPresent(key, (k, v) -> {+ if (v.removeThreadFromQueue() == 0) {+ return null; // Atomically remove if queue is empty+ }+ return v;+ });}}-}diff --git a/model/src/test/java/org/openremote/model/util/LockByKeyTest.java b/model/src/test/java/org/openremote/model/util/LockByKeyTest.javanew file mode 100644index 000000000..8eff6e7ff--- /dev/null+++ b/model/src/test/java/org/openremote/model/util/LockByKeyTest.java@@ -0,0 +1,138 @@+package org.openremote.model.util;++import org.junit.jupiter.api.Test;++import java.util.concurrent.*;++import static org.junit.jupiter.api.Assertions.*;++public class LockByKeyTest {+ /**+ * A controllable version of LockByKey that allows us to pause the unlock method at a critical point.+ */+ static class ControllableLockByKey extends LockByKey {++ private final CountDownLatch pauseLatch = new CountDownLatch(1);+ private final CountDownLatch resumeLatch = new CountDownLatch(1);++ @Override+ protected LockWrapper createLockWrapper() {+ return new LockWrapper() {+ @Override+ public int removeThreadFromQueue() {+ int result = super.removeThreadFromQueue();+ if (result == 0) {+ // Signal that we are at the critical point+ pauseLatch.countDown();+ // Wait for the test to tell us to continue+ try {+ resumeLatch.await();+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ }+ return result;+ }+ };+ }++ public void waitForPause() throws InterruptedException {+ pauseLatch.await(5, TimeUnit.SECONDS);+ }++ public void resume() {+ resumeLatch.countDown();+ }++ public int getQueueSize(String key) {+ LockWrapper lockWrapper = locks.get(key);+ if (lockWrapper != null) {+ return lockWrapper.numberOfThreadsInQueue.get();+ }+ return 0;+ }+ }++ @Test+ void testThreadStarvation() throws InterruptedException {+ final ControllableLockByKey lockByKey = new ControllableLockByKey();+ final String key = "testKey";+ final ExecutorService executor = Executors.newFixedThreadPool(3);+ final CountDownLatch threadWaitingLatch = new CountDownLatch(2);+ final CountDownLatch threadResumeLatch = new CountDownLatch(1);++ // Thread 1: Acquires and releases the lock, pausing in the middle of unlock+ Future<?> future1 = executor.submit(() -> {+ lockByKey.lock(key);+ lockByKey.unlock(key);+ });++ // Wait for Thread 1 to pause in the unlock method+ lockByKey.waitForPause();++ // Thread 2: Tries to acquire the lock while Thread 1 is trying to unlock+ Future<?> future2 = executor.submit(() -> {+ try {+ threadWaitingLatch.countDown();+ lockByKey.lock(key);+ threadResumeLatch.await(5, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ lockByKey.unlock(key);+ });++ // Thread 3: Tries to acquire the lock while Thread 1 is trying to unlock+ Future<?> future3 = executor.submit(() -> {+ try {+ Thread.sleep(1000);+ threadWaitingLatch.countDown();+ lockByKey.lock(key);+ threadResumeLatch.await(5, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ throw new RuntimeException(e);+ }+ lockByKey.unlock(key);+ });++ // Wait for Thread 2 & 3 to try and acquire the lock+ threadWaitingLatch.await(5, TimeUnit.SECONDS);++ var counter = 0;+ while (lockByKey.getQueueSize(key) != 2 && counter < 10) {+ Thread.sleep(100);+ counter++;+ }++ // Allow Thread 1 to complete its unlock and remove the LockWrapper+ lockByKey.resume();++ // Wait for Thread 1 to finish+ try {+ future1.get(1, TimeUnit.SECONDS);+ } catch (Exception e) {+ fail("Thread 1 failed to complete", e);+ }+ assertTrue(future1.isDone(), "Thread 1 should have completed");++ // Allow Thread 2 & 3 to continue+ threadResumeLatch.countDown();++ // Wait for Thread 2 & 3 to finish+ try {+ future2.get(1, TimeUnit.SECONDS);+ future3.get(1, TimeUnit.SECONDS);+ } catch (TimeoutException e) {+ // We expect a timeout here, which indicates a thread is stuck+ } catch (Exception e) {+ fail("An unexpected exception occurred", e);+ }++ assertTrue(future2.isDone(), "Thread 2 should have completed");++ // Thread 3 should complete+ assertTrue(future3.isDone(), "Thread 3 should have completed");++ executor.shutdownNow();+ }+}