The Unified and Autonomous Development Platform
feat: Use FIFO when CI/CD job acquires resources
Robin Shenb24ed9ed954d
diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.javaindex 7306494380..7acddb4f90 100644--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java@@ -42,6 +42,7 @@ import java.util.Comparator;import java.util.Date;import java.util.HashMap;import java.util.HashSet;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;import java.util.Optional;@@ -386,12 +387,12 @@ public class DefaultBuildService extends BaseEntityService<Build> implements Bui@Overridepublic Map<Long, Long> queryUnfinished() {Query<?> query = getSession().createQuery("select id, project.id from Build where "- + "status=:waiting or status=:pending or status=:running", Object[].class);+ + "status=:waiting or status=:pending or status=:running order by submitDate", Object[].class);query.setParameter("waiting", Build.Status.WAITING);query.setParameter("pending", Build.Status.PENDING);query.setParameter("running", Build.Status.RUNNING);- Map<Long, Long> result = new HashMap<>();+ Map<Long, Long> result = new LinkedHashMap<>();for (Object[] fields: (List<Object[]>)query.list())result.put((Long)fields[0], (Long)fields[1]);return result;diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.javaindex f08b46f613..0ce4fa7610 100644--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java@@ -16,13 +16,13 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.concurrent.Future;+import java.util.function.Function;+import java.util.function.Supplier;import org.jspecify.annotations.Nullable;import jakarta.inject.Inject;import jakarta.inject.Singleton;-import org.quartz.ScheduleBuilder;-import org.quartz.SimpleScheduleBuilder;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -38,21 +38,16 @@ import io.onedev.server.service.support.AgentCallable;import io.onedev.server.event.Listen;import io.onedev.server.event.agent.AgentConnected;import io.onedev.server.event.agent.AgentDisconnected;-import io.onedev.server.event.cluster.NodeConnected;-import io.onedev.server.event.system.SystemStarted;import io.onedev.server.event.system.SystemStarting;import io.onedev.server.event.system.SystemStopped;-import io.onedev.server.event.system.SystemStopping;import io.onedev.server.model.AbstractEntity;import io.onedev.server.persistence.TransactionService;import io.onedev.server.persistence.annotation.Transactional;import io.onedev.server.search.entity.agent.AgentQuery;-import io.onedev.server.taskschedule.SchedulableTask;-import io.onedev.server.taskschedule.TaskScheduler;import oshi.SystemInfo;@Singleton-public class DefaultResourceService implements ResourceService, Serializable, SchedulableTask {+public class DefaultResourceService implements ResourceService, Serializable {private static final Logger logger = LoggerFactory.getLogger(DefaultResourceService.class);@@ -62,23 +57,36 @@ public class DefaultResourceService implements ResourceService, Serializable, Scprivate final TransactionService transactionService;- private final TaskScheduler taskScheduler;-private volatile Map<String, Integer> cpuCounts;+ /*+ * Only the leader allocates resources. These counters intentionally are not replicated:+ * after failover, existing tasks may temporarily exceed the limit until they finish.+ * Each coordinating task releases its reservation on the same server that acquired it.+ */private final Map<String, Integer> concurrencyUsages = new HashMap<>();-- private volatile Map<String, Integer> concurrencyUsagesCache;-- private volatile String taskId;++ // Guarded by concurrencyUsages, in order of arrival at the leader.+ private final List<AllocationRequest> allocationRequests = new ArrayList<>();++ private static class AllocationRequest {++ private final String resourceName;++ private Collection<String> nodes = Set.of();++ private AllocationRequest(String resourceName) {+ this.resourceName = resourceName;+ }++ }@Injectpublic DefaultResourceService(AgentService agentService, TransactionService transactionService,- ClusterService clusterService, TaskScheduler taskScheduler) {+ ClusterService clusterService) {this.agentService = agentService;this.transactionService = transactionService;this.clusterService = clusterService;- this.taskScheduler = taskScheduler;}public Object writeReplace() throws ObjectStreamException {@@ -99,8 +107,6 @@ public class DefaultResourceService implements ResourceService, Serializable, Sclogger.debug("Error calling oshi", e);cpuCounts.put(localServer, 4);}- concurrencyUsagesCache = hazelcastInstance.getReplicatedMap("concurrencyUsagesCache");- removeNodeFromConcurrencyUsagesCache(localServer);}@Listen@@ -109,47 +115,6 @@ public class DefaultResourceService implements ResourceService, Serializable, SccpuCounts.remove(clusterService.getLocalServerAddress());}- @Listen- public void on(SystemStarted event) {- taskId = taskScheduler.schedule(this);- }-- @Listen- public void on(SystemStopping event) {- if (taskId != null)- taskScheduler.unschedule(taskId);- }-- @Listen- public void on(NodeConnected event) {- if (clusterService.isLeaderServer() && event.isRecovered()) {- clusterService.submitToServer(event.getServer(), () -> {- try {- while (true) {- synchronized (concurrencyUsages) {- if (concurrencyUsagesCache != null) {- concurrencyUsagesCache.putAll(concurrencyUsages);- break;- }- }- Thread.sleep(100);- }- } catch (Throwable e) {- logger.error("Error syncing concurrency usages cache", e);- }- return null;- });- }- }-- private void removeNodeFromConcurrencyUsagesCache(String node) {- var keysToRemove = concurrencyUsagesCache.keySet().stream()- .filter(it -> it.startsWith(node + ":"))- .collect(toList());- for (var key: keysToRemove)- concurrencyUsagesCache.remove(key);- }-@Transactional@Listenpublic void on(AgentConnected event) {@@ -157,13 +122,6 @@ public class DefaultResourceService implements ResourceService, Serializable, Scvar agentCpuCount = event.getAgent().getCpuCount();transactionService.runAfterCommit(() -> {cpuCounts.put(String.valueOf(agentId), agentCpuCount);- removeNodeFromConcurrencyUsagesCache(agentId.toString());- synchronized (concurrencyUsages) {- for (var entry : concurrencyUsages.entrySet()) {- if (entry.getKey().startsWith(agentId + ":"))- concurrencyUsagesCache.put(entry.getKey(), entry.getValue());- }- }});}@@ -200,20 +158,62 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc}}+ private String acquireNode(Supplier<Collection<String>> candidateNodes, String resourceName,+ int totalConcurrency, int requiredConcurrency) throws InterruptedException {+ var request = new AllocationRequest(resourceName);+ synchronized (concurrencyUsages) {+ allocationRequests.add(request);+ }+ try {+ while (true) {+ if (Thread.interrupted())+ throw new InterruptedException();+ if (!clusterService.isLeaderServer())+ throw new ExplicitException("Cluster leader changed, please retry later");+ // Agent queries may access the database, so resolve candidates outside the monitor.+ var nodes = new HashSet<>(candidateNodes.get());+ nodes.removeIf(node -> getEffectiveTotalConcurrency(node, totalConcurrency) < requiredConcurrency);+ synchronized (concurrencyUsages) {+ if (Thread.interrupted())+ throw new InterruptedException();+ request.nodes = nodes;+ var availableNodes = new HashSet<>(nodes);+ // Leave shared nodes to older requests, even if a smaller request could fit now.+ // Unrelated nodes/resources and requests with no eligible nodes do not block us.+ for (var earlier : allocationRequests) {+ if (earlier == request)+ break;+ if (earlier.resourceName.equals(resourceName))+ availableNodes.removeAll(earlier.nodes);+ }+ var node = allocateNode(availableNodes, resourceName, totalConcurrency, requiredConcurrency);+ if (node != null) {+ concurrencyUsages.merge(node + ":" + resourceName, requiredConcurrency, Integer::sum);+ return node;+ }+ // Poll capacity, eligibility, and leadership once per second to avoid wake-up storms.+ concurrencyUsages.wait(1000);+ }+ }+ } finally {+ synchronized (concurrencyUsages) {+ allocationRequests.remove(request);+ }+ }+ }++ // Called with concurrencyUsages locked, so selection and reservation are atomic.@Nullableprivate String allocateNode(Collection<String> nodes, String resourceName,- int totalConcurrency, int requiredConcurrency) {+ int totalConcurrency, int requiredConcurrency) {String allocatedNode = null;var maxScore = 0;var nodeList = new ArrayList<>(nodes);Collections.shuffle(nodeList);- for (var node: nodeList) {+ for (var node : nodeList) {var effectiveTotalConcurrency = getEffectiveTotalConcurrency(node, totalConcurrency);- var usedConcurrency = concurrencyUsagesCache.get(node + ":" + resourceName);- if (usedConcurrency == null)- usedConcurrency = 0;+ var usedConcurrency = concurrencyUsages.getOrDefault(node + ":" + resourceName, 0);var score = getAllocationScore(effectiveTotalConcurrency, usedConcurrency, requiredConcurrency);-if (score > maxScore) {allocatedNode = node;maxScore = score;@@ -222,69 +222,43 @@ public class DefaultResourceService implements ResourceService, Serializable, Screturn allocatedNode;}- private void acquireConcurrency(String concurrencyKey, int totalConcurrency, int acquireConcurrency) {- while (true) {- synchronized (concurrencyUsages) {- var usedCocurrency = concurrencyUsages.get(concurrencyKey);- if (usedCocurrency == null)- usedCocurrency = 0;- usedCocurrency += acquireConcurrency;- if (usedCocurrency <= totalConcurrency) {- concurrencyUsages.put(concurrencyKey, usedCocurrency);- concurrencyUsagesCache.put(concurrencyKey, usedCocurrency);- break;- }- }- try {- Thread.sleep(100);- } catch (InterruptedException e) {- throw new RuntimeException(e);- }+ private void releaseConcurrency(String concurrencyKey, int releaseConcurrency) {+ synchronized (concurrencyUsages) {+ var usedConcurrency = concurrencyUsages.get(concurrencyKey) - releaseConcurrency;+ if (usedConcurrency != 0)+ concurrencyUsages.put(concurrencyKey, usedConcurrency);+ else+ concurrencyUsages.remove(concurrencyKey);}}- private void releaseConcurrency(String concurrencyKey, int releaseConcurrency) {- synchronized (concurrencyUsages) {- var usedResources = concurrencyUsages.get(concurrencyKey);- usedResources -= releaseConcurrency;- concurrencyUsages.put(concurrencyKey, usedResources);- concurrencyUsagesCache.put(concurrencyKey, usedResources);+ private <T> T runTask(Supplier<Collection<String>> candidateNodes, String resourceName,+ int totalConcurrency, int requiredConcurrency, Function<String, T> task) throws InterruptedException {+ var node = acquireNode(candidateNodes, resourceName, totalConcurrency, requiredConcurrency);+ try {+ return task.apply(node);+ } finally {+ releaseConcurrency(node + ":" + resourceName, requiredConcurrency);}}@Overridepublic <T> Future<T> submitServerTask(@Nullable String pinnedServerAddress, String resourceName,- int totalConcurrency, int requiredConcurrency, ClusterTask<T> runnable) {- while (true) {- var candidateServers = new ArrayList<>(clusterService.getOnlineServers());- if (pinnedServerAddress != null)+ int totalConcurrency, int requiredConcurrency, ClusterTask<T> task) {+ return clusterService.submitToServer(clusterService.getLeaderServerAddress(), () -> runTask(() -> {+ var candidateServers = new ArrayList<>(clusterService.getOnlineServers());+ candidateServers.retainAll(new HashSet<>(clusterService.getServerAddresses()));+ if (pinnedServerAddress != null)candidateServers.retainAll(List.of(pinnedServerAddress));- var allocatedServer = allocateNode(candidateServers, resourceName, totalConcurrency, requiredConcurrency);- if (allocatedServer != null) {- return clusterService.submitToServer(allocatedServer, () -> {- int effectiveTotalConcurrency = getEffectiveTotalConcurrency(allocatedServer, totalConcurrency);- var concurrencyKey = allocatedServer + ":" + resourceName;- acquireConcurrency(concurrencyKey, effectiveTotalConcurrency, requiredConcurrency);- try {- return runnable.call();- } finally {- releaseConcurrency(concurrencyKey, requiredConcurrency);- }- });- }- try {- Thread.sleep(1000);- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- }+ return candidateServers;+ }, resourceName, totalConcurrency, requiredConcurrency,+ server -> clusterService.runOnServer(server, task)));}@Override- public <T> Future<T> submitAgentTask(@Nullable Long pinnedAgentId, AgentQuery agentQuery,- String resourceName, int totalConcurrency, int requiredConcurrency,- AgentCallable<T> task) {- while (true) {+ public <T> Future<T> submitAgentTask(@Nullable Long pinnedAgentId, AgentQuery agentQuery,+ String resourceName, int totalConcurrency, int requiredConcurrency, AgentCallable<T> task) {+ return clusterService.submitToServer(clusterService.getLeaderServerAddress(), () -> runTask(() -> {Set<Long> agentIds;if (pinnedAgentId != null) {agentIds = new HashSet<>();@@ -297,45 +271,17 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc.map(AbstractEntity::getId).collect(toSet());}- var agentIdString = allocateNode(- agentIds.stream().map(Object::toString).collect(toList()),- resourceName, totalConcurrency, requiredConcurrency);- var agentId = agentIdString != null? Long.valueOf(agentIdString): null;- if (agentId != null) {- var server = agentService.getAgentServer(agentId);- if (server == null)- throw new ExplicitException("Cannot find server managing allocated agent, please retry later");-- return clusterService.submitToServer(server, () -> {- var effectiveTotalConcurrency = getEffectiveTotalConcurrency(agentIdString, totalConcurrency);- var concurrencyKey = agentId + ":" + resourceName;- acquireConcurrency(concurrencyKey, effectiveTotalConcurrency, requiredConcurrency);- try {- updateLastUsedDate(agentId);- return task.call(agentId);- } finally {- releaseConcurrency(concurrencyKey, requiredConcurrency);- }- });- }- try {- Thread.sleep(1000);- } catch (InterruptedException e) {- throw new RuntimeException(e);- }- }- }-- @Override- public void execute() {- synchronized (concurrencyUsages) {- concurrencyUsagesCache.putAll(concurrencyUsages);- }+ return agentIds.stream().map(Object::toString).collect(toList());+ }, resourceName, totalConcurrency, requiredConcurrency, node -> {+ var agentId = Long.valueOf(node);+ var server = agentService.getAgentServer(agentId);+ if (server == null)+ throw new ExplicitException("Cannot find server managing allocated agent, please retry later");+ return clusterService.runOnServer(server, () -> {+ updateLastUsedDate(agentId);+ return task.call(agentId);+ });+ }));}- @Override- public ScheduleBuilder<?> getScheduleBuilder() {- return SimpleScheduleBuilder.repeatHourlyForever();- }-}diff --git a/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.java b/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.javanew file mode 100644index 0000000000..1e4fb225ea--- /dev/null+++ b/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.java@@ -0,0 +1,408 @@+package io.onedev.server.service.impl;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;++import java.util.ArrayList;+import java.util.Collections;+import java.util.List;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;++import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import com.hazelcast.core.HazelcastInstance;+import com.hazelcast.replicatedmap.ReplicatedMap;++import io.onedev.server.cluster.ClusterService;+import io.onedev.server.cluster.ClusterTask;+import io.onedev.server.event.system.SystemStarting;+import io.onedev.server.model.Agent;+import io.onedev.server.model.AgentLastUsedDate;+import io.onedev.server.persistence.TransactionService;+import io.onedev.server.search.entity.agent.AgentQuery;+import io.onedev.server.service.AgentService;++class DefaultResourceServiceTest {++ private final ExecutorService executor = Executors.newCachedThreadPool();++ private final ThreadLocal<String> executingServer = new ThreadLocal<>();++ private final AtomicReference<String> leader = new AtomicReference<>("leader");++ private final Map<Future<?>, AtomicReference<Thread>> taskThreads = new ConcurrentHashMap<>();++ private final Map<Thread, AtomicInteger> candidateQueries = new ConcurrentHashMap<>();++ private final ClusterService clusterService = mock(ClusterService.class);++ private final AgentService agentService = mock(AgentService.class);++ private DefaultResourceService resourceService;++ @BeforeEach+ void setup() throws Exception {+ when(clusterService.getLeaderServerAddress()).thenAnswer(it -> leader.get());+ when(clusterService.isLeaderServer()).thenAnswer(it -> leader.get().equals(executingServer.get()));+ when(clusterService.getLocalServerAddress()).thenReturn("leader");+ when(clusterService.getServerAddresses()).thenReturn(List.of("leader", "worker", "other-worker"));+ when(clusterService.getOnlineServers()).thenAnswer(it -> {+ assertEquals(leader.get(), executingServer.get(), "Selection must run on the leader");+ candidateQueries.computeIfAbsent(Thread.currentThread(), key -> new AtomicInteger()).incrementAndGet();+ // The online-server registry may still contain a departed Hazelcast member.+ return List.of("worker", "other-worker", "offline");+ });+ when(clusterService.submitToServer(anyString(), any())).thenAnswer(it -> {+ String server = it.getArgument(0);+ ClusterTask<?> task = it.getArgument(1);+ var thread = new AtomicReference<Thread>();+ var future = executor.submit(() -> {+ thread.set(Thread.currentThread());+ executingServer.set(server);+ try {+ return task.call();+ } finally {+ executingServer.remove();+ }+ });+ taskThreads.put(future, thread);+ return future;+ });+ when(clusterService.runOnServer(anyString(), any())).thenAnswer(it -> {+ String server = it.getArgument(0);+ ClusterTask<?> task = it.getArgument(1);+ Future<?> future = clusterService.submitToServer(server, task);+ try {+ return future.get();+ } catch (InterruptedException e) {+ // Match ClusterService's interruptible future wrapper.+ future.cancel(true);+ throw new RuntimeException(e);+ }+ });+ var hazelcast = mock(HazelcastInstance.class);+ @SuppressWarnings("unchecked")+ ReplicatedMap<String, Integer> cpuCounts = mock(ReplicatedMap.class);+ when(clusterService.getHazelcastInstance()).thenReturn(hazelcast);+ when(hazelcast.<String, Integer>getReplicatedMap("cpuCounts")).thenReturn(cpuCounts);+ when(cpuCounts.get(any())).thenReturn(2);+ resourceService = newResourceService();+ }++ private DefaultResourceService newResourceService() {+ var service = new DefaultResourceService(agentService, mock(TransactionService.class), clusterService);+ service.on(new SystemStarting());+ return service;+ }++ private void awaitWaiting(Future<?> future) throws Exception {+ awaitWaiting(future, 1);+ }++ private void awaitWaiting(Future<?> future, int minimumQueries) throws Exception {+ var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);+ while (System.nanoTime() < deadline) {+ if (future.isDone())+ fail("Expected a pending allocation, got: " + future.get());+ var thread = taskThreads.get(future).get();+ if (thread != null && thread.getState() == Thread.State.TIMED_WAITING+ && candidateQueries.get(thread).get() >= minimumQueries)+ return;+ Thread.sleep(5);+ }+ fail("Allocation did not start waiting");+ }++ @AfterEach+ void teardown() throws Exception {+ executor.shutdownNow();+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));+ }++ @Test+ void reservesCapacityAtomicallyForConcurrentSubmissions() throws Exception {+ var running = new AtomicInteger();+ var peak = new AtomicInteger();+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ List<Future<String>> futures = new ArrayList<>();+ for (int i = 0; i < 12; i++) {+ futures.add(resourceService.submitServerTask("worker", "executor", 3, 2, () -> {+ peak.accumulateAndGet(running.incrementAndGet(), Math::max);+ try {+ started.countDown();+ assertTrue(finish.await(5, TimeUnit.SECONDS));+ return executingServer.get();+ } finally {+ running.decrementAndGet();+ }+ }));+ }+ assertTrue(started.await(5, TimeUnit.SECONDS));+ assertThrows(TimeoutException.class, () -> futures.get(0).get(100, TimeUnit.MILLISECONDS));+ finish.countDown();+ for (var future : futures)+ assertEquals("worker", future.get(15, TimeUnit.SECONDS));+ assertEquals(1, peak.get());+ }++ @Test+ void usesCpuCapacityAndKeepsResourcesAndNodesIndependent() throws Exception {+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 0, 2, () -> {+ started.countDown();+ finish.await();+ return "first";+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ var queued = resourceService.submitServerTask("worker", "executor", 0, 1, () -> "queued");+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));+ assertEquals("other-worker", resourceService.submitServerTask(null, "executor", 0, 2,+ executingServer::get).get(5, TimeUnit.SECONDS));+ assertEquals("independent", resourceService.submitServerTask("worker", "another-executor", 0, 2,+ () -> "independent").get(5, TimeUnit.SECONDS));+ finish.countDown();+ assertEquals("first", first.get(5, TimeUnit.SECONDS));+ assertEquals("queued", queued.get(5, TimeUnit.SECONDS));+ }++ @Test+ void grantsCapacityInArrivalOrderAcrossTimedWakeups() throws Exception {+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ started.countDown();+ finish.await();+ return "first";+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ List<Integer> order = Collections.synchronizedList(new ArrayList<>());+ List<Future<Integer>> queued = new ArrayList<>();+ for (int i = 0; i < 4; i++) {+ int index = i;+ var future = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ order.add(index);+ return index;+ });+ awaitWaiting(future);+ queued.add(future);+ }+ // Periodic discovery must not move the oldest request to the back of the queue.+ awaitWaiting(queued.get(0), 2);+ finish.countDown();+ first.get(5, TimeUnit.SECONDS);+ for (var future : queued)+ future.get(5, TimeUnit.SECONDS);+ assertEquals(List.of(0, 1, 2, 3), order);+ }++ @Test+ void olderLargeRequestPreventsSmallerRequestsFromTakingSharedCapacity() throws Exception {+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 3, 1, () -> {+ started.countDown();+ finish.await();+ return "first";+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ List<String> order = Collections.synchronizedList(new ArrayList<>());+ var older = resourceService.submitServerTask("worker", "executor", 3, 3, () -> {+ order.add("older");+ return "older";+ });+ awaitWaiting(older);+ var younger = resourceService.submitServerTask("worker", "executor", 3, 1, () -> {+ order.add("younger");+ return "younger";+ });+ awaitWaiting(younger);+ assertEquals("other-worker", resourceService.submitServerTask(null, "executor", 3, 1,+ executingServer::get).get(5, TimeUnit.SECONDS));+ assertEquals("unrelated", resourceService.submitServerTask("worker", "another-executor", 3, 1,+ () -> "unrelated").get(5, TimeUnit.SECONDS));+ finish.countDown();+ first.get(5, TimeUnit.SECONDS);+ older.get(5, TimeUnit.SECONDS);+ younger.get(5, TimeUnit.SECONDS);+ assertEquals(List.of("older", "younger"), order);+ }++ @Test+ void cancellingQueueHeadLetsSmallerRequestUseAlreadyAvailableCapacity() throws Exception {+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 2, 1, () -> {+ started.countDown();+ finish.await();+ return "first";+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ var older = resourceService.submitServerTask("worker", "executor", 2, 2,+ () -> fail("Cancelled request ran"));+ awaitWaiting(older);+ var younger = resourceService.submitServerTask("worker", "executor", 2, 1, () -> "younger");+ awaitWaiting(younger);+ assertTrue(older.cancel(true));+ assertEquals("younger", younger.get(5, TimeUnit.SECONDS));+ assertFalse(first.isDone());+ finish.countDown();+ first.get(5, TimeUnit.SECONDS);+ }++ @Test+ void unavailableAndUndersizedNodesDoNotBlockOtherRequests() throws Exception {+ var offline = resourceService.submitServerTask("offline", "executor", 1, 1,+ () -> fail("Offline node selected"));+ awaitWaiting(offline);+ var tooLarge = resourceService.submitServerTask("worker", "executor", 1, 2,+ () -> fail("Capacity exceeded"));+ awaitWaiting(tooLarge);+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,+ () -> "next").get(5, TimeUnit.SECONDS));+ offline.cancel(true);+ tooLarge.cancel(true);+ }++ @Test+ void releasesCapacityAfterTaskAndSubmissionFailures() throws Exception {+ var failed = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ throw new IllegalStateException("Task failed");+ });+ assertThrows(ExecutionException.class, () -> failed.get(5, TimeUnit.SECONDS));+ when(clusterService.runOnServer(eq("worker"), any())).thenThrow(new IllegalStateException("Node left"));+ var rejected = resourceService.submitServerTask("worker", "executor", 1, 1, () -> "rejected");+ assertThrows(ExecutionException.class, () -> rejected.get(5, TimeUnit.SECONDS));+ when(clusterService.runOnServer(eq("worker"), any())).thenReturn("recovered");+ assertEquals("recovered", resourceService.submitServerTask("worker", "executor", 1, 1,+ () -> "recovered").get(5, TimeUnit.SECONDS));+ }++ @Test+ void cancellationInterruptsRunningTaskAndReleasesCapacity() throws Exception {+ var started = new CountDownLatch(1);+ var interrupted = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ started.countDown();+ try {+ new CountDownLatch(1).await();+ return "unexpected";+ } finally {+ interrupted.countDown();+ }+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ assertTrue(first.cancel(true));+ assertTrue(interrupted.await(5, TimeUnit.SECONDS));+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,+ () -> "next").get(5, TimeUnit.SECONDS));+ }++ @Test+ void cancellationWhileWaitingDoesNotDispatchOrLeakCapacity() throws Exception {+ var started = new CountDownLatch(1);+ var finish = new CountDownLatch(1);+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ started.countDown();+ finish.await();+ return "first";+ });+ assertTrue(started.await(5, TimeUnit.SECONDS));+ var queued = resourceService.submitServerTask("worker", "executor", 1, 1, () -> fail("Cancelled task ran"));+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));+ assertTrue(queued.cancel(true));+ finish.countDown();+ first.get(5, TimeUnit.SECONDS);+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,+ () -> "next").get(5, TimeUnit.SECONDS));+ verify(clusterService, times(2)).runOnServer(eq("worker"), any());+ }++ @Test+ void formerLeaderStopsAllocatingWaitingTasks() throws Exception {+ doReturn(List.of()).when(clusterService).getOnlineServers();+ var queued = resourceService.submitServerTask("worker", "executor", 1, 1, () -> "unexpected");+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));+ leader.set("new-leader");+ var failure = assertThrows(ExecutionException.class, () -> queued.get(5, TimeUnit.SECONDS));+ assertEquals("Cluster leader changed, please retry later", failure.getCause().getMessage());+ verify(clusterService, never()).runOnServer(anyString(), any());+ }++ @Test+ void failoverStartsFreshAndOldCompletionsDoNotReleaseNewReservations() throws Exception {+ var oldStarted = new CountDownLatch(1);+ var oldFinish = new CountDownLatch(1);+ var oldTask = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {+ oldStarted.countDown();+ oldFinish.await();+ return "old";+ });+ assertTrue(oldStarted.await(5, TimeUnit.SECONDS));+ leader.set("new-leader");+ var newLeaderService = newResourceService();+ var newStarted = new CountDownLatch(1);+ var newFinish = new CountDownLatch(1);+ var newTask = newLeaderService.submitServerTask("worker", "executor", 1, 1, () -> {+ newStarted.countDown();+ newFinish.await();+ return "new";+ });+ assertTrue(newStarted.await(5, TimeUnit.SECONDS));+ oldFinish.countDown();+ assertEquals("old", oldTask.get(5, TimeUnit.SECONDS));+ var queued = newLeaderService.submitServerTask("worker", "executor", 1, 1, () -> "queued");+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));+ newFinish.countDown();+ assertEquals("new", newTask.get(5, TimeUnit.SECONDS));+ assertEquals("queued", queued.get(5, TimeUnit.SECONDS));+ }++ @Test+ void filtersAgentsAndRecoversFromMissingAgentServer() throws Exception {+ var eligible = agent(1L, true, false);+ var paused = agent(2L, true, true);+ var offline = agent(3L, false, false);+ var query = new AgentQuery();+ when(agentService.query(query, 0, Integer.MAX_VALUE)).thenAnswer(it -> {+ assertEquals(leader.get(), executingServer.get());+ return List.of(eligible, paused, offline);+ });+ when(agentService.get(1L)).thenReturn(eligible);+ when(agentService.load(1L)).thenReturn(eligible);+ var failed = resourceService.submitAgentTask(null, query, "executor", 1, 1, id -> id);+ assertThrows(ExecutionException.class, () -> failed.get(5, TimeUnit.SECONDS));+ when(agentService.getAgentServer(1L)).thenReturn("worker");+ assertEquals(1L, resourceService.submitAgentTask(null, query, "executor", 1, 1, id -> {+ assertEquals("worker", executingServer.get());+ return id;+ }).get(5, TimeUnit.SECONDS));+ assertNotNull(eligible.getLastUsedDate().getValue());+ assertEquals(1L, resourceService.submitAgentTask(1L, query, "executor", 1, 1,+ id -> id).get(5, TimeUnit.SECONDS));+ }++ private Agent agent(Long id, boolean online, boolean paused) {+ var agent = mock(Agent.class);+ when(agent.getId()).thenReturn(id);+ when(agent.isOnline()).thenReturn(online);+ when(agent.isPaused()).thenReturn(paused);+ when(agent.getLastUsedDate()).thenReturn(new AgentLastUsedDate());+ return agent;+ }++}