Skip to content

The Unified and Autonomous Development Platform

feat: Use FIFO when CI/CD job acquires resources

Robin Shenb24ed9ed954d
diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java
index 7306494380..7acddb4f90 100644
--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java
+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultBuildService.java
@@ -42,6 +42,7 @@ import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -386,12 +387,12 @@ public class DefaultBuildService extends BaseEntityService<Build> implements Bui
@Override
public Map<Long, Long> queryUnfinished() {
Query<?> query = getSession().createQuery("select id, project.id from Build where "
- + "status=:waiting or status=:pending or status=:running", Object[].class);
+ + "status=:waiting or status=:pending or status=:running order by submitDate", Object[].class);
query.setParameter("waiting", Build.Status.WAITING);
query.setParameter("pending", Build.Status.PENDING);
query.setParameter("running", Build.Status.RUNNING);
- Map<Long, Long> result = new HashMap<>();
+ Map<Long, Long> result = new LinkedHashMap<>();
for (Object[] fields: (List<Object[]>)query.list())
result.put((Long)fields[0], (Long)fields[1]);
return result;
diff --git a/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java b/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java
index f08b46f613..0ce4fa7610 100644
--- a/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java
+++ b/server-core/src/main/java/io/onedev/server/service/impl/DefaultResourceService.java
@@ -16,13 +16,13 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
+import java.util.function.Function;
+import java.util.function.Supplier;
import org.jspecify.annotations.Nullable;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
-import org.quartz.ScheduleBuilder;
-import org.quartz.SimpleScheduleBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,21 +38,16 @@ import io.onedev.server.service.support.AgentCallable;
import io.onedev.server.event.Listen;
import io.onedev.server.event.agent.AgentConnected;
import io.onedev.server.event.agent.AgentDisconnected;
-import io.onedev.server.event.cluster.NodeConnected;
-import io.onedev.server.event.system.SystemStarted;
import io.onedev.server.event.system.SystemStarting;
import io.onedev.server.event.system.SystemStopped;
-import io.onedev.server.event.system.SystemStopping;
import io.onedev.server.model.AbstractEntity;
import io.onedev.server.persistence.TransactionService;
import io.onedev.server.persistence.annotation.Transactional;
import io.onedev.server.search.entity.agent.AgentQuery;
-import io.onedev.server.taskschedule.SchedulableTask;
-import io.onedev.server.taskschedule.TaskScheduler;
import oshi.SystemInfo;
@Singleton
-public class DefaultResourceService implements ResourceService, Serializable, SchedulableTask {
+public class DefaultResourceService implements ResourceService, Serializable {
private static final Logger logger = LoggerFactory.getLogger(DefaultResourceService.class);
@@ -62,23 +57,36 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
private final TransactionService transactionService;
- private final TaskScheduler taskScheduler;
-
private volatile Map<String, Integer> cpuCounts;
+ /*
+ * Only the leader allocates resources. These counters intentionally are not replicated:
+ * after failover, existing tasks may temporarily exceed the limit until they finish.
+ * Each coordinating task releases its reservation on the same server that acquired it.
+ */
private final Map<String, Integer> concurrencyUsages = new HashMap<>();
-
- private volatile Map<String, Integer> concurrencyUsagesCache;
-
- private volatile String taskId;
+
+ // Guarded by concurrencyUsages, in order of arrival at the leader.
+ private final List<AllocationRequest> allocationRequests = new ArrayList<>();
+
+ private static class AllocationRequest {
+
+ private final String resourceName;
+
+ private Collection<String> nodes = Set.of();
+
+ private AllocationRequest(String resourceName) {
+ this.resourceName = resourceName;
+ }
+
+ }
@Inject
public DefaultResourceService(AgentService agentService, TransactionService transactionService,
- ClusterService clusterService, TaskScheduler taskScheduler) {
+ ClusterService clusterService) {
this.agentService = agentService;
this.transactionService = transactionService;
this.clusterService = clusterService;
- this.taskScheduler = taskScheduler;
}
public Object writeReplace() throws ObjectStreamException {
@@ -99,8 +107,6 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
logger.debug("Error calling oshi", e);
cpuCounts.put(localServer, 4);
}
- concurrencyUsagesCache = hazelcastInstance.getReplicatedMap("concurrencyUsagesCache");
- removeNodeFromConcurrencyUsagesCache(localServer);
}
@Listen
@@ -109,47 +115,6 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
cpuCounts.remove(clusterService.getLocalServerAddress());
}
- @Listen
- public void on(SystemStarted event) {
- taskId = taskScheduler.schedule(this);
- }
-
- @Listen
- public void on(SystemStopping event) {
- if (taskId != null)
- taskScheduler.unschedule(taskId);
- }
-
- @Listen
- public void on(NodeConnected event) {
- if (clusterService.isLeaderServer() && event.isRecovered()) {
- clusterService.submitToServer(event.getServer(), () -> {
- try {
- while (true) {
- synchronized (concurrencyUsages) {
- if (concurrencyUsagesCache != null) {
- concurrencyUsagesCache.putAll(concurrencyUsages);
- break;
- }
- }
- Thread.sleep(100);
- }
- } catch (Throwable e) {
- logger.error("Error syncing concurrency usages cache", e);
- }
- return null;
- });
- }
- }
-
- private void removeNodeFromConcurrencyUsagesCache(String node) {
- var keysToRemove = concurrencyUsagesCache.keySet().stream()
- .filter(it -> it.startsWith(node + ":"))
- .collect(toList());
- for (var key: keysToRemove)
- concurrencyUsagesCache.remove(key);
- }
-
@Transactional
@Listen
public void on(AgentConnected event) {
@@ -157,13 +122,6 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
var agentCpuCount = event.getAgent().getCpuCount();
transactionService.runAfterCommit(() -> {
cpuCounts.put(String.valueOf(agentId), agentCpuCount);
- removeNodeFromConcurrencyUsagesCache(agentId.toString());
- synchronized (concurrencyUsages) {
- for (var entry : concurrencyUsages.entrySet()) {
- if (entry.getKey().startsWith(agentId + ":"))
- concurrencyUsagesCache.put(entry.getKey(), entry.getValue());
- }
- }
});
}
@@ -200,20 +158,62 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
}
}
+ private String acquireNode(Supplier<Collection<String>> candidateNodes, String resourceName,
+ int totalConcurrency, int requiredConcurrency) throws InterruptedException {
+ var request = new AllocationRequest(resourceName);
+ synchronized (concurrencyUsages) {
+ allocationRequests.add(request);
+ }
+ try {
+ while (true) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+ if (!clusterService.isLeaderServer())
+ throw new ExplicitException("Cluster leader changed, please retry later");
+ // Agent queries may access the database, so resolve candidates outside the monitor.
+ var nodes = new HashSet<>(candidateNodes.get());
+ nodes.removeIf(node -> getEffectiveTotalConcurrency(node, totalConcurrency) < requiredConcurrency);
+ synchronized (concurrencyUsages) {
+ if (Thread.interrupted())
+ throw new InterruptedException();
+ request.nodes = nodes;
+ var availableNodes = new HashSet<>(nodes);
+ // Leave shared nodes to older requests, even if a smaller request could fit now.
+ // Unrelated nodes/resources and requests with no eligible nodes do not block us.
+ for (var earlier : allocationRequests) {
+ if (earlier == request)
+ break;
+ if (earlier.resourceName.equals(resourceName))
+ availableNodes.removeAll(earlier.nodes);
+ }
+ var node = allocateNode(availableNodes, resourceName, totalConcurrency, requiredConcurrency);
+ if (node != null) {
+ concurrencyUsages.merge(node + ":" + resourceName, requiredConcurrency, Integer::sum);
+ return node;
+ }
+ // Poll capacity, eligibility, and leadership once per second to avoid wake-up storms.
+ concurrencyUsages.wait(1000);
+ }
+ }
+ } finally {
+ synchronized (concurrencyUsages) {
+ allocationRequests.remove(request);
+ }
+ }
+ }
+
+ // Called with concurrencyUsages locked, so selection and reservation are atomic.
@Nullable
private String allocateNode(Collection<String> nodes, String resourceName,
- int totalConcurrency, int requiredConcurrency) {
+ int totalConcurrency, int requiredConcurrency) {
String allocatedNode = null;
var maxScore = 0;
var nodeList = new ArrayList<>(nodes);
Collections.shuffle(nodeList);
- for (var node: nodeList) {
+ for (var node : nodeList) {
var effectiveTotalConcurrency = getEffectiveTotalConcurrency(node, totalConcurrency);
- var usedConcurrency = concurrencyUsagesCache.get(node + ":" + resourceName);
- if (usedConcurrency == null)
- usedConcurrency = 0;
+ var usedConcurrency = concurrencyUsages.getOrDefault(node + ":" + resourceName, 0);
var score = getAllocationScore(effectiveTotalConcurrency, usedConcurrency, requiredConcurrency);
-
if (score > maxScore) {
allocatedNode = node;
maxScore = score;
@@ -222,69 +222,43 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
return allocatedNode;
}
- private void acquireConcurrency(String concurrencyKey, int totalConcurrency, int acquireConcurrency) {
- while (true) {
- synchronized (concurrencyUsages) {
- var usedCocurrency = concurrencyUsages.get(concurrencyKey);
- if (usedCocurrency == null)
- usedCocurrency = 0;
- usedCocurrency += acquireConcurrency;
- if (usedCocurrency <= totalConcurrency) {
- concurrencyUsages.put(concurrencyKey, usedCocurrency);
- concurrencyUsagesCache.put(concurrencyKey, usedCocurrency);
- break;
- }
- }
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
+ private void releaseConcurrency(String concurrencyKey, int releaseConcurrency) {
+ synchronized (concurrencyUsages) {
+ var usedConcurrency = concurrencyUsages.get(concurrencyKey) - releaseConcurrency;
+ if (usedConcurrency != 0)
+ concurrencyUsages.put(concurrencyKey, usedConcurrency);
+ else
+ concurrencyUsages.remove(concurrencyKey);
}
}
- private void releaseConcurrency(String concurrencyKey, int releaseConcurrency) {
- synchronized (concurrencyUsages) {
- var usedResources = concurrencyUsages.get(concurrencyKey);
- usedResources -= releaseConcurrency;
- concurrencyUsages.put(concurrencyKey, usedResources);
- concurrencyUsagesCache.put(concurrencyKey, usedResources);
+ private <T> T runTask(Supplier<Collection<String>> candidateNodes, String resourceName,
+ int totalConcurrency, int requiredConcurrency, Function<String, T> task) throws InterruptedException {
+ var node = acquireNode(candidateNodes, resourceName, totalConcurrency, requiredConcurrency);
+ try {
+ return task.apply(node);
+ } finally {
+ releaseConcurrency(node + ":" + resourceName, requiredConcurrency);
}
}
@Override
public <T> Future<T> submitServerTask(@Nullable String pinnedServerAddress, String resourceName,
- int totalConcurrency, int requiredConcurrency, ClusterTask<T> runnable) {
- while (true) {
- var candidateServers = new ArrayList<>(clusterService.getOnlineServers());
- if (pinnedServerAddress != null)
+ int totalConcurrency, int requiredConcurrency, ClusterTask<T> task) {
+ return clusterService.submitToServer(clusterService.getLeaderServerAddress(), () -> runTask(() -> {
+ var candidateServers = new ArrayList<>(clusterService.getOnlineServers());
+ candidateServers.retainAll(new HashSet<>(clusterService.getServerAddresses()));
+ if (pinnedServerAddress != null)
candidateServers.retainAll(List.of(pinnedServerAddress));
- var allocatedServer = allocateNode(candidateServers, resourceName, totalConcurrency, requiredConcurrency);
- if (allocatedServer != null) {
- return clusterService.submitToServer(allocatedServer, () -> {
- int effectiveTotalConcurrency = getEffectiveTotalConcurrency(allocatedServer, totalConcurrency);
- var concurrencyKey = allocatedServer + ":" + resourceName;
- acquireConcurrency(concurrencyKey, effectiveTotalConcurrency, requiredConcurrency);
- try {
- return runnable.call();
- } finally {
- releaseConcurrency(concurrencyKey, requiredConcurrency);
- }
- });
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
+ return candidateServers;
+ }, resourceName, totalConcurrency, requiredConcurrency,
+ server -> clusterService.runOnServer(server, task)));
}
@Override
- public <T> Future<T> submitAgentTask(@Nullable Long pinnedAgentId, AgentQuery agentQuery,
- String resourceName, int totalConcurrency, int requiredConcurrency,
- AgentCallable<T> task) {
- while (true) {
+ public <T> Future<T> submitAgentTask(@Nullable Long pinnedAgentId, AgentQuery agentQuery,
+ String resourceName, int totalConcurrency, int requiredConcurrency, AgentCallable<T> task) {
+ return clusterService.submitToServer(clusterService.getLeaderServerAddress(), () -> runTask(() -> {
Set<Long> agentIds;
if (pinnedAgentId != null) {
agentIds = new HashSet<>();
@@ -297,45 +271,17 @@ public class DefaultResourceService implements ResourceService, Serializable, Sc
.map(AbstractEntity::getId)
.collect(toSet());
}
- var agentIdString = allocateNode(
- agentIds.stream().map(Object::toString).collect(toList()),
- resourceName, totalConcurrency, requiredConcurrency);
- var agentId = agentIdString != null? Long.valueOf(agentIdString): null;
- if (agentId != null) {
- var server = agentService.getAgentServer(agentId);
- if (server == null)
- throw new ExplicitException("Cannot find server managing allocated agent, please retry later");
-
- return clusterService.submitToServer(server, () -> {
- var effectiveTotalConcurrency = getEffectiveTotalConcurrency(agentIdString, totalConcurrency);
- var concurrencyKey = agentId + ":" + resourceName;
- acquireConcurrency(concurrencyKey, effectiveTotalConcurrency, requiredConcurrency);
- try {
- updateLastUsedDate(agentId);
- return task.call(agentId);
- } finally {
- releaseConcurrency(concurrencyKey, requiredConcurrency);
- }
- });
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override
- public void execute() {
- synchronized (concurrencyUsages) {
- concurrencyUsagesCache.putAll(concurrencyUsages);
- }
+ return agentIds.stream().map(Object::toString).collect(toList());
+ }, resourceName, totalConcurrency, requiredConcurrency, node -> {
+ var agentId = Long.valueOf(node);
+ var server = agentService.getAgentServer(agentId);
+ if (server == null)
+ throw new ExplicitException("Cannot find server managing allocated agent, please retry later");
+ return clusterService.runOnServer(server, () -> {
+ updateLastUsedDate(agentId);
+ return task.call(agentId);
+ });
+ }));
}
- @Override
- public ScheduleBuilder<?> getScheduleBuilder() {
- return SimpleScheduleBuilder.repeatHourlyForever();
- }
-
}
diff --git a/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.java b/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.java
new file mode 100644
index 0000000000..1e4fb225ea
--- /dev/null
+++ b/server-core/src/test/java/io/onedev/server/service/impl/DefaultResourceServiceTest.java
@@ -0,0 +1,408 @@
+package io.onedev.server.service.impl;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.replicatedmap.ReplicatedMap;
+
+import io.onedev.server.cluster.ClusterService;
+import io.onedev.server.cluster.ClusterTask;
+import io.onedev.server.event.system.SystemStarting;
+import io.onedev.server.model.Agent;
+import io.onedev.server.model.AgentLastUsedDate;
+import io.onedev.server.persistence.TransactionService;
+import io.onedev.server.search.entity.agent.AgentQuery;
+import io.onedev.server.service.AgentService;
+
+class DefaultResourceServiceTest {
+
+ private final ExecutorService executor = Executors.newCachedThreadPool();
+
+ private final ThreadLocal<String> executingServer = new ThreadLocal<>();
+
+ private final AtomicReference<String> leader = new AtomicReference<>("leader");
+
+ private final Map<Future<?>, AtomicReference<Thread>> taskThreads = new ConcurrentHashMap<>();
+
+ private final Map<Thread, AtomicInteger> candidateQueries = new ConcurrentHashMap<>();
+
+ private final ClusterService clusterService = mock(ClusterService.class);
+
+ private final AgentService agentService = mock(AgentService.class);
+
+ private DefaultResourceService resourceService;
+
+ @BeforeEach
+ void setup() throws Exception {
+ when(clusterService.getLeaderServerAddress()).thenAnswer(it -> leader.get());
+ when(clusterService.isLeaderServer()).thenAnswer(it -> leader.get().equals(executingServer.get()));
+ when(clusterService.getLocalServerAddress()).thenReturn("leader");
+ when(clusterService.getServerAddresses()).thenReturn(List.of("leader", "worker", "other-worker"));
+ when(clusterService.getOnlineServers()).thenAnswer(it -> {
+ assertEquals(leader.get(), executingServer.get(), "Selection must run on the leader");
+ candidateQueries.computeIfAbsent(Thread.currentThread(), key -> new AtomicInteger()).incrementAndGet();
+ // The online-server registry may still contain a departed Hazelcast member.
+ return List.of("worker", "other-worker", "offline");
+ });
+ when(clusterService.submitToServer(anyString(), any())).thenAnswer(it -> {
+ String server = it.getArgument(0);
+ ClusterTask<?> task = it.getArgument(1);
+ var thread = new AtomicReference<Thread>();
+ var future = executor.submit(() -> {
+ thread.set(Thread.currentThread());
+ executingServer.set(server);
+ try {
+ return task.call();
+ } finally {
+ executingServer.remove();
+ }
+ });
+ taskThreads.put(future, thread);
+ return future;
+ });
+ when(clusterService.runOnServer(anyString(), any())).thenAnswer(it -> {
+ String server = it.getArgument(0);
+ ClusterTask<?> task = it.getArgument(1);
+ Future<?> future = clusterService.submitToServer(server, task);
+ try {
+ return future.get();
+ } catch (InterruptedException e) {
+ // Match ClusterService's interruptible future wrapper.
+ future.cancel(true);
+ throw new RuntimeException(e);
+ }
+ });
+ var hazelcast = mock(HazelcastInstance.class);
+ @SuppressWarnings("unchecked")
+ ReplicatedMap<String, Integer> cpuCounts = mock(ReplicatedMap.class);
+ when(clusterService.getHazelcastInstance()).thenReturn(hazelcast);
+ when(hazelcast.<String, Integer>getReplicatedMap("cpuCounts")).thenReturn(cpuCounts);
+ when(cpuCounts.get(any())).thenReturn(2);
+ resourceService = newResourceService();
+ }
+
+ private DefaultResourceService newResourceService() {
+ var service = new DefaultResourceService(agentService, mock(TransactionService.class), clusterService);
+ service.on(new SystemStarting());
+ return service;
+ }
+
+ private void awaitWaiting(Future<?> future) throws Exception {
+ awaitWaiting(future, 1);
+ }
+
+ private void awaitWaiting(Future<?> future, int minimumQueries) throws Exception {
+ var deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (System.nanoTime() < deadline) {
+ if (future.isDone())
+ fail("Expected a pending allocation, got: " + future.get());
+ var thread = taskThreads.get(future).get();
+ if (thread != null && thread.getState() == Thread.State.TIMED_WAITING
+ && candidateQueries.get(thread).get() >= minimumQueries)
+ return;
+ Thread.sleep(5);
+ }
+ fail("Allocation did not start waiting");
+ }
+
+ @AfterEach
+ void teardown() throws Exception {
+ executor.shutdownNow();
+ assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void reservesCapacityAtomicallyForConcurrentSubmissions() throws Exception {
+ var running = new AtomicInteger();
+ var peak = new AtomicInteger();
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ List<Future<String>> futures = new ArrayList<>();
+ for (int i = 0; i < 12; i++) {
+ futures.add(resourceService.submitServerTask("worker", "executor", 3, 2, () -> {
+ peak.accumulateAndGet(running.incrementAndGet(), Math::max);
+ try {
+ started.countDown();
+ assertTrue(finish.await(5, TimeUnit.SECONDS));
+ return executingServer.get();
+ } finally {
+ running.decrementAndGet();
+ }
+ }));
+ }
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ assertThrows(TimeoutException.class, () -> futures.get(0).get(100, TimeUnit.MILLISECONDS));
+ finish.countDown();
+ for (var future : futures)
+ assertEquals("worker", future.get(15, TimeUnit.SECONDS));
+ assertEquals(1, peak.get());
+ }
+
+ @Test
+ void usesCpuCapacityAndKeepsResourcesAndNodesIndependent() throws Exception {
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 0, 2, () -> {
+ started.countDown();
+ finish.await();
+ return "first";
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ var queued = resourceService.submitServerTask("worker", "executor", 0, 1, () -> "queued");
+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));
+ assertEquals("other-worker", resourceService.submitServerTask(null, "executor", 0, 2,
+ executingServer::get).get(5, TimeUnit.SECONDS));
+ assertEquals("independent", resourceService.submitServerTask("worker", "another-executor", 0, 2,
+ () -> "independent").get(5, TimeUnit.SECONDS));
+ finish.countDown();
+ assertEquals("first", first.get(5, TimeUnit.SECONDS));
+ assertEquals("queued", queued.get(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void grantsCapacityInArrivalOrderAcrossTimedWakeups() throws Exception {
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ started.countDown();
+ finish.await();
+ return "first";
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ List<Integer> order = Collections.synchronizedList(new ArrayList<>());
+ List<Future<Integer>> queued = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ int index = i;
+ var future = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ order.add(index);
+ return index;
+ });
+ awaitWaiting(future);
+ queued.add(future);
+ }
+ // Periodic discovery must not move the oldest request to the back of the queue.
+ awaitWaiting(queued.get(0), 2);
+ finish.countDown();
+ first.get(5, TimeUnit.SECONDS);
+ for (var future : queued)
+ future.get(5, TimeUnit.SECONDS);
+ assertEquals(List.of(0, 1, 2, 3), order);
+ }
+
+ @Test
+ void olderLargeRequestPreventsSmallerRequestsFromTakingSharedCapacity() throws Exception {
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 3, 1, () -> {
+ started.countDown();
+ finish.await();
+ return "first";
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ List<String> order = Collections.synchronizedList(new ArrayList<>());
+ var older = resourceService.submitServerTask("worker", "executor", 3, 3, () -> {
+ order.add("older");
+ return "older";
+ });
+ awaitWaiting(older);
+ var younger = resourceService.submitServerTask("worker", "executor", 3, 1, () -> {
+ order.add("younger");
+ return "younger";
+ });
+ awaitWaiting(younger);
+ assertEquals("other-worker", resourceService.submitServerTask(null, "executor", 3, 1,
+ executingServer::get).get(5, TimeUnit.SECONDS));
+ assertEquals("unrelated", resourceService.submitServerTask("worker", "another-executor", 3, 1,
+ () -> "unrelated").get(5, TimeUnit.SECONDS));
+ finish.countDown();
+ first.get(5, TimeUnit.SECONDS);
+ older.get(5, TimeUnit.SECONDS);
+ younger.get(5, TimeUnit.SECONDS);
+ assertEquals(List.of("older", "younger"), order);
+ }
+
+ @Test
+ void cancellingQueueHeadLetsSmallerRequestUseAlreadyAvailableCapacity() throws Exception {
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 2, 1, () -> {
+ started.countDown();
+ finish.await();
+ return "first";
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ var older = resourceService.submitServerTask("worker", "executor", 2, 2,
+ () -> fail("Cancelled request ran"));
+ awaitWaiting(older);
+ var younger = resourceService.submitServerTask("worker", "executor", 2, 1, () -> "younger");
+ awaitWaiting(younger);
+ assertTrue(older.cancel(true));
+ assertEquals("younger", younger.get(5, TimeUnit.SECONDS));
+ assertFalse(first.isDone());
+ finish.countDown();
+ first.get(5, TimeUnit.SECONDS);
+ }
+
+ @Test
+ void unavailableAndUndersizedNodesDoNotBlockOtherRequests() throws Exception {
+ var offline = resourceService.submitServerTask("offline", "executor", 1, 1,
+ () -> fail("Offline node selected"));
+ awaitWaiting(offline);
+ var tooLarge = resourceService.submitServerTask("worker", "executor", 1, 2,
+ () -> fail("Capacity exceeded"));
+ awaitWaiting(tooLarge);
+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,
+ () -> "next").get(5, TimeUnit.SECONDS));
+ offline.cancel(true);
+ tooLarge.cancel(true);
+ }
+
+ @Test
+ void releasesCapacityAfterTaskAndSubmissionFailures() throws Exception {
+ var failed = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ throw new IllegalStateException("Task failed");
+ });
+ assertThrows(ExecutionException.class, () -> failed.get(5, TimeUnit.SECONDS));
+ when(clusterService.runOnServer(eq("worker"), any())).thenThrow(new IllegalStateException("Node left"));
+ var rejected = resourceService.submitServerTask("worker", "executor", 1, 1, () -> "rejected");
+ assertThrows(ExecutionException.class, () -> rejected.get(5, TimeUnit.SECONDS));
+ when(clusterService.runOnServer(eq("worker"), any())).thenReturn("recovered");
+ assertEquals("recovered", resourceService.submitServerTask("worker", "executor", 1, 1,
+ () -> "recovered").get(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void cancellationInterruptsRunningTaskAndReleasesCapacity() throws Exception {
+ var started = new CountDownLatch(1);
+ var interrupted = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ started.countDown();
+ try {
+ new CountDownLatch(1).await();
+ return "unexpected";
+ } finally {
+ interrupted.countDown();
+ }
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ assertTrue(first.cancel(true));
+ assertTrue(interrupted.await(5, TimeUnit.SECONDS));
+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,
+ () -> "next").get(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void cancellationWhileWaitingDoesNotDispatchOrLeakCapacity() throws Exception {
+ var started = new CountDownLatch(1);
+ var finish = new CountDownLatch(1);
+ var first = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ started.countDown();
+ finish.await();
+ return "first";
+ });
+ assertTrue(started.await(5, TimeUnit.SECONDS));
+ var queued = resourceService.submitServerTask("worker", "executor", 1, 1, () -> fail("Cancelled task ran"));
+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));
+ assertTrue(queued.cancel(true));
+ finish.countDown();
+ first.get(5, TimeUnit.SECONDS);
+ assertEquals("next", resourceService.submitServerTask("worker", "executor", 1, 1,
+ () -> "next").get(5, TimeUnit.SECONDS));
+ verify(clusterService, times(2)).runOnServer(eq("worker"), any());
+ }
+
+ @Test
+ void formerLeaderStopsAllocatingWaitingTasks() throws Exception {
+ doReturn(List.of()).when(clusterService).getOnlineServers();
+ var queued = resourceService.submitServerTask("worker", "executor", 1, 1, () -> "unexpected");
+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));
+ leader.set("new-leader");
+ var failure = assertThrows(ExecutionException.class, () -> queued.get(5, TimeUnit.SECONDS));
+ assertEquals("Cluster leader changed, please retry later", failure.getCause().getMessage());
+ verify(clusterService, never()).runOnServer(anyString(), any());
+ }
+
+ @Test
+ void failoverStartsFreshAndOldCompletionsDoNotReleaseNewReservations() throws Exception {
+ var oldStarted = new CountDownLatch(1);
+ var oldFinish = new CountDownLatch(1);
+ var oldTask = resourceService.submitServerTask("worker", "executor", 1, 1, () -> {
+ oldStarted.countDown();
+ oldFinish.await();
+ return "old";
+ });
+ assertTrue(oldStarted.await(5, TimeUnit.SECONDS));
+ leader.set("new-leader");
+ var newLeaderService = newResourceService();
+ var newStarted = new CountDownLatch(1);
+ var newFinish = new CountDownLatch(1);
+ var newTask = newLeaderService.submitServerTask("worker", "executor", 1, 1, () -> {
+ newStarted.countDown();
+ newFinish.await();
+ return "new";
+ });
+ assertTrue(newStarted.await(5, TimeUnit.SECONDS));
+ oldFinish.countDown();
+ assertEquals("old", oldTask.get(5, TimeUnit.SECONDS));
+ var queued = newLeaderService.submitServerTask("worker", "executor", 1, 1, () -> "queued");
+ assertThrows(TimeoutException.class, () -> queued.get(100, TimeUnit.MILLISECONDS));
+ newFinish.countDown();
+ assertEquals("new", newTask.get(5, TimeUnit.SECONDS));
+ assertEquals("queued", queued.get(5, TimeUnit.SECONDS));
+ }
+
+ @Test
+ void filtersAgentsAndRecoversFromMissingAgentServer() throws Exception {
+ var eligible = agent(1L, true, false);
+ var paused = agent(2L, true, true);
+ var offline = agent(3L, false, false);
+ var query = new AgentQuery();
+ when(agentService.query(query, 0, Integer.MAX_VALUE)).thenAnswer(it -> {
+ assertEquals(leader.get(), executingServer.get());
+ return List.of(eligible, paused, offline);
+ });
+ when(agentService.get(1L)).thenReturn(eligible);
+ when(agentService.load(1L)).thenReturn(eligible);
+ var failed = resourceService.submitAgentTask(null, query, "executor", 1, 1, id -> id);
+ assertThrows(ExecutionException.class, () -> failed.get(5, TimeUnit.SECONDS));
+ when(agentService.getAgentServer(1L)).thenReturn("worker");
+ assertEquals(1L, resourceService.submitAgentTask(null, query, "executor", 1, 1, id -> {
+ assertEquals("worker", executingServer.get());
+ return id;
+ }).get(5, TimeUnit.SECONDS));
+ assertNotNull(eligible.getLastUsedDate().getValue());
+ assertEquals(1L, resourceService.submitAgentTask(1L, query, "executor", 1, 1,
+ id -> id).get(5, TimeUnit.SECONDS));
+ }
+
+ private Agent agent(Long id, boolean online, boolean paused) {
+ var agent = mock(Agent.class);
+ when(agent.getId()).thenReturn(id);
+ when(agent.isOnline()).thenReturn(online);
+ when(agent.isPaused()).thenReturn(paused);
+ when(agent.getLastUsedDate()).thenReturn(new AgentLastUsedDate());
+ return agent;
+ }
+
+}