The Unified and Autonomous Development Platform
Waiting for agent resource to be released before stopping agent
Robin Shen29eba283560a
diff --git a/server-core/src/main/java/io/onedev/server/ServerSocket.java b/server-core/src/main/java/io/onedev/server/ServerSocket.javaindex a13c402e25..cb1c6c9c42 100644--- a/server-core/src/main/java/io/onedev/server/ServerSocket.java+++ b/server-core/src/main/java/io/onedev/server/ServerSocket.java@@ -19,6 +19,7 @@ import io.onedev.agent.AgentData;import io.onedev.agent.CallData;import io.onedev.agent.Message;import io.onedev.agent.MessageType;+import io.onedev.agent.WaitingForAgentResourceToBeReleased;import io.onedev.agent.WebsocketUtils;import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.StringUtils;@@ -27,6 +28,7 @@ import io.onedev.server.buildspec.job.JobContext;import io.onedev.server.buildspec.job.JobManager;import io.onedev.server.entitymanager.AgentManager;import io.onedev.server.exception.NotReadyException;+import io.onedev.server.job.resource.ResourceManager;import io.onedev.server.tasklog.JobLogManager;@WebSocket@@ -159,7 +161,12 @@ public class ServerSocket {private Serializable service(Serializable request) {try {- throw new ExplicitException("Unknown request: " + request.getClass());+ if (request instanceof WaitingForAgentResourceToBeReleased) {+ OneDev.getInstance(ResourceManager.class).waitingForAgentResourceToBeReleased(agentId);+ return null;+ } else {+ throw new ExplicitException("Unknown request: " + request.getClass());+ }} catch (Exception e) {logger.error("Error servicing websocket request", e);return e;diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.javaindex 995fe552a7..1df555f463 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java@@ -1055,7 +1055,12 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz@Overridepublic void run() {+ boolean waitingForJobFinishLogged = false;while (!jobExecutions.isEmpty() || thread != null) {+ if (thread == null && !waitingForJobFinishLogged) {+ logger.info("Waiting for running jobs to finish...");+ waitingForJobFinishLogged = true;+ }try {synchronized (this) {transactionManager.run(new Runnable() {diff --git a/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java b/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.javaindex 7ec132be93..d50a116dbe 100644--- a/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java+++ b/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java@@ -162,6 +162,7 @@ public class DefaultResourceManager implements ResourceManager {Set<Long> agentIds = agentManager.query(agentQuery, 0, Integer.MAX_VALUE).stream().map(it->it.getId()).collect(Collectors.toSet());Long agentId = 0L;+ ResourceHolder agentResourceHolder = null;synchronized(this) {logger.log("Waiting for resources...");String uuid = UUID.randomUUID().toString();@@ -171,13 +172,14 @@ public class DefaultResourceManager implements ResourceManager {if (serverResourceHolder.getSpareResources(serverResourceRequirements) != 0) {int maxSpareResources = 0;for (Long each: agentIds) {- ResourceHolder agentResourceHolder = agentResourceHolders.get(each);+ ResourceHolder eachAgentResourceHolder = agentResourceHolders.get(each);Boolean paused = agentPaused.get(each);- if (agentResourceHolder != null && paused != null && !paused) {- int spareResources = agentResourceHolder.getSpareResources(agentResourceRequirements);+ if (eachAgentResourceHolder != null && paused != null && !paused) {+ int spareResources = eachAgentResourceHolder.getSpareResources(agentResourceRequirements);if (spareResources > maxSpareResources) {agentId = each;maxSpareResources = spareResources;+ agentResourceHolder = eachAgentResourceHolder;}}}@@ -190,7 +192,7 @@ public class DefaultResourceManager implements ResourceManager {throw new RuntimeException(e);}}- agentResourceHolders.get(agentId).acquireResources(agentResourceRequirements);+ agentResourceHolder.acquireResources(agentResourceRequirements);serverResourceHolder.acquireResources(serverResourceRequirements);} finally {queryCaches.remove(uuid);@@ -207,9 +209,7 @@ public class DefaultResourceManager implements ResourceManager {} finally {synchronized (this) {serverResourceHolder.releaseResources(serverResourceRequirements);- ResourceHolder agentResourceHolder = agentResourceHolders.get(agentId);- if(agentResourceHolder != null)- agentResourceHolder.releaseResources(agentResourceRequirements);+ agentResourceHolder.releaseResources(agentResourceRequirements);notifyAll();}}@@ -227,4 +227,20 @@ public class DefaultResourceManager implements ResourceManager {}}++ @Override+ public void waitingForAgentResourceToBeReleased(Long agentId) {+ synchronized (this) {+ ResourceHolder agentResourceHolder = agentResourceHolders.remove(agentId);+ if (agentResourceHolder != null) {+ while (agentResourceHolder.hasUsedResources()) {+ try {+ wait();+ } catch (InterruptedException e) {+ }+ }+ }+ }+ }+}diff --git a/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java b/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.javaindex e8c2a19fde..96dd1d81fc 100644--- a/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java+++ b/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java@@ -84,4 +84,12 @@ public class ResourceHolder {totalResources.put(name, count);}+ public boolean hasUsedResources() {+ for (Integer usedCount: usedResources.values()) {+ if (usedCount != 0)+ return true;+ }+ return false;+ }+}diff --git a/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java b/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.javaindex e6af6510d2..5775951a2d 100644--- a/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java+++ b/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java@@ -12,4 +12,6 @@ public interface ResourceManager {void run(AgentAwareRunnable runnable, Map<String, Integer> serverResourceRequirements,AgentQuery agentQuery, Map<String, Integer> agentResourceRequirements, TaskLogger logger);+ void waitingForAgentResourceToBeReleased(Long agentId);+}diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.htmlindex 1f6c751851..e01d827c56 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html@@ -7,7 +7,7 @@<ol><li>Make sure <a href="https://openjdk.java.net" target="_blank">JDK 1.8 or higher</a> is installed</li><li>Make sure docker engine is installed and docker command line is available in system path</li>- <li>Make sure current user has permission to run docker containers+ <li>Make sure current user has permission to run docker containers</li><li>Make sure git version 2.11.1 or higher is installed and available in system path</li><li>Download <a wicket:id="agent" class="link-primary">agent package</a>. A new agent token will be included in the package</li><li>Extract the package into a folder</li>diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.javaindex aee2c7e8db..1453c0e008 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java@@ -91,6 +91,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {IModel<AgentToken> rowModel) {cellItem.add(new Label(componentId, rowModel.getObject().getValue()));}++ @Override+ public String getCssClass() {+ return "text-nowrap align-top font-size-sm text-monospace";+ }});@@ -106,6 +111,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {cellItem.add(new Label(componentId, "<i>No note</i>").setEscapeModelStrings(false));}+ @Override+ public String getCssClass() {+ return "align-top font-size-sm";+ }+});columns.add(new AbstractColumn<AgentToken, Void>(Model.of("")) {@@ -139,6 +149,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {cellItem.add(fragment);}+ @Override+ public String getCssClass() {+ return "text-nowrap align-top";+ }+});SortableDataProvider<AgentToken, Void> dataProvider = new LoadableDetachableDataProvider<AgentToken, Void>() {diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.cssindex 06096d8785..19490725f6 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css@@ -1,3 +1,3 @@.floating .token-list {width: 600px;-}\ No newline at end of file+}