Skip to content

The Unified and Autonomous Development Platform

Waiting for agent resource to be released before stopping agent

Robin Shen29eba283560a
diff --git a/server-core/src/main/java/io/onedev/server/ServerSocket.java b/server-core/src/main/java/io/onedev/server/ServerSocket.java
index a13c402e25..cb1c6c9c42 100644
--- a/server-core/src/main/java/io/onedev/server/ServerSocket.java
+++ b/server-core/src/main/java/io/onedev/server/ServerSocket.java
@@ -19,6 +19,7 @@ import io.onedev.agent.AgentData;
import io.onedev.agent.CallData;
import io.onedev.agent.Message;
import io.onedev.agent.MessageType;
+import io.onedev.agent.WaitingForAgentResourceToBeReleased;
import io.onedev.agent.WebsocketUtils;
import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.StringUtils;
@@ -27,6 +28,7 @@ import io.onedev.server.buildspec.job.JobContext;
import io.onedev.server.buildspec.job.JobManager;
import io.onedev.server.entitymanager.AgentManager;
import io.onedev.server.exception.NotReadyException;
+import io.onedev.server.job.resource.ResourceManager;
import io.onedev.server.tasklog.JobLogManager;
@WebSocket
@@ -159,7 +161,12 @@ public class ServerSocket {
private Serializable service(Serializable request) {
try {
- throw new ExplicitException("Unknown request: " + request.getClass());
+ if (request instanceof WaitingForAgentResourceToBeReleased) {
+ OneDev.getInstance(ResourceManager.class).waitingForAgentResourceToBeReleased(agentId);
+ return null;
+ } else {
+ throw new ExplicitException("Unknown request: " + request.getClass());
+ }
} catch (Exception e) {
logger.error("Error servicing websocket request", e);
return e;
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
index 995fe552a7..1df555f463 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
@@ -1055,7 +1055,12 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
@Override
public void run() {
+ boolean waitingForJobFinishLogged = false;
while (!jobExecutions.isEmpty() || thread != null) {
+ if (thread == null && !waitingForJobFinishLogged) {
+ logger.info("Waiting for running jobs to finish...");
+ waitingForJobFinishLogged = true;
+ }
try {
synchronized (this) {
transactionManager.run(new Runnable() {
diff --git a/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java b/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java
index 7ec132be93..d50a116dbe 100644
--- a/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/resource/DefaultResourceManager.java
@@ -162,6 +162,7 @@ public class DefaultResourceManager implements ResourceManager {
Set<Long> agentIds = agentManager.query(agentQuery, 0, Integer.MAX_VALUE)
.stream().map(it->it.getId()).collect(Collectors.toSet());
Long agentId = 0L;
+ ResourceHolder agentResourceHolder = null;
synchronized(this) {
logger.log("Waiting for resources...");
String uuid = UUID.randomUUID().toString();
@@ -171,13 +172,14 @@ public class DefaultResourceManager implements ResourceManager {
if (serverResourceHolder.getSpareResources(serverResourceRequirements) != 0) {
int maxSpareResources = 0;
for (Long each: agentIds) {
- ResourceHolder agentResourceHolder = agentResourceHolders.get(each);
+ ResourceHolder eachAgentResourceHolder = agentResourceHolders.get(each);
Boolean paused = agentPaused.get(each);
- if (agentResourceHolder != null && paused != null && !paused) {
- int spareResources = agentResourceHolder.getSpareResources(agentResourceRequirements);
+ if (eachAgentResourceHolder != null && paused != null && !paused) {
+ int spareResources = eachAgentResourceHolder.getSpareResources(agentResourceRequirements);
if (spareResources > maxSpareResources) {
agentId = each;
maxSpareResources = spareResources;
+ agentResourceHolder = eachAgentResourceHolder;
}
}
}
@@ -190,7 +192,7 @@ public class DefaultResourceManager implements ResourceManager {
throw new RuntimeException(e);
}
}
- agentResourceHolders.get(agentId).acquireResources(agentResourceRequirements);
+ agentResourceHolder.acquireResources(agentResourceRequirements);
serverResourceHolder.acquireResources(serverResourceRequirements);
} finally {
queryCaches.remove(uuid);
@@ -207,9 +209,7 @@ public class DefaultResourceManager implements ResourceManager {
} finally {
synchronized (this) {
serverResourceHolder.releaseResources(serverResourceRequirements);
- ResourceHolder agentResourceHolder = agentResourceHolders.get(agentId);
- if(agentResourceHolder != null)
- agentResourceHolder.releaseResources(agentResourceRequirements);
+ agentResourceHolder.releaseResources(agentResourceRequirements);
notifyAll();
}
}
@@ -227,4 +227,20 @@ public class DefaultResourceManager implements ResourceManager {
}
}
+
+ @Override
+ public void waitingForAgentResourceToBeReleased(Long agentId) {
+ synchronized (this) {
+ ResourceHolder agentResourceHolder = agentResourceHolders.remove(agentId);
+ if (agentResourceHolder != null) {
+ while (agentResourceHolder.hasUsedResources()) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+ }
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java b/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java
index e8c2a19fde..96dd1d81fc 100644
--- a/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java
+++ b/server-core/src/main/java/io/onedev/server/job/resource/ResourceHolder.java
@@ -84,4 +84,12 @@ public class ResourceHolder {
totalResources.put(name, count);
}
+ public boolean hasUsedResources() {
+ for (Integer usedCount: usedResources.values()) {
+ if (usedCount != 0)
+ return true;
+ }
+ return false;
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java b/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java
index e6af6510d2..5775951a2d 100644
--- a/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/resource/ResourceManager.java
@@ -12,4 +12,6 @@ public interface ResourceManager {
void run(AgentAwareRunnable runnable, Map<String, Integer> serverResourceRequirements,
AgentQuery agentQuery, Map<String, Integer> agentResourceRequirements, TaskLogger logger);
+ void waitingForAgentResourceToBeReleased(Long agentId);
+
}
diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html
index 1f6c751851..e01d827c56 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html
+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentDownloadPanel.html
@@ -7,7 +7,7 @@
<ol>
<li>Make sure <a href="https://openjdk.java.net" target="_blank">JDK 1.8 or higher</a> is installed</li>
<li>Make sure docker engine is installed and docker command line is available in system path</li>
- <li>Make sure current user has permission to run docker containers
+ <li>Make sure current user has permission to run docker containers</li>
<li>Make sure git version 2.11.1 or higher is installed and available in system path</li>
<li>Download <a wicket:id="agent" class="link-primary">agent package</a>. A new agent token will be included in the package</li>
<li>Extract the package into a folder</li>
diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java
index aee2c7e8db..1453c0e008 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.java
@@ -91,6 +91,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {
IModel<AgentToken> rowModel) {
cellItem.add(new Label(componentId, rowModel.getObject().getValue()));
}
+
+ @Override
+ public String getCssClass() {
+ return "text-nowrap align-top font-size-sm text-monospace";
+ }
});
@@ -106,6 +111,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {
cellItem.add(new Label(componentId, "<i>No note</i>").setEscapeModelStrings(false));
}
+ @Override
+ public String getCssClass() {
+ return "align-top font-size-sm";
+ }
+
});
columns.add(new AbstractColumn<AgentToken, Void>(Model.of("")) {
@@ -139,6 +149,11 @@ public class TokenListPanel extends GenericPanel<List<AgentToken>> {
cellItem.add(fragment);
}
+ @Override
+ public String getCssClass() {
+ return "text-nowrap align-top";
+ }
+
});
SortableDataProvider<AgentToken, Void> dataProvider = new LoadableDetachableDataProvider<AgentToken, Void>() {
diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css
index 06096d8785..19490725f6 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css
+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/agent.css
@@ -1,3 +1,3 @@
.floating .token-list {
width: 600px;
-}
\ No newline at end of file
+}