The Unified and Autonomous Development Platform
Agent management improvements
Robin Shen5e1679fa39d7
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultAgentManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultAgentManager.javaindex bd811d8619..104ee0605a 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultAgentManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultAgentManager.java@@ -153,6 +153,7 @@ public class DefaultAgentManager extends BaseEntityManager<Agent> implements Ageagent.setName(data.getName());agent.setCpu(data.getCpu());agent.setMemory(data.getMemory());+ agent.setTemporal(data.isTemporal());agent.setIpAddress(session.getRemoteAddress().getAddress().getHostAddress());save(agent);@@ -173,6 +174,7 @@ public class DefaultAgentManager extends BaseEntityManager<Agent> implements Ageagent.setIpAddress(session.getRemoteAddress().getAddress().getHostAddress());agent.setCpu(data.getCpu());agent.setMemory(data.getMemory());+ agent.setTemporal(data.isTemporal());save(agent);attributeManager.syncAttributes(agent, data.getAttributes());}@@ -195,7 +197,12 @@ public class DefaultAgentManager extends BaseEntityManager<Agent> implements Age@Overridepublic void agentDisconnected(Long agentId) {agentSessions.remove(agentId);- listenerRegistry.post(new AgentDisconnected(load(agentId)));+ Agent agent = load(agentId);+ if (agent.isTemporal()) {+ removeReferences(agent);+ dao.remove(agent);+ }+ listenerRegistry.post(new AgentDisconnected(agent));}@Override@@ -213,13 +220,16 @@ public class DefaultAgentManager extends BaseEntityManager<Agent> implements Age});}- @Transactional- @Override- public void delete(Agent agent) {+ private void removeReferences(Agent agent) {Query<?> query = getSession().createQuery("update Build set agent=null where agent=:agent");query.setParameter("agent", agent);query.executeUpdate();-+ }++ @Transactional+ @Override+ public void delete(Agent agent) {+ removeReferences(agent);dao.remove(agent);Session prevSession = agentSessions.remove(agent.getId());@@ -321,10 +331,7 @@ public class DefaultAgentManager extends BaseEntityManager<Agent> implements Age@Overridepublic void delete(Collection<Agent> agents) {for (Agent agent: agents) {- Query<?> query = getSession().createQuery("update Build set agent=null where agent=:agent");- query.setParameter("agent", agent);- query.executeUpdate();-+ removeReferences(agent);dao.remove(agent);Session prevSession = agentSessions.remove(agent.getId());diff --git a/server-core/src/main/java/io/onedev/server/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/migration/DataMigrator.javaindex fa28a127e9..1f85a2a606 100644--- a/server-core/src/main/java/io/onedev/server/migration/DataMigrator.java+++ b/server-core/src/main/java/io/onedev/server/migration/DataMigrator.java@@ -3710,6 +3710,7 @@ public class DataMigrator {commentCount = 0;element.element("commentCount").setText(String.valueOf(commentCount));}+ dom.writeToFile(file, false);} else if (file.getName().startsWith("PullRequests.xml")) {VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);for (Element element: dom.getRootElement().elements()) {@@ -3718,6 +3719,7 @@ public class DataMigrator {commentCount = 0;element.element("commentCount").setText(String.valueOf(commentCount));}+ dom.writeToFile(file, false);}}}@@ -3725,4 +3727,59 @@ public class DataMigrator {private void migrate79(File dataDir, Stack<Integer> versions) {}+ private void migrate80(File dataDir, Stack<Integer> versions) {+ Map<String, Integer> issueCommentCounts = new HashMap<>();+ Map<String, Integer> pullRequestCommentCounts = new HashMap<>();++ for (File file: dataDir.listFiles()) {+ if (file.getName().startsWith("IssueComments.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element: dom.getRootElement().elements()) {+ String issueId = element.elementTextTrim("issue");+ Integer commentCount = issueCommentCounts.get(issueId);+ if (commentCount == null)+ commentCount = 0;+ commentCount++;+ issueCommentCounts.put(issueId, commentCount);+ }+ } else if (file.getName().startsWith("PullRequestComments.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element: dom.getRootElement().elements()) {+ String pullRequestId = element.elementTextTrim("request");+ Integer commentCount = pullRequestCommentCounts.get(pullRequestId);+ if (commentCount == null)+ commentCount = 0;+ commentCount++;+ pullRequestCommentCounts.put(pullRequestId, commentCount);+ }+ }+ }+ for (File file: dataDir.listFiles()) {+ if (file.getName().startsWith("Issues.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element: dom.getRootElement().elements()) {+ Integer commentCount = issueCommentCounts.get(element.elementTextTrim("id"));+ if (commentCount == null)+ commentCount = 0;+ element.element("commentCount").setText(String.valueOf(commentCount));+ }+ dom.writeToFile(file, false);+ } else if (file.getName().startsWith("PullRequests.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element: dom.getRootElement().elements()) {+ Integer commentCount = pullRequestCommentCounts.get(element.elementTextTrim("id"));+ if (commentCount == null)+ commentCount = 0;+ element.element("commentCount").setText(String.valueOf(commentCount));+ }+ dom.writeToFile(file, false);+ } else if (file.getName().startsWith("Agents.xml")) {+ VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);+ for (Element element: dom.getRootElement().elements())+ element.addElement("temporal").setText(("false"));+ dom.writeToFile(file, false);+ }+ }+ }+}diff --git a/server-core/src/main/java/io/onedev/server/model/Agent.java b/server-core/src/main/java/io/onedev/server/model/Agent.javaindex bc73404b71..83a5ebff3e 100644--- a/server-core/src/main/java/io/onedev/server/model/Agent.java+++ b/server-core/src/main/java/io/onedev/server/model/Agent.java@@ -133,6 +133,8 @@ public class Agent extends AbstractEntity {private int memory;+ private boolean temporal;+private boolean paused;private Date lastUsedDate;@@ -211,6 +213,14 @@ public class Agent extends AbstractEntity {this.memory = memory;}+ public boolean isTemporal() {+ return temporal;+ }++ public void setTemporal(boolean temporal) {+ this.temporal = temporal;+ }+public Collection<AgentAttribute> getAttributes() {return attributes;}@@ -249,7 +259,7 @@ public class Agent extends AbstractEntity {public AgentData getAgentData() {return new AgentData(getToken().getValue(), new OsInfo(osName, osVersion, osArch),- name, cpu, memory, getAttributeMap());+ name, cpu, memory, temporal, getAttributeMap());}}diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.html b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.htmlindex c0370fb5df..8eef057ea6 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.html@@ -41,6 +41,9 @@<wicket:fragment wicket:id="memoryHeaderFrag"><span class="mr-1">Memory</span> <a title="Physical memory in mega bytes"><wicket:svg href="question-o" class="icon icon-sm"/></a></wicket:fragment>+ <wicket:fragment wicket:id="temporalHeaderFrag">+ <span class="mr-1">Temporal</span> <a title="Temporal agent will be removed as soon as it goes offline"><wicket:svg href="question-o" class="icon icon-sm"/></a>+ </wicket:fragment><wicket:fragment wicket:id="agentLinkFrag"><a wicket:id="link" class="d-flex align-items-center"><svg wicket:id="icon" class="icon icon-lg mr-1"></svg> <span wicket:id="label"></span></a></wicket:fragment>diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.java b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.javaindex 1762c7b9c6..0eb5354d5a 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentListPanel.java@@ -431,7 +431,9 @@ class AgentListPanel extends Panel {@Overrideprotected String getConfirmMessage() {- return "Type <code>yes</code> below to remove selected offline agents";+ return "Removed selected offline agents. Please note that tokens used by these agents will "+ + "NOT be removed, and they will reappear here if go online. "+ + "Type <code>yes</code> below to confirm";}@Override@@ -496,8 +498,8 @@ class AgentListPanel extends Panel {@Overrideprotected String getConfirmMessage() {- return "Unauthorize selected agents. Other agents sharing tokens with these agents will also be unauthorized. "- + "Type <code>yes</code> below to confirm";+ return "Removing selected agents as well as tokens used by them. Please note that other agents sharing "+ + "tokens with these agents will also be removed. Type <code>yes</code> below to confirm";}@Override@@ -758,7 +760,9 @@ class AgentListPanel extends Panel {@Overrideprotected String getConfirmMessage() {- return "Type <code>yes</code> below to remove all queried offline agents";+ return "Remove all queried offline agents. Please note that this will NOT remove "+ + "tokens used by these agents, and they will reappear here if go online. "+ + "Type <code>yes</code> below to confirm";}@Override@@ -822,8 +826,8 @@ class AgentListPanel extends Panel {@Overrideprotected String getConfirmMessage() {- return "Unauthorize all queried agents. Other agents sharing tokens with these agents will also be unauthorized. "- + "Type <code>yes</code> below to confirm";+ return "Removing all queried agents as well as tokens used by them. Please note that other agents sharing "+ + "tokens with these agents will also be removed. Type <code>yes</code> below to confirm";}@Override@@ -979,6 +983,11 @@ class AgentListPanel extends Panel {cellItem.add(new Label(componentId, agent.getIpAddress()));}+ @Override+ public String getCssClass() {+ return "d-none d-lg-table-cell";+ }+});columns.add(new AbstractColumn<Agent, Void>(Model.of("CPU")) {@@ -1021,6 +1030,20 @@ class AgentListPanel extends Panel {});+ columns.add(new AbstractColumn<Agent, Void>(Model.of("Temporal")) {++ @Override+ public void populateItem(Item<ICellPopulator<Agent>> cellItem, String componentId, IModel<Agent> rowModel) {+ cellItem.add(new Label(componentId, rowModel.getObject().isTemporal()));+ }++ @Override+ public Component getHeader(String componentId) {+ return new Fragment(componentId, "temporalHeaderFrag", AgentListPanel.this);+ }++ });+columns.add(new AbstractColumn<Agent, Void>(Model.of("Status")) {@Overridediff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.html b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.htmlindex 6a5b97db33..f9cfc71bc8 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.html+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.html@@ -7,11 +7,12 @@<table class="table mb-5"><thead><th>Name</th>- <th>IP Address</th>+ <th class="d-none d-lg-table-cell">IP Address</th><th class="d-none d-lg-table-cell">OS Version</th><th class="d-none d-lg-table-cell">OS Arch</th><th class="d-none d-lg-table-cell"><span class="mr-1">CPU</span> <a title="CPU capability in millis. This is normally (CPU cores)*1000"><wicket:svg href="question-o" class="icon icon-sm"/></a></th><th class="d-none d-lg-table-cell"><span class="mr-1">Memory</span> <a title="Physical memory in mega bytes"><wicket:svg href="question-o" class="icon icon-sm"/></a></th>+ <th><span class="mr-1">Temporal</span> <a title="Temporal agent will be removed as soon as it goes offline"><wicket:svg href="question-o" class="icon icon-sm"/></a></th><th>Status</th></thead><tbody>@@ -37,6 +38,9 @@<td class="d-none d-lg-table-cell"><span wicket:id="memory"></span></td>+ <td class="d-none d-lg-table-cell">+ <span wicket:id="temporal"></span>+ </td><td><span wicket:id="status"></span></td>diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.java b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.javaindex d4f7a45290..8e48010ebd 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.java+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/AgentOverviewPage.java@@ -90,6 +90,7 @@ public class AgentOverviewPage extends AgentDetailPage {add(new Label("osArch", getAgent().getOsArch()));add(new Label("cpu", getAgent().getCpu()));add(new Label("memory", getAgent().getMemory()));+ add(new Label("temporal", getAgent().isTemporal()));add(new AgentStatusBadge("status", agentModel));if (getAgent().isOnline()) {diff --git a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.html b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.htmlindex 6a9d03d6df..82aff2a2cc 100644--- a/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.html+++ b/server-core/src/main/java/io/onedev/server/web/page/admin/buildsetting/agent/TokenListPanel.html@@ -1,9 +1,8 @@<wicket:panel><div class="p-5 token-list">- <h6 class="text-center mb-4">Unused Agent Tokens</h6>+ <h6 class="text-center mb-4">Available Agent Tokens</h6><div class="alert alert-light alert-notice mb-5">- Agent tokens are used to authorize agents, and a single token can only be used by a single- agent. It should be configured via environment variable+ Agent tokens are used to authorize agents. It should be configured via environment variable<tt>agentToken</tt> if agent runs as docker container, or property <tt>agentToken</tt> infile <tt><agent dir>/conf/agent.properties</tt> if agent runs on bare metal/virtual machine.A token will be in-use and removed from this list if agent using it connects to server