Skip to content

The Unified and Autonomous Development Platform

Add logic to clean up build records

Robin Shen70ddd31b975c
diff --git a/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java b/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java
index ab10b7e276..18e46e721c 100644
--- a/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java
+++ b/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java
@@ -22,4 +22,6 @@ public interface BuildInfoManager {
@Nullable
Collection<ObjectId> getPrevCommits(Project project, Long buildId);
+ void delete(Project project, Long buildId);
+
}
diff --git a/core/src/main/java/io/onedev/server/manager/BuildManager.java b/core/src/main/java/io/onedev/server/manager/BuildManager.java
index 618b0dcf9f..b374df758c 100644
--- a/core/src/main/java/io/onedev/server/manager/BuildManager.java
+++ b/core/src/main/java/io/onedev/server/manager/BuildManager.java
@@ -32,4 +32,7 @@ public interface BuildManager extends EntityManager<Build> {
List<Build> query(Project project, User user, EntityQuery<Build> buildQuery, int firstResult, int maxResults);
int count(Project project, User user, EntityCriteria<Build> buildCriteria);
+
+ void cleanupBuilds(Configuration configuration);
+
}
diff --git a/core/src/main/java/io/onedev/server/manager/CacheManager.java b/core/src/main/java/io/onedev/server/manager/CacheManager.java
index b718ff4bfa..db8967aa59 100644
--- a/core/src/main/java/io/onedev/server/manager/CacheManager.java
+++ b/core/src/main/java/io/onedev/server/manager/CacheManager.java
@@ -48,7 +48,9 @@ public interface CacheManager {
Collection<Long> getIssueNumbers(Long projectId);
- Collection<Long> getBuildIds(Long projectId);
+ Collection<Long> getBuildIdsByProject(Long projectId);
+
+ Collection<Long> getBuildIdsByConfiguration(Long configurationId);
Collection<Long> filterBuildIds(Long projectId, Collection<String> commitHashes);
diff --git a/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java b/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java
index 3d799aac5c..ffa6d55974 100644
--- a/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java
+++ b/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java
@@ -12,5 +12,5 @@ public interface ConfigurationManager extends EntityManager<Configuration> {
Configuration find(Project project, String name);
void save(Configuration configuration, @Nullable String oldName);
-
+
}
diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java
index 20edf68fee..53f2979946 100644
--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java
+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java
@@ -225,5 +225,20 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemen
protected int getEnvVersion() {
return INFO_VERSION;
}
+
+ @Override
+ public void delete(Project project, Long buildId) {
+ Environment env = getEnv(project.getId().toString());
+ Store store = getStore(env, PREV_COMMITS_STORE);
+
+ env.executeInTransaction(new TransactionalExecutable() {
+
+ @Override
+ public void execute(Transaction txn) {
+ store.delete(txn, new LongByteIterable(buildId));
+ }
+
+ });
+ }
}
diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java
index 61f725d937..0575066ae0 100644
--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java
+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java
@@ -19,12 +19,18 @@ import org.hibernate.Session;
import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import org.hibernate.query.Query;
+import org.quartz.CronScheduleBuilder;
+import org.quartz.ScheduleBuilder;
+import io.onedev.launcher.loader.Listen;
import io.onedev.launcher.loader.ListenerRegistry;
import io.onedev.server.OneDev;
import io.onedev.server.event.build.BuildFinished;
import io.onedev.server.event.build.BuildStarted;
+import io.onedev.server.event.lifecycle.SystemStarted;
+import io.onedev.server.event.lifecycle.SystemStopping;
import io.onedev.server.manager.BuildManager;
+import io.onedev.server.manager.CacheManager;
import io.onedev.server.manager.ConfigurationManager;
import io.onedev.server.model.Build;
import io.onedev.server.model.Configuration;
@@ -42,16 +48,27 @@ import io.onedev.server.search.entity.EntitySort.Direction;
import io.onedev.server.search.entity.QueryBuildContext;
import io.onedev.server.search.entity.build.BuildQuery;
import io.onedev.server.search.entity.build.BuildQueryBuildContext;
+import io.onedev.utils.schedule.SchedulableTask;
+import io.onedev.utils.schedule.TaskScheduler;
@Singleton
-public class DefaultBuildManager extends AbstractEntityManager<Build> implements BuildManager {
+public class DefaultBuildManager extends AbstractEntityManager<Build> implements BuildManager, SchedulableTask {
+ private final TaskScheduler taskScheduler;
+
private final ListenerRegistry listenerRegistry;
+ private final ConfigurationManager configurationManager;
+
+ private String taskId;
+
@Inject
- public DefaultBuildManager(Dao dao, ListenerRegistry listenerRegistry) {
+ public DefaultBuildManager(Dao dao, ListenerRegistry listenerRegistry, TaskScheduler taskScheduler,
+ ConfigurationManager configurationManager) {
super(dao);
this.listenerRegistry = listenerRegistry;
+ this.taskScheduler = taskScheduler;
+ this.configurationManager = configurationManager;
}
@Transactional
@@ -207,4 +224,37 @@ public class DefaultBuildManager extends AbstractEntityManager<Build> implements
return getSession().createQuery(criteriaQuery).uniqueResult().intValue();
}
+ @Transactional
+ @Override
+ public void cleanupBuilds(Configuration configuration) {
+ configuration.getBuildCleanupStrategy().cleanup(configuration, getSession());
+ }
+
+ @Listen
+ public void on(SystemStarted event) {
+ taskId = taskScheduler.schedule(this);
+ }
+
+ @Listen
+ public void on(SystemStopping event) {
+ taskScheduler.unschedule(taskId);
+ }
+
+ @Override
+ public void execute() {
+ for (Long configurationId: OneDev.getInstance(CacheManager.class).getConfigurations().keySet())
+ cleanup(configurationId);
+ }
+
+ @Transactional
+ protected void cleanup(Long configurationId) {
+ Configuration configuration = configurationManager.load(configurationId);
+ cleanupBuilds(configuration);
+ }
+
+ @Override
+ public ScheduleBuilder<?> getScheduleBuilder() {
+ return CronScheduleBuilder.cronSchedule("0 0 1 * * ?");
+ }
+
}
diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java
index 398717a156..9b76e853df 100644
--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java
+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java
@@ -500,7 +500,7 @@ public class DefaultCacheManager implements CacheManager {
}
@Override
- public Collection<Long> getBuildIds(Long projectId) {
+ public Collection<Long> getBuildIdsByProject(Long projectId) {
buildsLock.readLock().lock();
configurationsLock.readLock().lock();
try {
@@ -517,6 +517,21 @@ public class DefaultCacheManager implements CacheManager {
}
}
+ @Override
+ public Collection<Long> getBuildIdsByConfiguration(Long configurationId) {
+ buildsLock.readLock().lock();
+ try {
+ Collection<Long> buildIds = new HashSet<>();
+ for (BuildFacade build: builds.values()) {
+ if (build.getConfigurationId().equals(configurationId))
+ buildIds.add(build.getId());
+ }
+ return buildIds;
+ } finally {
+ buildsLock.readLock().unlock();
+ }
+ }
+
@Override
public Collection<Long> filterBuildIds(Long projectId, Collection<String> commitHashes) {
buildsLock.readLock().lock();
diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java
index d9f172a2a3..51a39aa455 100644
--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java
+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java
@@ -60,5 +60,5 @@ public class DefaultConfigurationManager extends AbstractEntityManager<Configura
project.getIssueWorkflow().onDeleteConfiguration(configuration.getName());
}
}
-
+
}
diff --git a/core/src/main/java/io/onedev/server/model/Configuration.java b/core/src/main/java/io/onedev/server/model/Configuration.java
index 1babd89454..7c6f94fd97 100644
--- a/core/src/main/java/io/onedev/server/model/Configuration.java
+++ b/core/src/main/java/io/onedev/server/model/Configuration.java
@@ -17,6 +17,8 @@ import org.hibernate.annotations.Cache;
import org.hibernate.annotations.CacheConcurrencyStrategy;
import org.hibernate.validator.constraints.NotEmpty;
+import io.onedev.server.model.support.configuration.BuildCleanupStrategy;
+import io.onedev.server.model.support.configuration.DoNotCleanup;
import io.onedev.server.util.facade.ConfigurationFacade;
import io.onedev.server.util.validation.annotation.CommitHash;
import io.onedev.server.util.validation.annotation.ConfigurationName;
@@ -42,6 +44,8 @@ public class Configuration extends AbstractEntity {
private String name;
private String baseCommit;
+
+ private BuildCleanupStrategy buildCleanupStrategy = new DoNotCleanup();
@OneToMany(mappedBy="configuration", cascade=CascadeType.REMOVE)
private Collection<PullRequestBuild> pullRequestBuilds = new ArrayList<>();
@@ -68,7 +72,8 @@ public class Configuration extends AbstractEntity {
this.name = name;
}
- @Editable(order=200)
+ @Editable(order=200, description="Optionally specify a base commit to calculate changes for the first build in "
+ + "the configuration")
@CommitHash
public String getBaseCommit() {
return baseCommit;
@@ -78,6 +83,16 @@ public class Configuration extends AbstractEntity {
this.baseCommit = baseCommit;
}
+ @Editable(order=300, description="Optionally specify build cleanup strategy to improvement OneDev performance. "
+ + "Note that this cleanup will not affect builds at build server side")
+ public BuildCleanupStrategy getBuildCleanupStrategy() {
+ return buildCleanupStrategy;
+ }
+
+ public void setBuildCleanupStrategy(BuildCleanupStrategy buildCleanupStrategy) {
+ this.buildCleanupStrategy = buildCleanupStrategy;
+ }
+
public Collection<PullRequestBuild> getPullRequestBuilds() {
return pullRequestBuilds;
}
diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.java b/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.java
new file mode 100644
index 0000000000..b160d9130e
--- /dev/null
+++ b/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.java
@@ -0,0 +1,15 @@
+package io.onedev.server.model.support.configuration;
+
+import java.io.Serializable;
+
+import org.hibernate.Session;
+
+import io.onedev.server.model.Configuration;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable
+public interface BuildCleanupStrategy extends Serializable {
+
+ void cleanup(Configuration configuration, Session session);
+
+}
diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.java b/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.java
new file mode 100644
index 0000000000..3bec2cebce
--- /dev/null
+++ b/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.java
@@ -0,0 +1,17 @@
+package io.onedev.server.model.support.configuration;
+
+import org.hibernate.Session;
+
+import io.onedev.server.model.Configuration;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable(order=50, name="Do not clean up")
+public class DoNotCleanup implements BuildCleanupStrategy {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void cleanup(Configuration configuration, Session session) {
+ }
+
+}
diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.java b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.java
new file mode 100644
index 0000000000..6faf022ce7
--- /dev/null
+++ b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.java
@@ -0,0 +1,50 @@
+package io.onedev.server.model.support.configuration;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.validation.constraints.Min;
+
+import org.hibernate.Session;
+
+import edu.emory.mathcs.backport.java.util.Collections;
+import io.onedev.server.OneDev;
+import io.onedev.server.manager.BuildInfoManager;
+import io.onedev.server.manager.BuildManager;
+import io.onedev.server.manager.CacheManager;
+import io.onedev.server.model.Build;
+import io.onedev.server.model.Configuration;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable(order=200, name="Keep builds by count")
+public class KeepByCount implements BuildCleanupStrategy {
+
+ private static final long serialVersionUID = 1L;
+
+ private int count = 1000;
+
+ @Editable(name="Number of Builds to Keep")
+ @Min(value=1, message="Should keep at least one build")
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ @Override
+ public void cleanup(Configuration configuration, Session session) {
+ List<Long> buildIds = new ArrayList<>(OneDev.getInstance(CacheManager.class).getBuildIdsByConfiguration(configuration.getId()));
+ Collections.sort(buildIds);
+ BuildManager buildManager = OneDev.getInstance(BuildManager.class);
+ BuildInfoManager buildInfoManager = OneDev.getInstance(BuildInfoManager.class);
+ for (int i=0; i<buildIds.size()-count; i++) {
+ Long buildId = buildIds.get(i);
+ Build build = buildManager.load(buildId);
+ buildManager.delete(build);
+ buildInfoManager.delete(configuration.getProject(), build.getId());
+ }
+ }
+
+}
diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.java b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.java
new file mode 100644
index 0000000000..5822fadd21
--- /dev/null
+++ b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.java
@@ -0,0 +1,51 @@
+package io.onedev.server.model.support.configuration;
+
+import java.util.Date;
+import java.util.List;
+
+import javax.validation.constraints.Min;
+
+import org.hibernate.Session;
+import org.hibernate.query.Query;
+import org.joda.time.DateTime;
+
+import io.onedev.server.OneDev;
+import io.onedev.server.manager.BuildInfoManager;
+import io.onedev.server.manager.BuildManager;
+import io.onedev.server.model.Build;
+import io.onedev.server.model.Configuration;
+import io.onedev.server.web.editable.annotation.Editable;
+
+@Editable(order=100, name="Keep builds by days")
+public class KeepByDays implements BuildCleanupStrategy {
+
+ private static final long serialVersionUID = 1L;
+
+ private int count = 30;
+
+ @Editable(name="Number of Days to Keep")
+ @Min(value=1, message="Should keep at least one day")
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void cleanup(Configuration configuration, Session session) {
+ Date keepDate = new DateTime().minusDays(count).toDate();
+ Query<?> query = session.createQuery("from Build where configuration=:configuration and date<:date");
+ query.setParameter("configuration", configuration);
+ query.setParameter("date", keepDate);
+ BuildManager buildManager = OneDev.getInstance(BuildManager.class);
+ BuildInfoManager buildInfoManager = OneDev.getInstance(BuildInfoManager.class);
+ for (Build build: (List<Build>)query.list()) {
+ buildManager.delete(build);
+ buildInfoManager.delete(configuration.getProject(), build.getId());
+ }
+ }
+
+}
diff --git a/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java b/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java
index 869b8a187f..40e6616540 100644
--- a/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java
+++ b/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java
@@ -43,7 +43,7 @@ public class FixedIssueCriteria extends EntityCriteria<Build> {
descendents.add(each.name());
CacheManager cacheManager = OneDev.getInstance(CacheManager.class);
Collection<Long> inBuildIds = cacheManager.filterBuildIds(project.getId(), descendents);
- return inManyValues(context.getBuilder(), attribute, inBuildIds, cacheManager.getBuildIds(project.getId()));
+ return inManyValues(context.getBuilder(), attribute, inBuildIds, cacheManager.getBuildIdsByProject(project.getId()));
}
@Override
diff --git a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html
index 626c26b679..e5e0c1483c 100644
--- a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html
+++ b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html
@@ -17,5 +17,6 @@
</wicket:fragment>
<wicket:fragment wicket:id="actionFrag">
<a wicket:id="delete" title="Delete this configuration"><i class="fa fa-trash"></i></a>
+ <a wicket:id="cleanup" title="Cleanup builds according to build cleanup strategy"><i class="fa fa-eraser"></i></a>
</wicket:fragment>
</wicket:extend>
\ No newline at end of file
diff --git a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java
index 87a7fc95f5..7b558d8815 100644
--- a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java
+++ b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java
@@ -4,7 +4,10 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.wicket.Session;
import org.apache.wicket.ajax.AjaxRequestTarget;
+import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;
+import org.apache.wicket.ajax.markup.html.AjaxLink;
import org.apache.wicket.behavior.AttributeAppender;
import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;
import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;
@@ -28,8 +31,10 @@ import org.hibernate.criterion.Order;
import org.hibernate.criterion.Restrictions;
import io.onedev.server.OneDev;
+import io.onedev.server.manager.BuildManager;
import io.onedev.server.manager.ConfigurationManager;
import io.onedev.server.model.Configuration;
+import io.onedev.server.model.support.configuration.DoNotCleanup;
import io.onedev.server.persistence.dao.EntityCriteria;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.web.WebConstants;
@@ -38,6 +43,7 @@ import io.onedev.server.web.component.datatable.HistoryAwareDataTable;
import io.onedev.server.web.page.project.setting.ProjectSettingPage;
import io.onedev.server.web.util.ConfirmOnClick;
import io.onedev.server.web.util.PagingHistorySupport;
+import io.onedev.server.web.util.ajaxlistener.ConfirmListener;
@SuppressWarnings("serial")
public class ConfigurationListPage extends ProjectSettingPage {
@@ -132,6 +138,29 @@ public class ConfigurationListPage extends ProjectSettingPage {
}.add(new ConfirmOnClick("Do you really want to delete configuration '" + configuration.getName() + "'?")));
+ fragment.add(new AjaxLink<Void>("cleanup") {
+
+ @Override
+ protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {
+ super.updateAjaxAttributes(attributes);
+ String message = "Do you really want to clean up builds of configuration '" + configuration.getName() + "'?";
+ attributes.getAjaxCallListeners().add(new ConfirmListener(message));
+ }
+
+ @Override
+ public void onClick(AjaxRequestTarget target) {
+ OneDev.getInstance(BuildManager.class).cleanupBuilds(rowModel.getObject());
+ Session.get().success("Builds cleaned up");
+ }
+
+ @Override
+ protected void onConfigure() {
+ super.onConfigure();
+ setVisible(!(rowModel.getObject().getBuildCleanupStrategy() instanceof DoNotCleanup));
+ }
+
+ });
+
cellItem.add(fragment);
}
});