The Unified and Autonomous Development Platform
Add logic to clean up build records
Robin Shen70ddd31b975c
diff --git a/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java b/core/src/main/java/io/onedev/server/manager/BuildInfoManager.javaindex ab10b7e276..18e46e721c 100644--- a/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java+++ b/core/src/main/java/io/onedev/server/manager/BuildInfoManager.java@@ -22,4 +22,6 @@ public interface BuildInfoManager {@NullableCollection<ObjectId> getPrevCommits(Project project, Long buildId);+ void delete(Project project, Long buildId);+}diff --git a/core/src/main/java/io/onedev/server/manager/BuildManager.java b/core/src/main/java/io/onedev/server/manager/BuildManager.javaindex 618b0dcf9f..b374df758c 100644--- a/core/src/main/java/io/onedev/server/manager/BuildManager.java+++ b/core/src/main/java/io/onedev/server/manager/BuildManager.java@@ -32,4 +32,7 @@ public interface BuildManager extends EntityManager<Build> {List<Build> query(Project project, User user, EntityQuery<Build> buildQuery, int firstResult, int maxResults);int count(Project project, User user, EntityCriteria<Build> buildCriteria);++ void cleanupBuilds(Configuration configuration);+}diff --git a/core/src/main/java/io/onedev/server/manager/CacheManager.java b/core/src/main/java/io/onedev/server/manager/CacheManager.javaindex b718ff4bfa..db8967aa59 100644--- a/core/src/main/java/io/onedev/server/manager/CacheManager.java+++ b/core/src/main/java/io/onedev/server/manager/CacheManager.java@@ -48,7 +48,9 @@ public interface CacheManager {Collection<Long> getIssueNumbers(Long projectId);- Collection<Long> getBuildIds(Long projectId);+ Collection<Long> getBuildIdsByProject(Long projectId);++ Collection<Long> getBuildIdsByConfiguration(Long configurationId);Collection<Long> filterBuildIds(Long projectId, Collection<String> commitHashes);diff --git a/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java b/core/src/main/java/io/onedev/server/manager/ConfigurationManager.javaindex 3d799aac5c..ffa6d55974 100644--- a/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java+++ b/core/src/main/java/io/onedev/server/manager/ConfigurationManager.java@@ -12,5 +12,5 @@ public interface ConfigurationManager extends EntityManager<Configuration> {Configuration find(Project project, String name);void save(Configuration configuration, @Nullable String oldName);-+}diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.javaindex 20edf68fee..53f2979946 100644--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildInfoManager.java@@ -225,5 +225,20 @@ public class DefaultBuildInfoManager extends AbstractEnvironmentManager implemenprotected int getEnvVersion() {return INFO_VERSION;}++ @Override+ public void delete(Project project, Long buildId) {+ Environment env = getEnv(project.getId().toString());+ Store store = getStore(env, PREV_COMMITS_STORE);++ env.executeInTransaction(new TransactionalExecutable() {++ @Override+ public void execute(Transaction txn) {+ store.delete(txn, new LongByteIterable(buildId));+ }++ });+ }}diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.javaindex 61f725d937..0575066ae0 100644--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultBuildManager.java@@ -19,12 +19,18 @@ import org.hibernate.Session;import org.hibernate.criterion.Order;import org.hibernate.criterion.Restrictions;import org.hibernate.query.Query;+import org.quartz.CronScheduleBuilder;+import org.quartz.ScheduleBuilder;+import io.onedev.launcher.loader.Listen;import io.onedev.launcher.loader.ListenerRegistry;import io.onedev.server.OneDev;import io.onedev.server.event.build.BuildFinished;import io.onedev.server.event.build.BuildStarted;+import io.onedev.server.event.lifecycle.SystemStarted;+import io.onedev.server.event.lifecycle.SystemStopping;import io.onedev.server.manager.BuildManager;+import io.onedev.server.manager.CacheManager;import io.onedev.server.manager.ConfigurationManager;import io.onedev.server.model.Build;import io.onedev.server.model.Configuration;@@ -42,16 +48,27 @@ import io.onedev.server.search.entity.EntitySort.Direction;import io.onedev.server.search.entity.QueryBuildContext;import io.onedev.server.search.entity.build.BuildQuery;import io.onedev.server.search.entity.build.BuildQueryBuildContext;+import io.onedev.utils.schedule.SchedulableTask;+import io.onedev.utils.schedule.TaskScheduler;@Singleton-public class DefaultBuildManager extends AbstractEntityManager<Build> implements BuildManager {+public class DefaultBuildManager extends AbstractEntityManager<Build> implements BuildManager, SchedulableTask {+ private final TaskScheduler taskScheduler;+private final ListenerRegistry listenerRegistry;+ private final ConfigurationManager configurationManager;++ private String taskId;+@Inject- public DefaultBuildManager(Dao dao, ListenerRegistry listenerRegistry) {+ public DefaultBuildManager(Dao dao, ListenerRegistry listenerRegistry, TaskScheduler taskScheduler,+ ConfigurationManager configurationManager) {super(dao);this.listenerRegistry = listenerRegistry;+ this.taskScheduler = taskScheduler;+ this.configurationManager = configurationManager;}@Transactional@@ -207,4 +224,37 @@ public class DefaultBuildManager extends AbstractEntityManager<Build> implementsreturn getSession().createQuery(criteriaQuery).uniqueResult().intValue();}+ @Transactional+ @Override+ public void cleanupBuilds(Configuration configuration) {+ configuration.getBuildCleanupStrategy().cleanup(configuration, getSession());+ }++ @Listen+ public void on(SystemStarted event) {+ taskId = taskScheduler.schedule(this);+ }++ @Listen+ public void on(SystemStopping event) {+ taskScheduler.unschedule(taskId);+ }++ @Override+ public void execute() {+ for (Long configurationId: OneDev.getInstance(CacheManager.class).getConfigurations().keySet())+ cleanup(configurationId);+ }++ @Transactional+ protected void cleanup(Long configurationId) {+ Configuration configuration = configurationManager.load(configurationId);+ cleanupBuilds(configuration);+ }++ @Override+ public ScheduleBuilder<?> getScheduleBuilder() {+ return CronScheduleBuilder.cronSchedule("0 0 1 * * ?");+ }+}diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.javaindex 398717a156..9b76e853df 100644--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultCacheManager.java@@ -500,7 +500,7 @@ public class DefaultCacheManager implements CacheManager {}@Override- public Collection<Long> getBuildIds(Long projectId) {+ public Collection<Long> getBuildIdsByProject(Long projectId) {buildsLock.readLock().lock();configurationsLock.readLock().lock();try {@@ -517,6 +517,21 @@ public class DefaultCacheManager implements CacheManager {}}+ @Override+ public Collection<Long> getBuildIdsByConfiguration(Long configurationId) {+ buildsLock.readLock().lock();+ try {+ Collection<Long> buildIds = new HashSet<>();+ for (BuildFacade build: builds.values()) {+ if (build.getConfigurationId().equals(configurationId))+ buildIds.add(build.getId());+ }+ return buildIds;+ } finally {+ buildsLock.readLock().unlock();+ }+ }+@Overridepublic Collection<Long> filterBuildIds(Long projectId, Collection<String> commitHashes) {buildsLock.readLock().lock();diff --git a/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java b/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.javaindex d9f172a2a3..51a39aa455 100644--- a/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java+++ b/core/src/main/java/io/onedev/server/manager/impl/DefaultConfigurationManager.java@@ -60,5 +60,5 @@ public class DefaultConfigurationManager extends AbstractEntityManager<Configuraproject.getIssueWorkflow().onDeleteConfiguration(configuration.getName());}}-+}diff --git a/core/src/main/java/io/onedev/server/model/Configuration.java b/core/src/main/java/io/onedev/server/model/Configuration.javaindex 1babd89454..7c6f94fd97 100644--- a/core/src/main/java/io/onedev/server/model/Configuration.java+++ b/core/src/main/java/io/onedev/server/model/Configuration.java@@ -17,6 +17,8 @@ import org.hibernate.annotations.Cache;import org.hibernate.annotations.CacheConcurrencyStrategy;import org.hibernate.validator.constraints.NotEmpty;+import io.onedev.server.model.support.configuration.BuildCleanupStrategy;+import io.onedev.server.model.support.configuration.DoNotCleanup;import io.onedev.server.util.facade.ConfigurationFacade;import io.onedev.server.util.validation.annotation.CommitHash;import io.onedev.server.util.validation.annotation.ConfigurationName;@@ -42,6 +44,8 @@ public class Configuration extends AbstractEntity {private String name;private String baseCommit;++ private BuildCleanupStrategy buildCleanupStrategy = new DoNotCleanup();@OneToMany(mappedBy="configuration", cascade=CascadeType.REMOVE)private Collection<PullRequestBuild> pullRequestBuilds = new ArrayList<>();@@ -68,7 +72,8 @@ public class Configuration extends AbstractEntity {this.name = name;}- @Editable(order=200)+ @Editable(order=200, description="Optionally specify a base commit to calculate changes for the first build in "+ + "the configuration")@CommitHashpublic String getBaseCommit() {return baseCommit;@@ -78,6 +83,16 @@ public class Configuration extends AbstractEntity {this.baseCommit = baseCommit;}+ @Editable(order=300, description="Optionally specify build cleanup strategy to improvement OneDev performance. "+ + "Note that this cleanup will not affect builds at build server side")+ public BuildCleanupStrategy getBuildCleanupStrategy() {+ return buildCleanupStrategy;+ }++ public void setBuildCleanupStrategy(BuildCleanupStrategy buildCleanupStrategy) {+ this.buildCleanupStrategy = buildCleanupStrategy;+ }+public Collection<PullRequestBuild> getPullRequestBuilds() {return pullRequestBuilds;}diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.java b/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.javanew file mode 100644index 0000000000..b160d9130e--- /dev/null+++ b/core/src/main/java/io/onedev/server/model/support/configuration/BuildCleanupStrategy.java@@ -0,0 +1,15 @@+package io.onedev.server.model.support.configuration;++import java.io.Serializable;++import org.hibernate.Session;++import io.onedev.server.model.Configuration;+import io.onedev.server.web.editable.annotation.Editable;++@Editable+public interface BuildCleanupStrategy extends Serializable {++ void cleanup(Configuration configuration, Session session);++}diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.java b/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.javanew file mode 100644index 0000000000..3bec2cebce--- /dev/null+++ b/core/src/main/java/io/onedev/server/model/support/configuration/DoNotCleanup.java@@ -0,0 +1,17 @@+package io.onedev.server.model.support.configuration;++import org.hibernate.Session;++import io.onedev.server.model.Configuration;+import io.onedev.server.web.editable.annotation.Editable;++@Editable(order=50, name="Do not clean up")+public class DoNotCleanup implements BuildCleanupStrategy {++ private static final long serialVersionUID = 1L;++ @Override+ public void cleanup(Configuration configuration, Session session) {+ }++}diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.java b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.javanew file mode 100644index 0000000000..6faf022ce7--- /dev/null+++ b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByCount.java@@ -0,0 +1,50 @@+package io.onedev.server.model.support.configuration;++import java.util.ArrayList;+import java.util.List;++import javax.validation.constraints.Min;++import org.hibernate.Session;++import edu.emory.mathcs.backport.java.util.Collections;+import io.onedev.server.OneDev;+import io.onedev.server.manager.BuildInfoManager;+import io.onedev.server.manager.BuildManager;+import io.onedev.server.manager.CacheManager;+import io.onedev.server.model.Build;+import io.onedev.server.model.Configuration;+import io.onedev.server.web.editable.annotation.Editable;++@Editable(order=200, name="Keep builds by count")+public class KeepByCount implements BuildCleanupStrategy {++ private static final long serialVersionUID = 1L;++ private int count = 1000;++ @Editable(name="Number of Builds to Keep")+ @Min(value=1, message="Should keep at least one build")+ public int getCount() {+ return count;+ }++ public void setCount(int count) {+ this.count = count;+ }++ @Override+ public void cleanup(Configuration configuration, Session session) {+ List<Long> buildIds = new ArrayList<>(OneDev.getInstance(CacheManager.class).getBuildIdsByConfiguration(configuration.getId()));+ Collections.sort(buildIds);+ BuildManager buildManager = OneDev.getInstance(BuildManager.class);+ BuildInfoManager buildInfoManager = OneDev.getInstance(BuildInfoManager.class);+ for (int i=0; i<buildIds.size()-count; i++) {+ Long buildId = buildIds.get(i);+ Build build = buildManager.load(buildId);+ buildManager.delete(build);+ buildInfoManager.delete(configuration.getProject(), build.getId());+ }+ }++}diff --git a/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.java b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.javanew file mode 100644index 0000000000..5822fadd21--- /dev/null+++ b/core/src/main/java/io/onedev/server/model/support/configuration/KeepByDays.java@@ -0,0 +1,51 @@+package io.onedev.server.model.support.configuration;++import java.util.Date;+import java.util.List;++import javax.validation.constraints.Min;++import org.hibernate.Session;+import org.hibernate.query.Query;+import org.joda.time.DateTime;++import io.onedev.server.OneDev;+import io.onedev.server.manager.BuildInfoManager;+import io.onedev.server.manager.BuildManager;+import io.onedev.server.model.Build;+import io.onedev.server.model.Configuration;+import io.onedev.server.web.editable.annotation.Editable;++@Editable(order=100, name="Keep builds by days")+public class KeepByDays implements BuildCleanupStrategy {++ private static final long serialVersionUID = 1L;++ private int count = 30;++ @Editable(name="Number of Days to Keep")+ @Min(value=1, message="Should keep at least one day")+ public int getCount() {+ return count;+ }++ public void setCount(int count) {+ this.count = count;+ }++ @SuppressWarnings("unchecked")+ @Override+ public void cleanup(Configuration configuration, Session session) {+ Date keepDate = new DateTime().minusDays(count).toDate();+ Query<?> query = session.createQuery("from Build where configuration=:configuration and date<:date");+ query.setParameter("configuration", configuration);+ query.setParameter("date", keepDate);+ BuildManager buildManager = OneDev.getInstance(BuildManager.class);+ BuildInfoManager buildInfoManager = OneDev.getInstance(BuildInfoManager.class);+ for (Build build: (List<Build>)query.list()) {+ buildManager.delete(build);+ buildInfoManager.delete(configuration.getProject(), build.getId());+ }+ }++}diff --git a/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java b/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.javaindex 869b8a187f..40e6616540 100644--- a/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java+++ b/core/src/main/java/io/onedev/server/search/entity/build/FixedIssueCriteria.java@@ -43,7 +43,7 @@ public class FixedIssueCriteria extends EntityCriteria<Build> {descendents.add(each.name());CacheManager cacheManager = OneDev.getInstance(CacheManager.class);Collection<Long> inBuildIds = cacheManager.filterBuildIds(project.getId(), descendents);- return inManyValues(context.getBuilder(), attribute, inBuildIds, cacheManager.getBuildIds(project.getId()));+ return inManyValues(context.getBuilder(), attribute, inBuildIds, cacheManager.getBuildIdsByProject(project.getId()));}@Overridediff --git a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.htmlindex 626c26b679..e5e0c1483c 100644--- a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html+++ b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.html@@ -17,5 +17,6 @@</wicket:fragment><wicket:fragment wicket:id="actionFrag"><a wicket:id="delete" title="Delete this configuration"><i class="fa fa-trash"></i></a>+ <a wicket:id="cleanup" title="Cleanup builds according to build cleanup strategy"><i class="fa fa-eraser"></i></a></wicket:fragment></wicket:extend>\ No newline at end of filediff --git a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.javaindex 87a7fc95f5..7b558d8815 100644--- a/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java+++ b/core/src/main/java/io/onedev/server/web/page/project/setting/configuration/ConfigurationListPage.java@@ -4,7 +4,10 @@ import java.util.ArrayList;import java.util.Iterator;import java.util.List;+import org.apache.wicket.Session;import org.apache.wicket.ajax.AjaxRequestTarget;+import org.apache.wicket.ajax.attributes.AjaxRequestAttributes;+import org.apache.wicket.ajax.markup.html.AjaxLink;import org.apache.wicket.behavior.AttributeAppender;import org.apache.wicket.extensions.markup.html.repeater.data.grid.ICellPopulator;import org.apache.wicket.extensions.markup.html.repeater.data.table.AbstractColumn;@@ -28,8 +31,10 @@ import org.hibernate.criterion.Order;import org.hibernate.criterion.Restrictions;import io.onedev.server.OneDev;+import io.onedev.server.manager.BuildManager;import io.onedev.server.manager.ConfigurationManager;import io.onedev.server.model.Configuration;+import io.onedev.server.model.support.configuration.DoNotCleanup;import io.onedev.server.persistence.dao.EntityCriteria;import io.onedev.server.security.SecurityUtils;import io.onedev.server.web.WebConstants;@@ -38,6 +43,7 @@ import io.onedev.server.web.component.datatable.HistoryAwareDataTable;import io.onedev.server.web.page.project.setting.ProjectSettingPage;import io.onedev.server.web.util.ConfirmOnClick;import io.onedev.server.web.util.PagingHistorySupport;+import io.onedev.server.web.util.ajaxlistener.ConfirmListener;@SuppressWarnings("serial")public class ConfigurationListPage extends ProjectSettingPage {@@ -132,6 +138,29 @@ public class ConfigurationListPage extends ProjectSettingPage {}.add(new ConfirmOnClick("Do you really want to delete configuration '" + configuration.getName() + "'?")));+ fragment.add(new AjaxLink<Void>("cleanup") {++ @Override+ protected void updateAjaxAttributes(AjaxRequestAttributes attributes) {+ super.updateAjaxAttributes(attributes);+ String message = "Do you really want to clean up builds of configuration '" + configuration.getName() + "'?";+ attributes.getAjaxCallListeners().add(new ConfirmListener(message));+ }++ @Override+ public void onClick(AjaxRequestTarget target) {+ OneDev.getInstance(BuildManager.class).cleanupBuilds(rowModel.getObject());+ Session.get().success("Builds cleaned up");+ }++ @Override+ protected void onConfigure() {+ super.onConfigure();+ setVisible(!(rowModel.getObject().getBuildCleanupStrategy() instanceof DoNotCleanup));+ }++ });+cellItem.add(fragment);}});