Skip to content

The Unified and Autonomous Development Platform

Fix cron job scheduling issue

Robin Shenf5bc428305cd
diff --git a/pom.xml b/pom.xml
index f705eaf10d..0cca04805f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -620,7 +620,7 @@
</repositories>
<properties>
<commons.version>2.7.3</commons.version>
- <agent.version>1.7.15</agent.version>
+ <agent.version>1.7.16</agent.version>
<slf4j.version>1.7.36</slf4j.version>
<logback.version>1.2.11</logback.version>
<antlr.version>4.7.2</antlr.version>
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/ProjectManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/ProjectManager.java
index 45b7f9eda3..511dfaabee 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/ProjectManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/ProjectManager.java
@@ -3,6 +3,7 @@ package io.onedev.server.entitymanager;
import java.io.File;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Future;
@@ -11,6 +12,7 @@ import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.Path;
import javax.persistence.criteria.Predicate;
+import io.onedev.server.cluster.ProjectServer;
import io.onedev.server.model.support.code.GitPackConfig;
import org.apache.shiro.authz.Permission;
import org.eclipse.jgit.lib.Repository;
@@ -73,6 +75,8 @@ public interface ProjectManager extends EntityManager<Project> {
Collection<Long> getIds();
+ Map<Long, ProjectServer> getStorageServers();
+
Collection<Long> getSubtreeIds(Long projectId);
Collection<Project> getPermittedProjects(Permission permission);
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
index fa8c6a55ef..22f646a39f 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultProjectManager.java
@@ -1,57 +1,12 @@
package io.onedev.server.entitymanager.impl;
-import java.io.File;
-import java.io.IOException;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.nio.file.Files;
-import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.stream.Collectors;
-
-import javax.annotation.Nullable;
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import javax.persistence.criteria.CriteriaBuilder;
-import javax.persistence.criteria.CriteriaQuery;
-import javax.persistence.criteria.From;
-import javax.persistence.criteria.Path;
-import javax.persistence.criteria.Predicate;
-import javax.persistence.criteria.Root;
-import javax.ws.rs.core.MediaType;
-
-import io.onedev.server.model.support.code.GitPackConfig;
-import org.apache.commons.lang3.tuple.ImmutableTriple;
-import org.apache.shiro.authz.Permission;
-import org.apache.shiro.authz.UnauthorizedException;
-import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.lib.ConfigConstants;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.lib.StoredConfig;
-import org.hibernate.Session;
-import org.hibernate.criterion.Restrictions;
-import org.hibernate.query.Query;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.beust.jcommander.internal.Lists;
import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
-import com.hazelcast.cluster.Member;
import com.hazelcast.cluster.MembershipEvent;
import com.hazelcast.cluster.MembershipListener;
-import com.hazelcast.core.EntryEvent;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.map.IMap;
-import com.hazelcast.map.listener.EntryAddedListener;
-import com.hazelcast.map.listener.EntryRemovedListener;
-import com.hazelcast.map.listener.EntryUpdatedListener;
-
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.ExplicitException;
@@ -59,16 +14,10 @@ import io.onedev.commons.utils.FileUtils;
import io.onedev.commons.utils.LockUtils;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.server.cluster.ClusterManager;
+import io.onedev.server.cluster.ClusterRunnable;
import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.cluster.ProjectServer;
-import io.onedev.server.entitymanager.BuildManager;
-import io.onedev.server.entitymanager.IssueManager;
-import io.onedev.server.entitymanager.LinkSpecManager;
-import io.onedev.server.entitymanager.ProjectManager;
-import io.onedev.server.entitymanager.ProjectUpdateManager;
-import io.onedev.server.entitymanager.RoleManager;
-import io.onedev.server.entitymanager.SettingManager;
-import io.onedev.server.entitymanager.UserAuthorizationManager;
+import io.onedev.server.entitymanager.*;
import io.onedev.server.event.Listen;
import io.onedev.server.event.ListenerRegistry;
import io.onedev.server.event.entity.EntityPersisted;
@@ -87,21 +36,11 @@ import io.onedev.server.git.service.GitService;
import io.onedev.server.git.service.RefFacade;
import io.onedev.server.infomanager.CommitInfoManager;
import io.onedev.server.job.JobManager;
-import io.onedev.server.model.Build;
-import io.onedev.server.model.Group;
-import io.onedev.server.model.GroupAuthorization;
-import io.onedev.server.model.Issue;
-import io.onedev.server.model.LinkSpec;
-import io.onedev.server.model.Milestone;
-import io.onedev.server.model.Project;
-import io.onedev.server.model.ProjectUpdate;
-import io.onedev.server.model.PullRequest;
-import io.onedev.server.model.Role;
-import io.onedev.server.model.User;
-import io.onedev.server.model.UserAuthorization;
+import io.onedev.server.model.*;
+import io.onedev.server.model.support.administration.GlobalProjectSetting;
import io.onedev.server.model.support.code.BranchProtection;
+import io.onedev.server.model.support.code.GitPackConfig;
import io.onedev.server.model.support.code.TagProtection;
-import io.onedev.server.model.support.administration.GlobalProjectSetting;
import io.onedev.server.persistence.SessionManager;
import io.onedev.server.persistence.TransactionManager;
import io.onedev.server.persistence.annotation.Sessional;
@@ -125,6 +64,37 @@ import io.onedev.server.util.facade.ProjectFacade;
import io.onedev.server.util.patternset.PatternSet;
import io.onedev.server.util.usage.Usage;
import io.onedev.server.web.avatar.AvatarManager;
+import org.apache.commons.lang3.tuple.ImmutableTriple;
+import org.apache.shiro.authz.Permission;
+import org.apache.shiro.authz.UnauthorizedException;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.StoredConfig;
+import org.hibernate.Session;
+import org.hibernate.criterion.Restrictions;
+import org.hibernate.query.Query;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import javax.persistence.criteria.*;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.io.IOException;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.nio.file.Files;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
@Singleton
public class DefaultProjectManager extends BaseEntityManager<Project>
@@ -329,12 +299,27 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
Project project = (Project) event.getEntity();
Long projectId = project.getId();
- transactionManager.runAfterCommit(new Runnable() {
+ transactionManager.runAfterCommit(new ClusterRunnable() {
+
+ private static final long serialVersionUID = 1L;
@Override
public void run() {
cache.remove(projectId);
- storageServers.remove(projectId);
+ ProjectServer server = storageServers.remove(projectId);
+ if (server != null) {
+ clusterManager.submitToServer(server.getPrimary(), new ClusterTask<Void>() {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Void call() throws Exception {
+ jobManager.unschedule(projectId);
+ return null;
+ }
+
+ });
+ }
}
});
@@ -749,7 +734,6 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
storageServers = hazelcastInstance.getMap("projectStorageServers");
- storageServers.addEntryListener(new StorageEntryListener(), true);
UUID localServerUUID = clusterManager.getLocalServerUUID();
for (var file: storageManager.getProjectsDir().listFiles()) {
var projectId = Long.valueOf(file.getName());
@@ -1049,6 +1033,11 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
else
return null;
}
+
+ @Override
+ public Map<Long, ProjectServer> getStorageServers() {
+ return storageServers;
+ }
@Override
public <T> T runOnProjectServer(Long projectId, ClusterTask<T> task) {
@@ -1105,73 +1094,4 @@ public class DefaultProjectManager extends BaseEntityManager<Project>
});
}
- private class StorageEntryListener implements EntryAddedListener<Long, ProjectServer>,
- EntryRemovedListener<Long, ProjectServer>, EntryUpdatedListener<Long, ProjectServer>, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public void entryUpdated(EntryEvent<Long, ProjectServer> event) {
- Long projectId = event.getKey();
- UUID oldServerUUID = event.getOldValue().getPrimary();
- Member oldServer = clusterManager.getServer(oldServerUUID, false);
- if (oldServer != null && oldServer.equals(clusterManager.getLocalServerUUID())) {
- sessionManager.run(new Runnable() {
-
- @Override
- public void run() {
- jobManager.unschedule(load(projectId));
- }
-
- });
- }
-
- if (event.getValue().getPrimary().equals(clusterManager.getLocalServerUUID())) {
- sessionManager.run(new Runnable() {
-
- @Override
- public void run() {
- jobManager.schedule(load(projectId));
- }
-
- });
- }
- }
-
- @Override
- public void entryRemoved(EntryEvent<Long, ProjectServer> event) {
- Long projectId = event.getKey();
- UUID oldServerUUID = event.getOldValue().getPrimary();
- Member oldServer = clusterManager.getServer(oldServerUUID, false);
- if (oldServer != null && oldServer.getUuid().equals(clusterManager.getLocalServerUUID())) {
- sessionManager.run(new Runnable() {
-
- @Override
- public void run() {
- jobManager.unschedule(load(projectId));
- }
-
- });
-
- }
- }
-
- @Override
- public void entryAdded(EntryEvent<Long, ProjectServer> event) {
- Long projectId = event.getKey();
-
- if (event.getValue().getPrimary().equals(clusterManager.getLocalServerUUID())) {
- sessionManager.run(new Runnable() {
-
- @Override
- public void run() {
- jobManager.schedule(load(projectId));
- }
-
- });
- }
- }
-
- }
-
}
diff --git a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
index 510a5643f8..5ee333f34f 100644
--- a/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/DefaultJobManager.java
@@ -31,10 +31,7 @@ import io.onedev.server.cluster.ClusterTask;
import io.onedev.server.entitymanager.*;
import io.onedev.server.event.Listen;
import io.onedev.server.event.ListenerRegistry;
-import io.onedev.server.event.project.DefaultBranchChanged;
-import io.onedev.server.event.project.ProjectEvent;
-import io.onedev.server.event.project.RefUpdated;
-import io.onedev.server.event.project.ScheduledTimeReaches;
+import io.onedev.server.event.project.*;
import io.onedev.server.event.project.build.*;
import io.onedev.server.event.project.pullrequest.PullRequestEvent;
import io.onedev.server.event.system.SystemStarted;
@@ -169,8 +166,6 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
private volatile Map<String, Collection<String>> allocatedCaches;
- private final Collection<Long> projectIdsToScheduleAtStart = ConcurrentHashMap.newKeySet();
-
@Inject
public DefaultJobManager(BuildManager buildManager, UserManager userManager, ListenerRegistry listenerRegistry,
SettingManager settingManager, TransactionManager transactionManager, LogManager logManager,
@@ -659,6 +654,11 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
buildManager.save(build);
listenerRegistry.post(new BuildFinished(build));
}
+
+ @Listen
+ public void on(ProjectCreated event) {
+ schedule(event.getProject());
+ }
@Sessional
@Listen
@@ -986,13 +986,14 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
allocatedCaches = clusterManager.getHazelcastInstance().getMap("allocatedCaches");
jobServers = clusterManager.getHazelcastInstance().getMap("jobRunServers");
- thread = new Thread(this);
- thread.start();
-
- for (Long projectId: projectIdsToScheduleAtStart) {
- schedule(projectManager.load(projectId));
+ for (var entry: projectManager.getStorageServers().entrySet()) {
+ if (entry.getValue().getPrimary().equals(clusterManager.getLocalServerUUID())) {
+ schedule(projectManager.load(entry.getKey()));
+ }
}
- projectIdsToScheduleAtStart.clear();
+
+ thread = new Thread(this);
+ thread.start();
}
@Listen
@@ -1012,62 +1013,54 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
@Sessional
@Override
public void schedule(Project project) {
- if (thread != null) {
- Collection<String> tasksOfProject = new HashSet<>();
- try {
- String defaultBranch = project.getDefaultBranch();
- if (defaultBranch != null) {
- ObjectId commitId = project.getObjectId(defaultBranch, false);
- if (commitId != null) {
- JobSecretAuthorizationContext.push(new JobSecretAuthorizationContext(project, commitId, null));
- try {
- BuildSpec buildSpec = project.getBuildSpec(commitId);
- if (buildSpec != null) {
- validateBuildSpec(project, commitId, buildSpec);
- ScheduledTimeReaches event = new ScheduledTimeReaches(project);
- for (Job job : buildSpec.getJobMap().values()) {
- for (JobTrigger trigger : job.getTriggers()) {
- if (trigger instanceof ScheduleTrigger) {
- ScheduleTrigger scheduledTrigger = (ScheduleTrigger) trigger;
- SubmitReason reason = trigger.matches(event, job);
- if (reason != null) {
- String taskId = taskScheduler.schedule(newSchedulableTask(project, commitId, job, scheduledTrigger, reason));
- tasksOfProject.add(taskId);
- }
+ Collection<String> tasksOfProject = new HashSet<>();
+ try {
+ String defaultBranch = project.getDefaultBranch();
+ if (defaultBranch != null) {
+ ObjectId commitId = project.getObjectId(defaultBranch, false);
+ if (commitId != null) {
+ JobSecretAuthorizationContext.push(new JobSecretAuthorizationContext(project, commitId, null));
+ try {
+ BuildSpec buildSpec = project.getBuildSpec(commitId);
+ if (buildSpec != null) {
+ validateBuildSpec(project, commitId, buildSpec);
+ ScheduledTimeReaches event = new ScheduledTimeReaches(project);
+ for (Job job : buildSpec.getJobMap().values()) {
+ for (JobTrigger trigger : job.getTriggers()) {
+ if (trigger instanceof ScheduleTrigger) {
+ ScheduleTrigger scheduledTrigger = (ScheduleTrigger) trigger;
+ SubmitReason reason = trigger.matches(event, job);
+ if (reason != null) {
+ String taskId = taskScheduler.schedule(newSchedulableTask(project, commitId, job, scheduledTrigger, reason));
+ tasksOfProject.add(taskId);
}
}
}
}
- } catch (BuildSpecParseException e) {
- logger.warn("Malformed build spec (project: {}, branch: {})",
- project.getPath(), defaultBranch);
- } finally {
- JobSecretAuthorizationContext.pop();
}
+ } catch (BuildSpecParseException e) {
+ logger.warn("Malformed build spec (project: {}, branch: {})",
+ project.getPath(), defaultBranch);
+ } finally {
+ JobSecretAuthorizationContext.pop();
}
}
- } catch (Exception e) {
- logger.error("Error scheduling project '" + project.getPath() + "'", e);
- } finally {
- tasksOfProject = scheduledTasks.put(project.getId(), tasksOfProject);
- if (tasksOfProject != null)
- tasksOfProject.stream().forEach(it -> taskScheduler.unschedule(it));
}
- } else {
- projectIdsToScheduleAtStart.add(project.getId());
+ } catch (Exception e) {
+ logger.error("Error scheduling project '" + project.getPath() + "'", e);
+ } finally {
+ tasksOfProject = scheduledTasks.put(project.getId(), tasksOfProject);
+ if (tasksOfProject != null)
+ tasksOfProject.forEach(taskScheduler::unschedule);
}
}
@Sessional
@Override
- public void unschedule(Project project) {
- if (thread != null) {
- var tasksOfProject = scheduledTasks.remove(project.getId());
- if (tasksOfProject != null)
- tasksOfProject.stream().forEach(it -> taskScheduler.unschedule(it));
- } else {
- projectIdsToScheduleAtStart.remove(project.getId());
- }
+ public void unschedule(Long projectId) {
+ var tasksOfProject = scheduledTasks.remove(projectId);
+ if (tasksOfProject != null)
+ tasksOfProject.forEach(taskScheduler::unschedule);
}
private SchedulableTask newSchedulableTask(Project project, ObjectId commitId, Job job,
diff --git a/server-core/src/main/java/io/onedev/server/job/JobManager.java b/server-core/src/main/java/io/onedev/server/job/JobManager.java
index 42b210e2d2..264d961bf6 100644
--- a/server-core/src/main/java/io/onedev/server/job/JobManager.java
+++ b/server-core/src/main/java/io/onedev/server/job/JobManager.java
@@ -25,7 +25,7 @@ public interface JobManager {
void schedule(Project project);
- void unschedule(Project project);
+ void unschedule(Long projectId);
@Nullable
Build submit(Project project, ObjectId commitId, String jobName,