The Unified and Autonomous Development Platform
feat: Job cache upload strategy and share in project tree (#1836)
Robin Shenb9a0ba2cb4cf
diff --git a/pom.xml b/pom.xmlindex d4fd5d28f0..e09b140ec1 100644--- a/pom.xml+++ b/pom.xml@@ -631,7 +631,7 @@</repositories><properties><commons.version>2.9.2</commons.version>- <agent.version>1.11.7</agent.version>+ <agent.version>1.12.0</agent.version><slf4j.version>2.0.9</slf4j.version><logback.version>1.4.14</logback.version><antlr.version>4.7.2</antlr.version>diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.javaindex a8cea00648..832a6a8c69 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java@@ -1999,5 +1999,18 @@ public class BuildSpec implements Serializable, Validatable {}});}++ private void migrate32(VersionedYamlDoc doc, Stack<Integer> versions) {+ migrateSteps(doc, versions, stepsNode -> {+ for (var itStepNode = stepsNode.getValue().iterator(); itStepNode.hasNext();) {+ MappingNode stepNode = (MappingNode) itStepNode.next();+ if (stepNode.getTag().getValue().equals("!SetupCacheStep")) {+ stepNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "uploadStrategy"),+ new ScalarNode(Tag.STR, "UPLOAD_IF_NOT_HIT")));+ }+ }+ });+ }}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.javaindex 56cabdcc14..a7280ce223 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/PullRepository.java@@ -78,9 +78,9 @@ public class PullRepository extends SyncRepository {this.targetProject = targetProject;}- @Editable(order=650, name="Access Token for Target Project", description = "Specify a secret to be used " +- "as access token with project management permission for target project. Note that access token " +- "is not required when sync to current or child project and build commit is reachable from " ++ @Editable(order=650, name="Access Token for Target Project", description = "Specify a secret whose value is " ++ "an access token with management permission for above project. Note that this property is not " ++ "required is not required if sync to current or child project and build commit is reachable from " +"default branch")@ChoiceProvider("getSecretChoices")public String getAccessTokenSecret() {diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.javaindex 2466c3226f..48673cf72b 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java@@ -3,20 +3,22 @@ package io.onedev.server.buildspec.step;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.k8shelper.SetupCacheFacade;import io.onedev.k8shelper.StepFacade;-import io.onedev.server.annotation.ChoiceProvider;-import io.onedev.server.annotation.Editable;-import io.onedev.server.annotation.Interpolative;+import io.onedev.server.annotation.*;import io.onedev.server.buildspec.BuildSpec;import io.onedev.server.buildspec.param.ParamCombination;import io.onedev.server.model.Build;import io.onedev.server.model.Project;import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;+import io.onedev.server.util.EditContext;import javax.validation.constraints.NotEmpty;+import javax.validation.constraints.NotNull;import javax.validation.constraints.Size;import java.util.ArrayList;import java.util.List;+import static io.onedev.k8shelper.SetupCacheFacade.UploadStrategy.NEVER_UPLOAD;+import static io.onedev.k8shelper.SetupCacheFacade.UploadStrategy.UPLOAD_IF_NOT_HIT;import static java.util.stream.Collectors.toList;@Editable(order=55, name="Set Up Cache", description = "Set up job cache to speed up job execution. " +@@ -25,16 +27,21 @@ import static java.util.stream.Collectors.toList;public class SetupCacheStep extends Step {private static final long serialVersionUID = 1L;-+private String key;private List<String> loadKeys = new ArrayList<>();private List<String> paths;+ private SetupCacheFacade.UploadStrategy uploadStrategy = UPLOAD_IF_NOT_HIT;++ private String uploadProjectPath;+private String uploadAccessTokenSecret;- @Editable(order=100, name="Cache Key", description = "This key is used to determine if there is a cache hit. " ++ @Editable(order=100, name="Cache Key", description = "This key is used to determine if there is a cache hit in " ++ "project hierarchy (search from current project to root project in order, same for load keys below). " +"A cache is considered hit if its key is exactly the same as the key defined here")@Interpolative(variableSuggester="suggestVariables")@NotEmpty@@ -46,10 +53,10 @@ public class SetupCacheStep extends Step {this.key = key;}- @Editable(order=200, name="Cache Load Keys", description = "In case cache is not hit via above key, OneDev will " +- "loop through load keys defined here in order until a matching cache is found. A cache is considered " +- "matching if its key is prefixed with the load key. If multiple caches matches, the most recent cache " +- "will be returned")+ @Editable(order=200, name="Load Keys", description = "In case cache is not hit via above key, " ++ "OneDev will loop through load keys defined here in order until a matching cache is found " ++ "in project hierarchy. A cache is considered matching if its key is prefixed with the load " ++ "key. If multiple caches matches, the most recent cache will be returned")@Interpolative(variableSuggester="suggestVariables")public List<String> getLoadKeys() {return loadKeys;@@ -76,15 +83,40 @@ public class SetupCacheStep extends Step {this.paths = paths;}+ @Editable(order=400, description = "Specify cache upload strategy after build succeeds. " ++ "<var>Upload If Not Hit</var> means to upload when cache is not found with " ++ "cache key, and <var>Upload If Not Found</var> means cache is not found with both " ++ "cache key and load keys")+ @NotNull+ public SetupCacheFacade.UploadStrategy getUploadStrategy() {+ return uploadStrategy;+ }++ public void setUploadStrategy(SetupCacheFacade.UploadStrategy uploadStrategy) {+ this.uploadStrategy = uploadStrategy;+ }++ @Editable(order=450, placeholder = "Current project", description = "In case cache needs to be uploaded, this property " ++ "specifies target project for the upload. Leave empty for current project")+ @ProjectChoice+ @ShowCondition("isMayUpload")+ public String getUploadProjectPath() {+ return uploadProjectPath;+ }++ public void setUploadProjectPath(String uploadProjectPath) {+ this.uploadProjectPath = uploadProjectPath;+ }+private static List<InputSuggestion> suggestStaticVariables(String matchWith) {return BuildSpec.suggestVariables(matchWith, true, false, false);}- @Editable(order=500, description = "When build is successful, OneDev tries to upload caches not " +- "hit previously (note matching caches via load keys also considered not hit). This upload " +- "is permitted only when the build commit is reachable from default branch, or an access " +- "token is provided with upload cache permission")+ @Editable(order=500, description = "Specify a secret whose value is an access token with upload cache permission " ++ "for above project. Note that this property is not required if upload cache to current or child project " ++ "and build commit is reachable from default branch")@ChoiceProvider("getAccessTokenSecretChoices")+ @ShowCondition("isMayUpload")public String getUploadAccessTokenSecret() {return uploadAccessTokenSecret;}@@ -92,6 +124,10 @@ public class SetupCacheStep extends Step {public void setUploadAccessTokenSecret(String uploadAccessTokenSecret) {this.uploadAccessTokenSecret = uploadAccessTokenSecret;}++ private static boolean isMayUpload() {+ return EditContext.get().getInputValue("uploadStrategy") != NEVER_UPLOAD;+ }private static List<String> getAccessTokenSecretChoices() {return Project.get().getHierarchyJobSecrets()@@ -100,12 +136,12 @@ public class SetupCacheStep extends Step {@Overridepublic StepFacade getFacade(Build build, JobExecutor jobExecutor, String jobToken, ParamCombination paramCombination) {- String accessToken;+ String uploadAccessToken;if (getUploadAccessTokenSecret() != null)- accessToken = build.getJobAuthorizationContext().getSecretValue(getUploadAccessTokenSecret());+ uploadAccessToken = build.getJobAuthorizationContext().getSecretValue(getUploadAccessTokenSecret());else- accessToken = null;- return new SetupCacheFacade(key, loadKeys, paths, accessToken);+ uploadAccessToken = null;+ return new SetupCacheFacade(key, loadKeys, paths, uploadStrategy, uploadProjectPath, uploadAccessToken);}}diff --git a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.javaindex 22ebd2fa3b..34b82c790e 100644--- a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java+++ b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java@@ -224,11 +224,13 @@ public class ClusterResource {@Produces(MediaType.APPLICATION_OCTET_STREAM)@GETpublic Response downloadCache(- @QueryParam("projectId") Long projectId, @QueryParam("cacheId") Long cacheId,- @QueryParam("cachePaths") String cachePaths) {+ @QueryParam("projectId") Long projectId,+ @QueryParam("cacheId") Long cacheId,+ @QueryParam("cachePaths") String joinedCachePaths) {if (!SecurityUtils.isSystem())throw new UnauthorizedException("This api can only be accessed via cluster credential");- StreamingOutput out = os -> jobCacheManager.downloadCacheLocal(projectId, cacheId, Splitter.on('\n').splitToList(cachePaths), os, null);+ var cachePaths = Splitter.on('\n').splitToList(joinedCachePaths);+ StreamingOutput out = os -> jobCacheManager.downloadCache(projectId, cacheId, cachePaths, os);return ok(out).build();}@@ -455,7 +457,7 @@ public class ClusterResource {InputStream cacheStream) {if (!SecurityUtils.isSystem())throw new UnauthorizedException("This api can only be accessed via cluster credential");- jobCacheManager.uploadCacheLocal(projectId, cacheId, Splitter.on('\n').splitToList(cachePaths), cacheStream);+ jobCacheManager.uploadCache(projectId, cacheId, Splitter.on('\n').splitToList(cachePaths), cacheStream);return ok().build();}diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.javaindex d07f7ad4c9..cf11c44333 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java@@ -1,41 +1,35 @@package io.onedev.server.entitymanager;import io.onedev.server.model.JobCache;-import io.onedev.server.model.Project;import io.onedev.server.persistence.dao.EntityManager;+import org.apache.commons.lang3.tuple.Pair;import javax.annotation.Nullable;import java.io.InputStream;import java.io.OutputStream;import java.util.List;-import java.util.concurrent.atomic.AtomicBoolean;-import java.util.function.BiConsumer;import java.util.function.Consumer;-import java.util.function.Function;public interface JobCacheManager extends EntityManager<JobCache> {-- void downloadCache(Long projectId, String cacheKey, List<String> cachePaths,- OutputStream cacheStream);- boolean downloadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,- Consumer<InputStream> cacheStreamHandler);-- void downloadCache(Long projectId, List<String> cacheLoadKey,- List<String> cachePaths, OutputStream cacheStream);+ @Nullable+ Pair<Long, Long> getCacheInfoForDownload(Long projectId, String cacheKey);- boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKey, List<String> cachePaths,- Consumer<InputStream> cacheStreamHandler);+ @Nullable+ Pair<Long, Long> getCacheInfoForDownload(Long projectId, List<String> loadKeys);++ @Nullable Long getCacheIdForUpload(Long projectId, String cacheKey);- void downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,- OutputStream cacheStream, @Nullable AtomicBoolean cacheHit);+ boolean downloadCache(Long projectId, Long cacheId, List<String> cachePaths,+ Consumer<InputStream> cacheStreamHandler);- void uploadCache(Long projectId, String cacheKey, List<String> cachePaths, InputStream cacheStream);+ void downloadCache(Long projectId, Long cacheId, List<String> cachePaths,+ OutputStream cacheStream);- void uploadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths, InputStream cacheStream);+ void uploadCache(Long projectId, Long cacheId, List<String> cachePaths, InputStream cacheStream);- void uploadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,- Consumer<OutputStream> cacheStreamHandler);+ void uploadCache(Long projectId, String cacheKey, List<String> cachePaths,+ Consumer<OutputStream> cacheStreamHandler);@NullableLong getCacheSize(Long projectId, Long cacheId);diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.javaindex 369ee39885..5ec7989e8c 100644--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java@@ -5,8 +5,6 @@ import io.onedev.commons.loader.ManagedSerializedForm;import io.onedev.commons.utils.ExceptionUtils;import io.onedev.commons.utils.FileUtils;import io.onedev.k8shelper.CacheHelper;-import io.onedev.k8shelper.KubernetesHelper;-import io.onedev.server.cluster.ClusterManager;import io.onedev.server.entitymanager.JobCacheManager;import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.event.Listen;@@ -14,16 +12,15 @@ import io.onedev.server.event.system.SystemStarted;import io.onedev.server.event.system.SystemStopping;import io.onedev.server.model.JobCache;import io.onedev.server.model.Project;-import io.onedev.server.persistence.SessionManager;import io.onedev.server.persistence.TransactionManager;-import io.onedev.server.persistence.annotation.Sessional;import io.onedev.server.persistence.annotation.Transactional;import io.onedev.server.persistence.dao.BaseEntityManager;import io.onedev.server.persistence.dao.Dao;import io.onedev.server.taskschedule.SchedulableTask;import io.onedev.server.taskschedule.TaskScheduler;import io.onedev.server.util.IOUtils;-import org.glassfish.jersey.client.ClientProperties;+import org.apache.commons.lang3.tuple.ImmutablePair;+import org.apache.commons.lang3.tuple.Pair;import org.hibernate.criterion.Restrictions;import org.hibernate.exception.ConstraintViolationException;import org.joda.time.DateTime;@@ -35,18 +32,9 @@ import org.slf4j.LoggerFactory;import javax.annotation.Nullable;import javax.inject.Inject;import javax.inject.Singleton;-import javax.ws.rs.client.Client;-import javax.ws.rs.client.ClientBuilder;-import javax.ws.rs.client.Entity;-import javax.ws.rs.client.Invocation;-import javax.ws.rs.core.HttpHeaders;-import javax.ws.rs.core.MediaType;-import javax.ws.rs.core.Response;-import javax.ws.rs.core.StreamingOutput;import java.io.*;import java.util.Date;import java.util.List;-import java.util.concurrent.atomic.AtomicBoolean;import java.util.function.Consumer;import static io.onedev.commons.utils.LockUtils.read;@@ -55,7 +43,6 @@ import static io.onedev.server.model.JobCache.*;import static io.onedev.server.util.IOUtils.BUFFER_SIZE;import static java.nio.charset.StandardCharsets.UTF_8;import static java.util.Comparator.comparing;-import static org.apache.commons.io.IOUtils.copy;@Singletonpublic class DefaultJobCacheManager extends BaseEntityManager<JobCache>@@ -67,31 +54,23 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>private final ProjectManager projectManager;- private final SessionManager sessionManager;-private final TransactionManager transactionManager;- private final ClusterManager clusterManager;-private final TaskScheduler taskScheduler;private volatile String taskId;@Inject- public DefaultJobCacheManager(Dao dao, ProjectManager projectManager, SessionManager sessionManager,- TransactionManager transactionManager, ClusterManager clusterManager,+ public DefaultJobCacheManager(Dao dao, ProjectManager projectManager, TransactionManager transactionManager,TaskScheduler taskScheduler) {super(dao);this.projectManager = projectManager;- this.sessionManager = sessionManager;this.transactionManager = transactionManager;- this.clusterManager = clusterManager;this.taskScheduler = taskScheduler;}- @Sessional@Nullable- protected JobCache find(Project project, String cacheKey) {+ private JobCache find(Project project, String cacheKey) {var criteria = newCriteria();criteria.add(Restrictions.eq(PROP_PROJECT, project));criteria.add(Restrictions.eq(PROP_KEY, cacheKey));@@ -122,29 +101,43 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>}}- private Long getDownloadCacheId(Long projectId, String cacheKey) {- return sessionManager.call(() -> {- var project = projectManager.load(projectId);+ @Transactional+ @Override+ public Pair<Long, Long> getCacheInfoForDownload(Long projectId, String cacheKey) {+ var project = projectManager.load(projectId);+ do {var cache = find(project, cacheKey);- if (cache != null)- return cache.getId();- else- return null;- });+ if (cache != null) {+ cache.setAccessDate(new Date());+ return new ImmutablePair<>(project.getId(), cache.getId());+ }+ project = project.getParent();+ } while (project != null);+ return null;}-++ @Transactional@Override- public void downloadCache(Long projectId, String cacheKey, List<String> cachePaths,- OutputStream cacheStream) {- var cacheId = getDownloadCacheId(projectId, cacheKey);- if (cacheId != null)- downloadCache(projectId, cacheId, cachePaths, cacheStream);- else- writeStream(cacheStream, 0);+ public Pair<Long, Long> getCacheInfoForDownload(Long projectId, List<String> loadKeys) {+ var project = projectManager.load(projectId);+ do {+ for (var loadKey: loadKeys) {+ var cache = project.getJobCaches().stream()+ .filter(it->it.getKey().startsWith(loadKey))+ .max(comparing(JobCache::getAccessDate));+ if (cache.isPresent()) {+ cache.get().setAccessDate(new Date());+ return new ImmutablePair<>(project.getId(), cache.get().getId());+ }+ }+ project = project.getParent();+ } while (project != null);+ return null;}- private boolean downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,- Consumer<InputStream> cacheStreamHandler) {+ @Override+ public boolean downloadCache(Long projectId, Long cacheId, List<String> cachePaths,+ Consumer<InputStream> cacheStreamHandler) {return read(JobCache.getLockName(projectId, cacheId), () -> {var is = openCacheInputStream(projectId, cacheId, cachePaths);if (is != null) try (is) {@@ -157,49 +150,23 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>}@Override- public boolean downloadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,- Consumer<InputStream> cacheStreamHandler) {- var cacheId = getDownloadCacheId(projectId, cacheKey);- if (cacheId != null)- return downloadCacheLocal(projectId, cacheId, cachePaths, cacheStreamHandler);- else- return false;- }-- private Long getDownloadCacheId(Long projectId, List<String> cacheLoadKeys) {- return sessionManager.call(() -> {- var project = projectManager.load(projectId);- for (var cacheLoadKey: cacheLoadKeys) {- var cache = project.getJobCaches().stream()- .filter(it->it.getKey().startsWith(cacheLoadKey))- .max(comparing(JobCache::getAccessDate));- if (cache.isPresent())- return cache.get().getId();- }- return null;- });- }-- @Override- public void downloadCache(Long projectId, List<String> cacheLoadKeys, List<String> cachePaths,+ public void downloadCache(Long projectId, Long cacheId, List<String> cachePaths,OutputStream cacheStream) {- var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);- if (cacheId != null)- downloadCache(projectId, cacheId, cachePaths, cacheStream);- else- writeStream(cacheStream, 0);+ read(JobCache.getLockName(projectId, cacheId), () -> {+ var is = openCacheInputStream(projectId, cacheId, cachePaths);+ if (is != null) {+ try (is) {+ writeStream(cacheStream, 1);+ IOUtils.copy(is, cacheStream, BUFFER_SIZE);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ } else {+ writeStream(cacheStream, 0);+ }+ });}- @Override- public boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKeys, List<String> cachePaths,- Consumer<InputStream> cacheStreamHandler) {- var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);- if (cacheId != null)- return downloadCacheLocal(projectId, cacheId, cachePaths, cacheStreamHandler);- else- return false;- }-@Nullableprivate InputStream openCacheInputStream(Long projectId, Long cacheId, List<String> cachePaths) {var cacheHome = projectManager.getCacheDir(projectId);@@ -221,75 +188,7 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>}@Override- public void downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,- OutputStream cacheStream, @Nullable AtomicBoolean cacheHit) {- read(JobCache.getLockName(projectId, cacheId), () -> {- var is = openCacheInputStream(projectId, cacheId, cachePaths);- if (is != null) {- try (is) {- writeStream(cacheStream, 1);- if (cacheHit != null)- cacheHit.set(true);- else- writeStream(cacheStream, 1);- IOUtils.copy(is, cacheStream, BUFFER_SIZE);- } catch (IOException e) {- throw new RuntimeException(e);- }- } else {- writeStream(cacheStream, 0);- if (cacheHit != null)- cacheHit.set(false);- else- writeStream(cacheStream, 0);- }- });- }-- private void downloadCache(Long projectId, Long cacheId, List<String> cachePaths, OutputStream cacheStream) {- var localServer = clusterManager.getLocalServerAddress();- var activeServer = projectManager.getActiveServer(projectId, true);- boolean found;- if (localServer.equals(activeServer)) {- var cacheHit = new AtomicBoolean();- downloadCacheLocal(projectId, cacheId, cachePaths, cacheStream, cacheHit);- found = cacheHit.get();- } else {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");- try {- String serverUrl = clusterManager.getServerUrl(activeServer);- var target = client.target(serverUrl)- .path("~api/cluster/cache")- .queryParam("projectId", projectId)- .queryParam("cacheId", cacheId)- .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));- Invocation.Builder builder = target.request();- builder.header(HttpHeaders.AUTHORIZATION,- KubernetesHelper.BEARER + " " + clusterManager.getCredential());- try (Response response = builder.get()) {- KubernetesHelper.checkStatus(response);- try (var is = response.readEntity(InputStream.class)) {- found = is.read() == 1;- IOUtils.copy(is, cacheStream, BUFFER_SIZE);- } catch (IOException e) {- throw new RuntimeException(e);- }- }- } finally {- client.close();- }- }-- if (found) {- transactionManager.run(() -> {- var cache = load(cacheId);- cache.setAccessDate(new Date());- });- };- }-- private Long getUploadCacheId(Long projectId, String cacheKey) {+ public Long getCacheIdForUpload(Long projectId, String cacheKey) {Long cacheId;while (true) {try {@@ -321,38 +220,6 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>return cacheId;}- @Override- public void uploadCache(Long projectId, String cacheKey, List<String> cachePaths,- InputStream cacheStream) {- Long cacheId = getUploadCacheId(projectId, cacheKey);-- var localServer = clusterManager.getLocalServerAddress();- var activeServer = projectManager.getActiveServer(projectId, true);- if (localServer.equals(activeServer)) {- uploadCacheLocal(projectId, cacheId, cachePaths, cacheStream);- } else {- Client client = ClientBuilder.newClient();- client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");- try {- String serverUrl = clusterManager.getServerUrl(activeServer);- var target = client.target(serverUrl)- .path("~api/cluster/cache")- .queryParam("projectId", projectId)- .queryParam("cacheId", cacheId)- .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));- Invocation.Builder builder = target.request();- builder.header(HttpHeaders.AUTHORIZATION,- KubernetesHelper.BEARER + " " + clusterManager.getCredential());- StreamingOutput output = os -> copy(cacheStream, os, BUFFER_SIZE);- try (Response response = builder.post(Entity.entity(output, MediaType.APPLICATION_OCTET_STREAM))) {- KubernetesHelper.checkStatus(response);- }- } finally {- client.close();- }- }- }-private OutputStream openCacheOutputStream(Long projectId, Long cacheId, List<String> cachePaths) {var cacheHome = projectManager.getCacheDir(projectId);var cacheDir = new File(cacheHome, String.valueOf(cacheId));@@ -411,9 +278,9 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>}@Override- public void uploadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,- Consumer<OutputStream> cacheStreamHandler) {- Long cacheId = getUploadCacheId(projectId, cacheKey);+ public void uploadCache(Long projectId, String cacheKey, List<String> cachePaths,+ Consumer<OutputStream> cacheStreamHandler) {+ Long cacheId = getCacheIdForUpload(projectId, cacheKey);write(JobCache.getLockName(projectId, cacheId), () -> {try (var os = openCacheOutputStream(projectId, cacheId, cachePaths)) {cacheStreamHandler.accept(os);@@ -424,8 +291,8 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>}@Override- public void uploadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,- InputStream cacheStream) {+ public void uploadCache(Long projectId, Long cacheId, List<String> cachePaths,+ InputStream cacheStream) {write(JobCache.getLockName(projectId, cacheId), () -> {try (var os = openCacheOutputStream(projectId, cacheId, cachePaths)) {IOUtils.copy(cacheStream, os, BUFFER_SIZE);diff --git a/server-core/src/main/java/io/onedev/server/job/JobContext.java b/server-core/src/main/java/io/onedev/server/job/JobContext.javaindex 848ede8475..4cfd746936 100644--- a/server-core/src/main/java/io/onedev/server/job/JobContext.java+++ b/server-core/src/main/java/io/onedev/server/job/JobContext.java@@ -3,6 +3,9 @@ package io.onedev.server.job;import io.onedev.k8shelper.Action;import io.onedev.k8shelper.LeafFacade;import io.onedev.k8shelper.ServiceFacade;+import io.onedev.server.OneDev;+import io.onedev.server.entitymanager.ProjectManager;+import io.onedev.server.model.Project;import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;import org.eclipse.jgit.lib.ObjectId;@@ -118,4 +121,10 @@ public class JobContext implements Serializable {return LeafFacade.of(actions, stepPosition);}+ public boolean canManageProject(Project targetProject) {+ var project = OneDev.getInstance(ProjectManager.class).load(projectId);+ return project.isCommitOnBranch(commitId, project.getDefaultBranch())+ && project.isSelfOrAncestorOf(targetProject);+ }+}diff --git a/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java b/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.javaindex a6bac497ad..4b892a49f5 100644--- a/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java+++ b/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java@@ -1,22 +1,32 @@package io.onedev.server.job;+import com.google.common.base.Joiner;+import io.onedev.commons.utils.ExplicitException;import io.onedev.commons.utils.TaskLogger;import io.onedev.k8shelper.CacheHelper;+import io.onedev.k8shelper.KubernetesHelper;import io.onedev.server.OneDev;+import io.onedev.server.cluster.ClusterManager;import io.onedev.server.entitymanager.JobCacheManager;import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.entitymanager.UserManager;+import io.onedev.server.model.Project;import io.onedev.server.persistence.SessionManager;import io.onedev.server.security.SecurityUtils;+import org.apache.commons.lang3.tuple.Pair;+import org.glassfish.jersey.client.ClientProperties;import javax.annotation.Nullable;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.core.HttpHeaders;+import javax.ws.rs.core.Response;import java.io.File;-import java.util.ArrayList;-import java.util.LinkedHashMap;+import java.io.IOException;+import java.io.InputStream;import java.util.List;-import static io.onedev.k8shelper.CacheHelper.untar;-public class ServerCacheHelper extends CacheHelper {private final JobContext jobContext;@@ -27,50 +37,92 @@ public class ServerCacheHelper extends CacheHelper {}@Override- protected boolean downloadCache(String cacheKey, LinkedHashMap<String, File> cacheDirs) {- return getCacheManager().downloadCacheLocal(- jobContext.getProjectId(),- cacheKey,- new ArrayList<>(cacheDirs.keySet()),- is -> untar(new ArrayList<>(cacheDirs.values()), is));+ protected boolean downloadCache(String cacheKey, List<String> cachePaths, List<File> cacheDirs) {+ var cacheInfo = getCacheManager().getCacheInfoForDownload(jobContext.getProjectId(), cacheKey);+ return downloadCache(cacheInfo, cachePaths, cacheDirs);}@Override- protected boolean downloadCache(List<String> cacheLoadKeys, LinkedHashMap<String, File> cacheDirs) {- return getCacheManager().downloadCacheLocal(- jobContext.getProjectId(),- cacheLoadKeys,- new ArrayList<>(cacheDirs.keySet()),- is -> untar(new ArrayList<>(cacheDirs.values()), is));+ protected boolean downloadCache(List<String> loadKeys, List<String> cachePaths, List<File> cacheDirs) {+ var cacheInfo = getCacheManager().getCacheInfoForDownload(jobContext.getProjectId(), loadKeys);+ return downloadCache(cacheInfo, cachePaths, cacheDirs);+ }++ private boolean downloadCache(Pair<Long, Long> cacheInfo, List<String> cachePaths, List<File> cacheDirs) {+ if (cacheInfo != null) {+ var projectId = cacheInfo.getLeft();+ var cacheId = cacheInfo.getRight();+ var activeServer = getProjectManager().getActiveServer(projectId, true);+ if (activeServer.equals(getClusterManager().getLocalServerAddress())) {+ return getCacheManager().downloadCache(projectId, cacheId, cachePaths, is -> untar(cacheDirs, is));+ } else {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");+ try {+ String serverUrl = getClusterManager().getServerUrl(activeServer);+ var target = client.target(serverUrl)+ .path("~api/cluster/cache")+ .queryParam("projectId", projectId)+ .queryParam("cacheId", cacheId)+ .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));+ Invocation.Builder builder = target.request();+ builder.header(HttpHeaders.AUTHORIZATION,+ KubernetesHelper.BEARER + " " + getClusterManager().getCredential());+ try (Response response = builder.get()) {+ KubernetesHelper.checkStatus(response);+ try (var is = response.readEntity(InputStream.class)) {+ if (is.read() == 1) {+ untar(cacheDirs, is);+ return true;+ } else {+ return false;+ }+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ } finally {+ client.close();+ }+ }+ } else {+ return false;+ }}@Override- protected boolean uploadCache(String cacheKey, LinkedHashMap<String, File> cacheDirs,- @Nullable String accessToken) {- var authorized = getSessionManager().call(() -> {- var projectId = jobContext.getProjectId();- var project = getProjectManager().load(projectId);- if (project.isCommitOnBranch(jobContext.getCommitId(), project.getDefaultBranch())) {- return true;- } else if (accessToken != null) {- var user = getUserManager().findByAccessToken(accessToken);- return user != null && SecurityUtils.canUploadCache(user.asSubject(), project);+ protected boolean uploadCache(String cacheKey, List<String> cachePaths, List<File> cacheDirs,+ @Nullable String projectPath, @Nullable String accessToken) {+ var projectId = getSessionManager().call(() -> {+ Project uploadProject;+ if (projectPath != null) {+ uploadProject = getProjectManager().findByPath(projectPath);+ if (uploadProject == null)+ throw new ExplicitException("Upload project not found: " + projectPath);} else {- return false;+ uploadProject = getProjectManager().load(jobContext.getProjectId());}+ if (jobContext.canManageProject(uploadProject)) {+ return uploadProject.getId();+ } else if (accessToken != null) {+ var user = getUserManager().findByAccessToken(accessToken);+ if (user != null && SecurityUtils.canUploadCache(user.asSubject(), uploadProject))+ return uploadProject.getId();+ }+ return null;});- if (authorized) {- getCacheManager().uploadCacheLocal(- jobContext.getProjectId(),- cacheKey,- new ArrayList<>(cacheDirs.keySet()),- os -> tar(new ArrayList<>(cacheDirs.values()), os));+ if (projectId != null) {+ getCacheManager().uploadCache(projectId, cacheKey, cachePaths, os -> tar(cacheDirs, os));return true;} else {return false;}}+ private ClusterManager getClusterManager() {+ return OneDev.getInstance(ClusterManager.class);+ }+private SessionManager getSessionManager() {return OneDev.getInstance(SessionManager.class);}diff --git a/server-core/src/main/java/io/onedev/server/web/component/markdown/markdown.js b/server-core/src/main/java/io/onedev/server/web/component/markdown/markdown.jsindex 1173e2fcfa..98810c5e0d 100644--- a/server-core/src/main/java/io/onedev/server/web/component/markdown/markdown.js+++ b/server-core/src/main/java/io/onedev/server/web/component/markdown/markdown.js@@ -1299,6 +1299,8 @@ onedev.server.markdown = {$input.focus();document.execCommand("insertText", false, quoted + "\n\n");onedev.server.markdown.fireInputEvent($input);+ $input.blur();+ $input.focus();}};\ No newline at end of filediff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.javaindex 28137d51a2..2252c832e9 100644--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java@@ -1,5 +1,6 @@package io.onedev.server.plugin.executor.kubernetes;+import com.google.common.base.Joiner;import com.google.common.base.Splitter;import io.onedev.commons.utils.FileUtils;import io.onedev.commons.utils.StringUtils;@@ -7,25 +8,31 @@ import io.onedev.commons.utils.TarUtils;import io.onedev.commons.utils.TaskLogger;import io.onedev.k8shelper.K8sJobData;import io.onedev.k8shelper.KubernetesHelper;+import io.onedev.server.cluster.ClusterManager;import io.onedev.server.entitymanager.JobCacheManager;import io.onedev.server.entitymanager.ProjectManager;import io.onedev.server.job.JobContext;import io.onedev.server.job.JobManager;+import io.onedev.server.model.Project;import io.onedev.server.persistence.SessionManager;import io.onedev.server.rest.annotation.Api;import io.onedev.server.security.SecurityUtils;+import io.onedev.server.util.IOUtils;import org.apache.commons.lang.SerializationUtils;+import org.apache.commons.lang3.tuple.Pair;import org.apache.shiro.authz.UnauthorizedException;+import org.glassfish.jersey.client.ClientProperties;import javax.annotation.Nullable;import javax.inject.Inject;import javax.inject.Singleton;import javax.servlet.http.HttpServletRequest;import javax.ws.rs.*;-import javax.ws.rs.core.Context;-import javax.ws.rs.core.MediaType;-import javax.ws.rs.core.Response;-import javax.ws.rs.core.StreamingOutput;+import javax.ws.rs.client.Client;+import javax.ws.rs.client.ClientBuilder;+import javax.ws.rs.client.Entity;+import javax.ws.rs.client.Invocation;+import javax.ws.rs.core.*;import java.io.File;import java.io.IOException;import java.io.InputStream;@@ -36,6 +43,8 @@ import java.util.Map;import static io.onedev.k8shelper.KubernetesHelper.readInt;import static io.onedev.k8shelper.KubernetesHelper.readString;+import static io.onedev.server.util.IOUtils.BUFFER_SIZE;+import static org.apache.commons.io.IOUtils.copy;@Api(internal=true)@Path("/k8s")@@ -51,16 +60,20 @@ public class KubernetesResource {private final ProjectManager projectManager;+ private final ClusterManager clusterManager;+@Contextprivate HttpServletRequest request;@Injectpublic KubernetesResource(JobManager jobManager, JobCacheManager jobCacheManager,- SessionManager sessionManager, ProjectManager projectManager) {+ SessionManager sessionManager, ProjectManager projectManager,+ ClusterManager clusterManager) {this.jobManager = jobManager;this.jobCacheManager = jobCacheManager;this.sessionManager = sessionManager;this.projectManager = projectManager;+ this.clusterManager = clusterManager;}@Path("/job-data")@@ -154,38 +167,78 @@ public class KubernetesResource {public StreamingOutput downloadCache(@QueryParam("jobToken") String jobToken,@QueryParam("cacheKey") @Nullable String cacheKey,- @QueryParam("cacheLoadKeys") @Nullable String joinedCacheLoadKeys,- @QueryParam("cachePaths") String cachePaths) {+ @QueryParam("loadKeys") @Nullable String joinedLoadKeys,+ @QueryParam("cachePaths") String joinedCachePaths) {return os -> {sessionManager.closeSession();try {var jobContext = jobManager.getJobContext(jobToken, true);+ Pair<Long, Long> cacheInfo;if (cacheKey != null) {- jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, Splitter.on('\n').splitToList(cachePaths), os);+ cacheInfo = jobCacheManager.getCacheInfoForDownload(jobContext.getProjectId(), cacheKey);} else {- var cacheLoadKeys = Splitter.on('\n').splitToList(joinedCacheLoadKeys);- jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, Splitter.on('\n').splitToList(cachePaths), os);+ var loadKeys = Splitter.on('\n').splitToList(joinedLoadKeys);+ cacheInfo = jobCacheManager.getCacheInfoForDownload(jobContext.getProjectId(), loadKeys);+ }+ if (cacheInfo != null) {+ var cachePaths = Splitter.on('\n').splitToList(joinedCachePaths);+ var activeServer = projectManager.getActiveServer(cacheInfo.getLeft(), true);+ if (activeServer.equals(clusterManager.getLocalServerAddress())) {+ jobCacheManager.downloadCache(cacheInfo.getLeft(), cacheInfo.getRight(), cachePaths, os);+ } else {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");+ try {+ String serverUrl = clusterManager.getServerUrl(activeServer);+ var target = client.target(serverUrl)+ .path("~api/cluster/cache")+ .queryParam("projectId", cacheInfo.getLeft())+ .queryParam("cacheId", cacheInfo.getRight())+ .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));+ Invocation.Builder builder = target.request();+ builder.header(HttpHeaders.AUTHORIZATION,+ KubernetesHelper.BEARER + " " + clusterManager.getCredential());+ try (Response response = builder.get()) {+ KubernetesHelper.checkStatus(response);+ try (var is = response.readEntity(InputStream.class)) {+ IOUtils.copy(is, os, BUFFER_SIZE);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ } finally {+ client.close();+ }+ }+ } else {+ os.write(0);}} finally {sessionManager.openSession();}};}-+@Path("/upload-cache")@GET- public Response checkUploadCache(+ public Long checkUploadCache(@QueryParam("jobToken") String jobToken,- @QueryParam("cacheKey") String cacheKey,- @QueryParam("cachePaths") String cachePaths) {+ @QueryParam("projectPath") @Nullable String projectPath) {var jobContext = jobManager.getJobContext(jobToken, true);- var project = projectManager.load(jobContext.getProjectId());- if (project.isCommitOnBranch(jobContext.getCommitId(), project.getDefaultBranch())- || SecurityUtils.canUploadCache(project)) {- return Response.ok().build();+ Project uploadProject;+ if (projectPath == null) {+ uploadProject = projectManager.load(jobContext.getProjectId());} else {- throw new UnauthorizedException("Not authorized to upload cache");+ uploadProject = projectManager.findByPath(projectPath);+ if (uploadProject == null)+ throw new NotFoundException("Project not found: " + projectPath);}+ if (jobContext.canManageProject(uploadProject))+ return uploadProject.getId();+ else if (SecurityUtils.canUploadCache(uploadProject))+ return uploadProject.getId();+ else+ throw new UnauthorizedException();}@Path("/upload-cache")@@ -193,14 +246,39 @@ public class KubernetesResource {@POSTpublic Response uploadCache(@QueryParam("jobToken") String jobToken,+ @QueryParam("projectPath") @Nullable String projectPath,@QueryParam("cacheKey") String cacheKey,- @QueryParam("cachePaths") String cachePaths,+ @QueryParam("cachePaths") String joinedCachePaths,InputStream is) {- checkUploadCache(jobToken, cacheKey, cachePaths);- var jobContext = jobManager.getJobContext(jobToken, true);+ var projectId = checkUploadCache(jobToken, projectPath);sessionManager.closeSession();try {- jobCacheManager.uploadCache(jobContext.getProjectId(), cacheKey, Splitter.on('\n').splitToList(cachePaths), is);+ Long cacheId = jobCacheManager.getCacheIdForUpload(projectId, cacheKey);+ var cachePaths = Splitter.on('\n').splitToList(joinedCachePaths);+ var activeServer = projectManager.getActiveServer(projectId, true);+ if (activeServer.equals(clusterManager.getLocalServerAddress())) {+ jobCacheManager.uploadCache(projectId, cacheId, cachePaths, is);+ } else {+ Client client = ClientBuilder.newClient();+ client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");+ try {+ String serverUrl = clusterManager.getServerUrl(activeServer);+ var target = client.target(serverUrl)+ .path("~api/cluster/cache")+ .queryParam("projectId", projectId)+ .queryParam("cacheId", cacheId)+ .queryParam("cachePaths", joinedCachePaths);+ Invocation.Builder builder = target.request();+ builder.header(HttpHeaders.AUTHORIZATION,+ KubernetesHelper.BEARER + " " + clusterManager.getCredential());+ StreamingOutput output = os -> copy(is, os, BUFFER_SIZE);+ try (Response response = builder.post(Entity.entity(output, MediaType.APPLICATION_OCTET_STREAM))) {+ KubernetesHelper.checkStatus(response);+ }+ } finally {+ client.close();+ }+ }return Response.ok().build();} finally {sessionManager.openSession();diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.javaindex cf16ff7e67..91d4c922e3 100644--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java@@ -535,7 +535,7 @@ public class ServerDockerExecutor extends JobExecutor implements RegistryLoginAw}, new ArrayList<>());if (successful)- cacheHelper.uploadCaches();+ cacheHelper.buildSuccessful();return successful;} finally {diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.javaindex f2f913bb87..e5ecdaf296 100644--- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java+++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java@@ -245,7 +245,7 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat}, new ArrayList<>());if (successful)- cacheHelper.uploadCaches();+ cacheHelper.buildSuccessful();return successful;} finally {// Fix https://code.onedev.io/onedev/server/~issues/597