Skip to content

The Unified and Autonomous Development Platform

feat: Able to specify multiple paths in same cache step (#1788)

Robin Shen71c8f01400a3
diff --git a/pom.xml b/pom.xml
index 7705f809cd..cc11e2ec68 100644
--- a/pom.xml
+++ b/pom.xml
@@ -630,8 +630,8 @@
</repository>
</repositories>
<properties>
- <commons.version>2.8.10</commons.version>
- <agent.version>1.10.14</agent.version>
+ <commons.version>2.8.11</commons.version>
+ <agent.version>1.10.16</agent.version>
<slf4j.version>2.0.9</slf4j.version>
<logback.version>1.4.14</logback.version>
<antlr.version>4.7.2</antlr.version>
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
index 928da4618c..8f2f7b1fb0 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
@@ -45,6 +45,7 @@ import javax.validation.Validator;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.*;
+import java.util.function.Consumer;
import java.util.function.Function;
@Editable
@@ -1906,5 +1907,60 @@ public class BuildSpec implements Serializable, Validatable {
}
}
}
+
+ private void migrateSteps(VersionedYamlDoc doc, Stack<Integer> versions,
+ Consumer<SequenceNode> stepMigrator) {
+ for (NodeTuple specTuple: doc.getValue()) {
+ String specObjectKey = ((ScalarNode)specTuple.getKeyNode()).getValue();
+ if (specObjectKey.equals("jobs")) {
+ SequenceNode jobsNode = (SequenceNode) specTuple.getValueNode();
+ for (Node jobsNodeItem: jobsNode.getValue()) {
+ MappingNode jobNode = (MappingNode) jobsNodeItem;
+ for (NodeTuple jobTuple: jobNode.getValue()) {
+ String jobTupleKey = ((ScalarNode)jobTuple.getKeyNode()).getValue();
+ if (jobTupleKey.equals("steps"))
+ stepMigrator.accept((SequenceNode) jobTuple.getValueNode());
+ }
+ }
+ } else if (specObjectKey.equals("stepTemplates")) {
+ SequenceNode stepTemplatesNode = (SequenceNode) specTuple.getValueNode();
+ for (Node stepTemplatesNodeItem: stepTemplatesNode.getValue()) {
+ MappingNode stepTemplateNode = (MappingNode) stepTemplatesNodeItem;
+ for (NodeTuple stepTemplateTuple: stepTemplateNode.getValue()) {
+ String stepTemplateTupleKey = ((ScalarNode)stepTemplateTuple.getKeyNode()).getValue();
+ if (stepTemplateTupleKey.equals("steps"))
+ stepMigrator.accept((SequenceNode)stepTemplateTuple.getValueNode());
+ }
+ }
+ }
+ }
+ }
+
+ private void migrate30(VersionedYamlDoc doc, Stack<Integer> versions) {
+ migrateSteps(doc, versions, stepsNode -> {
+ for (var itStepNode = stepsNode.getValue().iterator(); itStepNode.hasNext();) {
+ MappingNode stepNode = (MappingNode) itStepNode.next();
+ if (stepNode.getTag().getValue().equals("!SetupCacheStep")) {
+ String path = null;
+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {
+ var stepTuple = itStepTuple.next();
+ if (((ScalarNode) stepTuple.getKeyNode()).getValue().equals("path")) {
+ path = ((ScalarNode) stepTuple.getValueNode()).getValue();
+ itStepTuple.remove();
+ break;
+ }
+ }
+ if (path != null) {
+ List<Node> pathNodes = new ArrayList<>();
+ pathNodes.add(new ScalarNode(Tag.STR, path));
+ stepNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "paths"),
+ new SequenceNode(Tag.SEQ, pathNodes, FlowStyle.BLOCK)));
+
+ }
+ }
+ }
+ });
+ }
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
index 212ecc19cc..2466c3226f 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
@@ -13,6 +13,7 @@ import io.onedev.server.model.Project;
import io.onedev.server.model.support.administration.jobexecutor.JobExecutor;
import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.Size;
import java.util.ArrayList;
import java.util.List;
@@ -29,7 +30,7 @@ public class SetupCacheStep extends Step {
private List<String> loadKeys = new ArrayList<>();
- private String path;
+ private List<String> paths;
private String uploadAccessTokenSecret;
@@ -62,17 +63,17 @@ public class SetupCacheStep extends Step {
return BuildSpec.suggestVariables(matchWith, true, true, false);
}
- @Editable(order=300, name="Cache Path", description = "For docker aware executors, this path is inside container, " +
+ @Editable(order=300, name="Cache Paths", description = "For docker aware executors, this path is inside container, " +
"and accepts both absolute path and relative path (relative to <a href='https://docs.onedev.io/concepts#job-workspace' target='_blank'>job workspace</a>). " +
"For shell related executors which runs on host machine directly, only relative path is accepted")
@Interpolative(variableSuggester="suggestStaticVariables")
- @NotEmpty
- public String getPath() {
- return path;
+ @Size(min=1, max=100)
+ public List<String> getPaths() {
+ return paths;
}
- public void setPath(String path) {
- this.path = path;
+ public void setPaths(List<String> paths) {
+ this.paths = paths;
}
private static List<InputSuggestion> suggestStaticVariables(String matchWith) {
@@ -104,7 +105,7 @@ public class SetupCacheStep extends Step {
accessToken = build.getJobAuthorizationContext().getSecretValue(getUploadAccessTokenSecret());
else
accessToken = null;
- return new SetupCacheFacade(key, loadKeys, path, accessToken);
+ return new SetupCacheFacade(key, loadKeys, paths, accessToken);
}
}
diff --git a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
index 5cf917734e..22ebd2fa3b 100644
--- a/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
+++ b/server-core/src/main/java/io/onedev/server/cluster/ClusterResource.java
@@ -1,7 +1,9 @@
package io.onedev.server.cluster;
+import com.google.common.base.Splitter;
import com.google.common.collect.Sets;
import io.onedev.commons.utils.FileUtils;
+import io.onedev.commons.utils.TarUtils;
import io.onedev.server.OneDev;
import io.onedev.server.StorageManager;
import io.onedev.server.attachment.AttachmentManager;
@@ -40,7 +42,6 @@ import java.io.*;
import java.util.Map;
import java.util.concurrent.ExecutionException;
-import static io.onedev.commons.utils.TarUtils.tar;
import static io.onedev.commons.utils.LockUtils.read;
import static io.onedev.commons.utils.LockUtils.write;
import static io.onedev.server.model.Build.getArtifactsLockName;
@@ -101,13 +102,13 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> read(readLock, () -> {
+ StreamingOutput output = os -> read(readLock, () -> {
File directory = new File(projectManager.getProjectDir(projectId), path);
PatternSet patternSet = PatternSet.parse(patterns);
patternSet.getExcludes().add(SHARE_TEST_DIR + "/**");
- tar(directory, patternSet.getIncludes(), patternSet.getExcludes(), output, false);
+ TarUtils.tar(directory, patternSet.getIncludes(), patternSet.getExcludes(), os, false);
});
- return ok(os).build();
+ return ok(output).build();
}
@Path("/assets")
@@ -117,8 +118,8 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> tar(OneDev.getAssetsDir(), Sets.newHashSet("**"), null, output, false);
- return ok(os).build();
+ StreamingOutput output = os -> TarUtils.tar(OneDev.getAssetsDir(), Sets.newHashSet("**"), null, os, false);
+ return ok(output).build();
}
@Path("/project-file")
@@ -153,14 +154,14 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> read(getArtifactsLockName(projectId, buildNumber), () -> {
+ StreamingOutput output = os -> read(getArtifactsLockName(projectId, buildNumber), () -> {
File artifactsDir = buildManager.getArtifactsDir(projectId, buildNumber);
PatternSet patternSet = PatternSet.parse(artifacts);
patternSet.getExcludes().add(SHARE_TEST_DIR + "/**");
- tar(artifactsDir, patternSet.getIncludes(), patternSet.getExcludes(), output, false);
+ TarUtils.tar(artifactsDir, patternSet.getIncludes(), patternSet.getExcludes(), os, false);
return null;
});
- return ok(os).build();
+ return ok(output).build();
}
@Path("/artifact")
@@ -224,10 +225,10 @@ public class ClusterResource {
@GET
public Response downloadCache(
@QueryParam("projectId") Long projectId, @QueryParam("cacheId") Long cacheId,
- @QueryParam("cachePath") String cachePath) {
+ @QueryParam("cachePaths") String cachePaths) {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput out = os -> jobCacheManager.downloadCacheLocal(projectId, cacheId, cachePath, os, null);
+ StreamingOutput out = os -> jobCacheManager.downloadCacheLocal(projectId, cacheId, Splitter.on('\n').splitToList(cachePaths), os, null);
return ok(out).build();
}
@@ -313,16 +314,16 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> {
+ StreamingOutput output = os -> {
File tempDir = FileUtils.createTempDir("commit-info");
try {
commitInfoManager.export(projectId, tempDir);
- tar(tempDir, output, false);
+ TarUtils.tar(tempDir, os, false);
} finally {
FileUtils.deleteDir(tempDir);
}
};
- return ok(os).build();
+ return ok(output).build();
}
@Path("/visit-info")
@@ -332,16 +333,16 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> {
+ StreamingOutput output = os -> {
File tempDir = FileUtils.createTempDir("visit-info");
try {
visitInfoManager.export(projectId, tempDir);
- tar(tempDir, output, false);
+ TarUtils.tar(tempDir, os, false);
} finally {
FileUtils.deleteDir(tempDir);
}
};
- return ok(os).build();
+ return ok(output).build();
}
@Path("/lfs")
@@ -397,12 +398,12 @@ public class ClusterResource {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- StreamingOutput os = output -> read(attachmentManager.getAttachmentLockName(projectId, attachmentGroup), () -> {
- tar(attachmentManager.getAttachmentGroupDir(projectId, attachmentGroup),
- Sets.newHashSet("**"), Sets.newHashSet(), output, false);
+ StreamingOutput output = os -> read(attachmentManager.getAttachmentLockName(projectId, attachmentGroup), () -> {
+ TarUtils.tar(attachmentManager.getAttachmentGroupDir(projectId, attachmentGroup),
+ Sets.newHashSet("**"), Sets.newHashSet(), os, false);
return null;
});
- return ok(os).build();
+ return ok(output).build();
}
@Path("/artifact")
@@ -450,11 +451,11 @@ public class ClusterResource {
public Response uploadCache(
@QueryParam("projectId") Long projectId,
@QueryParam("cacheId") Long cacheId,
- @QueryParam("cachePath") String cachePath,
+ @QueryParam("cachePaths") String cachePaths,
InputStream cacheStream) {
if (!SecurityUtils.isSystem())
throw new UnauthorizedException("This api can only be accessed via cluster credential");
- jobCacheManager.uploadCacheLocal(projectId, cacheId, cachePath, cacheStream);
+ jobCacheManager.uploadCacheLocal(projectId, cacheId, Splitter.on('\n').splitToList(cachePaths), cacheStream);
return ok().build();
}
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java
index 480c9a1212..d07f7ad4c9 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/JobCacheManager.java
@@ -9,29 +9,32 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
public interface JobCacheManager extends EntityManager<JobCache> {
- void downloadCache(Long projectId, String cacheKey, String cachePath, OutputStream cacheStream);
+ void downloadCache(Long projectId, String cacheKey, List<String> cachePaths,
+ OutputStream cacheStream);
- boolean downloadCacheLocal(Long projectId, String cacheKey, String cachePath,
- Consumer<InputStream> cacheStreamHandler);
+ boolean downloadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,
+ Consumer<InputStream> cacheStreamHandler);
- void downloadCache(Long projectId, List<String> cacheLoadKey, String cachePath, OutputStream cacheStream);
+ void downloadCache(Long projectId, List<String> cacheLoadKey,
+ List<String> cachePaths, OutputStream cacheStream);
- boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKey, String cachePath,
- Consumer<InputStream> cacheStreamHandler);
+ boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKey, List<String> cachePaths,
+ Consumer<InputStream> cacheStreamHandler);
- void downloadCacheLocal(Long projectId, Long cacheId, String cachePath,
+ void downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,
OutputStream cacheStream, @Nullable AtomicBoolean cacheHit);
- void uploadCache(Long projectId, String cacheKey, String cachePath, InputStream cacheStream);
+ void uploadCache(Long projectId, String cacheKey, List<String> cachePaths, InputStream cacheStream);
- void uploadCacheLocal(Long projectId, Long cacheId, String cachePath, InputStream cacheStream);
+ void uploadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths, InputStream cacheStream);
- void uploadCacheLocal(Long projectId, String cacheKey, String cachePath,
+ void uploadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,
Consumer<OutputStream> cacheStreamHandler);
@Nullable
diff --git a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java
index d3c09c62ef..dfa01d5b70 100644
--- a/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java
+++ b/server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultJobCacheManager.java
@@ -1,8 +1,10 @@
package io.onedev.server.entitymanager.impl;
+import com.google.common.base.Joiner;
import io.onedev.commons.loader.ManagedSerializedForm;
import io.onedev.commons.utils.ExceptionUtils;
import io.onedev.commons.utils.FileUtils;
+import io.onedev.k8shelper.CacheHelper;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.server.cluster.ClusterManager;
import io.onedev.server.entitymanager.JobCacheManager;
@@ -59,7 +61,7 @@ import static org.apache.commons.io.IOUtils.copy;
public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
implements JobCacheManager, Serializable, SchedulableTask {
- private static final int CACHE_VERSION = 1;
+ private static final int CACHE_VERSION = 2;
private static final Logger logger = LoggerFactory.getLogger(DefaultJobCacheManager.class);
@@ -132,18 +134,19 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public void downloadCache(Long projectId, String cacheKey, String cachePath, OutputStream cacheStream) {
+ public void downloadCache(Long projectId, String cacheKey, List<String> cachePaths,
+ OutputStream cacheStream) {
var cacheId = getDownloadCacheId(projectId, cacheKey);
if (cacheId != null)
- downloadCache(projectId, cacheId, cachePath, cacheStream);
+ downloadCache(projectId, cacheId, cachePaths, cacheStream);
else
writeStream(cacheStream, 0);
}
- private boolean downloadCacheLocal(Long projectId, Long cacheId, String cachePath,
+ private boolean downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,
Consumer<InputStream> cacheStreamHandler) {
return read(JobCache.getLockName(projectId, cacheId), () -> {
- var is = openCacheInputStream(projectId, cacheId, cachePath);
+ var is = openCacheInputStream(projectId, cacheId, cachePaths);
if (is != null) try (is) {
cacheStreamHandler.accept(is);
return true;
@@ -154,11 +157,11 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public boolean downloadCacheLocal(Long projectId, String cacheKey, String cachePath,
+ public boolean downloadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,
Consumer<InputStream> cacheStreamHandler) {
var cacheId = getDownloadCacheId(projectId, cacheKey);
if (cacheId != null)
- return downloadCacheLocal(projectId, cacheId, cachePath, cacheStreamHandler);
+ return downloadCacheLocal(projectId, cacheId, cachePaths, cacheStreamHandler);
else
return false;
}
@@ -178,35 +181,38 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public void downloadCache(Long projectId, List<String> cacheLoadKeys, String cachePath,
+ public void downloadCache(Long projectId, List<String> cacheLoadKeys, List<String> cachePaths,
OutputStream cacheStream) {
var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);
if (cacheId != null)
- downloadCache(projectId, cacheId, cachePath, cacheStream);
+ downloadCache(projectId, cacheId, cachePaths, cacheStream);
else
writeStream(cacheStream, 0);
}
@Override
- public boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKeys, String cachePath,
+ public boolean downloadCacheLocal(Long projectId, List<String> cacheLoadKeys, List<String> cachePaths,
Consumer<InputStream> cacheStreamHandler) {
var cacheId = getDownloadCacheId(projectId, cacheLoadKeys);
if (cacheId != null)
- return downloadCacheLocal(projectId, cacheId, cachePath, cacheStreamHandler);
+ return downloadCacheLocal(projectId, cacheId, cachePaths, cacheStreamHandler);
else
return false;
}
@Nullable
- private InputStream openCacheInputStream(Long projectId, Long cacheId, String cachePath) {
+ private InputStream openCacheInputStream(Long projectId, Long cacheId, List<String> cachePaths) {
var cacheHome = projectManager.getCacheDir(projectId);
var cacheDir = new File(cacheHome, String.valueOf(cacheId));
if (cacheDir.exists()) {
var stamp = readString(new File(cacheDir, "stamp"));
- if (stamp.equals(CACHE_VERSION + ":" + cachePath)) {
+ if (stamp.equals(CACHE_VERSION + ":" + Joiner.on('\n').join(cachePaths))) {
try {
- return new FileInputStream(new File(cacheDir, "data"));
- } catch (FileNotFoundException e) {
+ var marks = FileUtils.readFileToByteArray(new File(cacheDir, "marks"));
+ return new SequenceInputStream(
+ new ByteArrayInputStream(marks),
+ new FileInputStream(new File(cacheDir, "data")));
+ } catch (IOException e) {
throw new RuntimeException(e);
}
}
@@ -215,10 +221,10 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public void downloadCacheLocal(Long projectId, Long cacheId, String cachePath,
+ public void downloadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,
OutputStream cacheStream, @Nullable AtomicBoolean cacheHit) {
read(JobCache.getLockName(projectId, cacheId), () -> {
- var is = openCacheInputStream(projectId, cacheId, cachePath);
+ var is = openCacheInputStream(projectId, cacheId, cachePaths);
if (is != null) {
try (is) {
writeStream(cacheStream, 1);
@@ -240,13 +246,13 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
});
}
- private void downloadCache(Long projectId, Long cacheId, String cachePath, OutputStream cacheStream) {
+ private void downloadCache(Long projectId, Long cacheId, List<String> cachePaths, OutputStream cacheStream) {
var localServer = clusterManager.getLocalServerAddress();
var activeServer = projectManager.getActiveServer(projectId, true);
boolean found;
if (localServer.equals(activeServer)) {
var cacheHit = new AtomicBoolean();
- downloadCacheLocal(projectId, cacheId, cachePath, cacheStream, cacheHit);
+ downloadCacheLocal(projectId, cacheId, cachePaths, cacheStream, cacheHit);
found = cacheHit.get();
} else {
Client client = ClientBuilder.newClient();
@@ -257,7 +263,7 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
.path("~api/cluster/cache")
.queryParam("projectId", projectId)
.queryParam("cacheId", cacheId)
- .queryParam("cachePath", cachePath);
+ .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));
Invocation.Builder builder = target.request();
builder.header(HttpHeaders.AUTHORIZATION,
KubernetesHelper.BEARER + " " + clusterManager.getCredential());
@@ -316,13 +322,14 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public void uploadCache(Long projectId, String cacheKey, String cachePath, InputStream cacheStream) {
+ public void uploadCache(Long projectId, String cacheKey, List<String> cachePaths,
+ InputStream cacheStream) {
Long cacheId = getUploadCacheId(projectId, cacheKey);
var localServer = clusterManager.getLocalServerAddress();
var activeServer = projectManager.getActiveServer(projectId, true);
if (localServer.equals(activeServer)) {
- uploadCacheLocal(projectId, cacheId, cachePath, cacheStream);
+ uploadCacheLocal(projectId, cacheId, cachePaths, cacheStream);
} else {
Client client = ClientBuilder.newClient();
client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
@@ -332,7 +339,7 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
.path("~api/cluster/cache")
.queryParam("projectId", projectId)
.queryParam("cacheId", cacheId)
- .queryParam("cachePath", cachePath);
+ .queryParam("cachePaths", Joiner.on('\n').join(cachePaths));
Invocation.Builder builder = target.request();
builder.header(HttpHeaders.AUTHORIZATION,
KubernetesHelper.BEARER + " " + clusterManager.getCredential());
@@ -346,24 +353,69 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
}
- private OutputStream openCacheOutputStream(Long projectId, Long cacheId, String cachePath) {
+ private OutputStream openCacheOutputStream(Long projectId, Long cacheId, List<String> cachePaths) {
var cacheHome = projectManager.getCacheDir(projectId);
var cacheDir = new File(cacheHome, String.valueOf(cacheId));
FileUtils.createDir(cacheDir);
- writeString(new File(cacheDir, "stamp"), CACHE_VERSION + ":" + cachePath);
+ writeString(new File(cacheDir, "stamp"), CACHE_VERSION + ":" + Joiner.on('\n').join(cachePaths));
try {
- return new FileOutputStream(new File(cacheDir, "data"));
+ return new FilterOutputStream(new FileOutputStream(new File(cacheDir, "data"))) {
+
+ private final byte[] buffer = new byte[CacheHelper.MARK_BUFFER_SIZE];
+
+ private final byte[] singeByteArray = new byte[1];
+
+ private int ptr;
+
+ private void append(byte data) {
+ singeByteArray[0] = data;
+ append(singeByteArray, 0, 1);
+ }
+
+ private void append(byte[] data, int offset, int length) {
+ if (length <= buffer.length - ptr) {
+ System.arraycopy(data, offset, buffer, ptr, length);
+ ptr += length;
+ } else if (length < buffer.length) {
+ System.arraycopy(buffer, length - buffer.length + ptr, buffer, 0, buffer.length - length);
+ System.arraycopy(data, offset, buffer, buffer.length - length, length);
+ ptr = buffer.length;
+ } else {
+ System.arraycopy(data, offset + length - buffer.length, buffer, 0, buffer.length);
+ ptr = buffer.length;
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ super.write(b);
+ append((byte)b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ append(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ FileUtils.writeByteArrayToFile(new File(cacheDir, "marks"), buffer);
+ }
+
+ };
} catch (FileNotFoundException e) {
throw new RuntimeException(e);
}
}
-
+
@Override
- public void uploadCacheLocal(Long projectId, String cacheKey, String cachePath,
+ public void uploadCacheLocal(Long projectId, String cacheKey, List<String> cachePaths,
Consumer<OutputStream> cacheStreamHandler) {
Long cacheId = getUploadCacheId(projectId, cacheKey);
write(JobCache.getLockName(projectId, cacheId), () -> {
- try (var os = openCacheOutputStream(projectId, cacheId, cachePath)) {
+ try (var os = openCacheOutputStream(projectId, cacheId, cachePaths)) {
cacheStreamHandler.accept(os);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -372,9 +424,10 @@ public class DefaultJobCacheManager extends BaseEntityManager<JobCache>
}
@Override
- public void uploadCacheLocal(Long projectId, Long cacheId, String cachePath, InputStream cacheStream) {
+ public void uploadCacheLocal(Long projectId, Long cacheId, List<String> cachePaths,
+ InputStream cacheStream) {
write(JobCache.getLockName(projectId, cacheId), () -> {
- try (var os = openCacheOutputStream(projectId, cacheId, cachePath)) {
+ try (var os = openCacheOutputStream(projectId, cacheId, cachePaths)) {
IOUtils.copy(cacheStream, os, BUFFER_SIZE);
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/server-core/src/main/java/io/onedev/server/git/GitFilter.java b/server-core/src/main/java/io/onedev/server/git/GitFilter.java
index 9d000cc3a6..14886c3653 100644
--- a/server-core/src/main/java/io/onedev/server/git/GitFilter.java
+++ b/server-core/src/main/java/io/onedev/server/git/GitFilter.java
@@ -15,11 +15,9 @@ import io.onedev.server.model.User;
import io.onedev.server.persistence.SessionManager;
import io.onedev.server.security.CodePullAuthorizationSource;
import io.onedev.server.security.SecurityUtils;
-import io.onedev.server.util.InputStreamWrapper;
import io.onedev.server.util.OutputStreamWrapper;
import io.onedev.server.util.concurrent.PrioritizedRunnable;
import io.onedev.server.util.concurrent.WorkExecutor;
-import io.onedev.server.util.facade.ProjectFacade;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.shiro.authz.UnauthorizedException;
@@ -40,10 +38,7 @@ import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@@ -134,7 +129,7 @@ public class GitFilter implements Filter {
var hookEnvs = HookUtils.getHookEnvs(projectId, userId);
- InputStream stdin = new InputStreamWrapper(ServletUtils.getInputStream(request)) {
+ InputStream stdin = new FilterInputStream(ServletUtils.getInputStream(request)) {
@Override
public void close() {
diff --git a/server-core/src/main/java/io/onedev/server/git/SshCommand.java b/server-core/src/main/java/io/onedev/server/git/SshCommand.java
index c538c3dadc..604b0b6466 100644
--- a/server-core/src/main/java/io/onedev/server/git/SshCommand.java
+++ b/server-core/src/main/java/io/onedev/server/git/SshCommand.java
@@ -16,7 +16,6 @@ import io.onedev.server.persistence.SessionManager;
import io.onedev.server.security.SecurityUtils;
import io.onedev.server.ssh.SshAuthenticator;
import io.onedev.server.ssh.SshManager;
-import io.onedev.server.util.InputStreamWrapper;
import io.onedev.server.util.OutputStreamWrapper;
import io.onedev.server.util.concurrent.PrioritizedRunnable;
import io.onedev.server.util.concurrent.WorkExecutor;
@@ -203,7 +202,7 @@ class SshCommand implements Command, ServerSessionAware {
@Override
public void setInputStream(InputStream in) {
- this.in = new InputStreamWrapper(in) {
+ this.in = new FilterInputStream(in) {
@Override
public void close() {
diff --git a/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java b/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java
index 9e45c6f462..a6bac497ad 100644
--- a/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java
+++ b/server-core/src/main/java/io/onedev/server/job/ServerCacheHelper.java
@@ -1,6 +1,5 @@
package io.onedev.server.job;
-import io.onedev.commons.utils.TarUtils;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.k8shelper.CacheHelper;
import io.onedev.server.OneDev;
@@ -12,8 +11,12 @@ import io.onedev.server.security.SecurityUtils;
import javax.annotation.Nullable;
import java.io.File;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import static io.onedev.k8shelper.CacheHelper.untar;
+
public class ServerCacheHelper extends CacheHelper {
private final JobContext jobContext;
@@ -24,20 +27,26 @@ public class ServerCacheHelper extends CacheHelper {
}
@Override
- protected boolean downloadCache(String cacheKey, String cachePath, File cacheDir) {
- return getCacheManager().downloadCacheLocal(jobContext.getProjectId(),
- cacheKey, cachePath, is -> TarUtils.untar(is, cacheDir, true));
+ protected boolean downloadCache(String cacheKey, LinkedHashMap<String, File> cacheDirs) {
+ return getCacheManager().downloadCacheLocal(
+ jobContext.getProjectId(),
+ cacheKey,
+ new ArrayList<>(cacheDirs.keySet()),
+ is -> untar(new ArrayList<>(cacheDirs.values()), is));
}
@Override
- protected boolean downloadCache(List<String> cacheLoadKeys, String cachePath, File cacheDir) {
- return getCacheManager().downloadCacheLocal(jobContext.getProjectId(),
- cacheLoadKeys, cachePath, is -> TarUtils.untar(is, cacheDir, true));
+ protected boolean downloadCache(List<String> cacheLoadKeys, LinkedHashMap<String, File> cacheDirs) {
+ return getCacheManager().downloadCacheLocal(
+ jobContext.getProjectId(),
+ cacheLoadKeys,
+ new ArrayList<>(cacheDirs.keySet()),
+ is -> untar(new ArrayList<>(cacheDirs.values()), is));
}
@Override
- protected boolean uploadCache(String cacheKey, String cachePath,
- @Nullable String accessToken, File cacheDir) {
+ protected boolean uploadCache(String cacheKey, LinkedHashMap<String, File> cacheDirs,
+ @Nullable String accessToken) {
var authorized = getSessionManager().call(() -> {
var projectId = jobContext.getProjectId();
var project = getProjectManager().load(projectId);
@@ -51,8 +60,11 @@ public class ServerCacheHelper extends CacheHelper {
}
});
if (authorized) {
- getCacheManager().uploadCacheLocal(jobContext.getProjectId(), cacheKey,
- cachePath, os -> TarUtils.tar(cacheDir, os, true));
+ getCacheManager().uploadCacheLocal(
+ jobContext.getProjectId(),
+ cacheKey,
+ new ArrayList<>(cacheDirs.keySet()),
+ os -> tar(new ArrayList<>(cacheDirs.values()), os));
return true;
} else {
return false;
diff --git a/server-core/src/main/java/io/onedev/server/util/InputStreamWrapper.java b/server-core/src/main/java/io/onedev/server/util/InputStreamWrapper.java
deleted file mode 100644
index 2b813bdb1b..0000000000
--- a/server-core/src/main/java/io/onedev/server/util/InputStreamWrapper.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package io.onedev.server.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-public class InputStreamWrapper extends InputStream {
-
- private final InputStream delegate;
-
- public InputStreamWrapper(InputStream delegate) {
- this.delegate = delegate;
- }
-
- @Override
- public int read() throws IOException {
- return delegate.read();
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- return delegate.read(b);
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- return delegate.read(b, off, len);
- }
-
- @Override
- public long skip(long n) throws IOException {
- return delegate.skip(n);
- }
-
- @Override
- public int available() throws IOException {
- return delegate.available();
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
- }
-
- @Override
- public synchronized void mark(int readlimit) {
- delegate.mark(readlimit);
- }
-
- @Override
- public synchronized void reset() throws IOException {
- delegate.reset();
- }
-
- @Override
- public boolean markSupported() {
- return delegate.markSupported();
- }
-
-}
diff --git a/server-core/src/main/java/io/onedev/server/util/OutputStreamWrapper.java b/server-core/src/main/java/io/onedev/server/util/OutputStreamWrapper.java
index b3cc665a70..80c52be8cd 100644
--- a/server-core/src/main/java/io/onedev/server/util/OutputStreamWrapper.java
+++ b/server-core/src/main/java/io/onedev/server/util/OutputStreamWrapper.java
@@ -1,39 +1,23 @@
package io.onedev.server.util;
+import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-public class OutputStreamWrapper extends OutputStream {
-
- private final OutputStream delegate;
-
- public OutputStreamWrapper(OutputStream delegate) {
- this.delegate = delegate;
- }
+public class OutputStreamWrapper extends FilterOutputStream {
- @Override
- public void write(int b) throws IOException {
- delegate.write(b);
+ public OutputStreamWrapper(OutputStream os) {
+ super(os);
}
@Override
public void write(byte[] b) throws IOException {
- delegate.write(b);
+ out.write(b);
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
- delegate.write(b, off, len);
- }
-
- @Override
- public void flush() throws IOException {
- delegate.flush();
- }
-
- @Override
- public void close() throws IOException {
- delegate.close();
+ out.write(b, off, len);
}
}
diff --git a/server-core/src/main/java/io/onedev/server/web/resource/AgentLibResource.java b/server-core/src/main/java/io/onedev/server/web/resource/AgentLibResource.java
index 73a9a7b29f..b04d1dc2be 100644
--- a/server-core/src/main/java/io/onedev/server/web/resource/AgentLibResource.java
+++ b/server-core/src/main/java/io/onedev/server/web/resource/AgentLibResource.java
@@ -51,8 +51,9 @@ public class AgentLibResource extends AbstractResource {
FileUtils.copyFileToDirectory(file, tempDir);
}
- OutputStream os = attributes.getResponse().getOutputStream();
- TarUtils.tar(tempDir, os, false);
+ try(var os = attributes.getResponse().getOutputStream()) {
+ TarUtils.tar(tempDir, os, false);
+ }
} finally {
FileUtils.deleteDir(tempDir);
}
diff --git a/server-core/src/main/java/io/onedev/server/web/resource/AgentResource.java b/server-core/src/main/java/io/onedev/server/web/resource/AgentResource.java
index eaf2e8c04a..732242290e 100644
--- a/server-core/src/main/java/io/onedev/server/web/resource/AgentResource.java
+++ b/server-core/src/main/java/io/onedev/server/web/resource/AgentResource.java
@@ -64,15 +64,15 @@ public class AgentResource extends AbstractResource {
OneDev.getInstance(AgentTokenManager.class).createOrUpdate(token);
props.setProperty("agentToken", token.getValue());
- try (OutputStream os = new FileOutputStream(new File(agentDir, "agent/conf/agent.properties"))) {
+ try (var os = new FileOutputStream(new File(agentDir, "agent/conf/agent.properties"))) {
String comment = "For a list of supported agent properties, please visit:\n"
+ "https://docs.onedev.io/administration-guide/agent-management#agent-propertiesenvironments";
props.store(os, comment);
}
try (
- InputStream is = Agent.class.getClassLoader().getResourceAsStream("agent/conf/logback.xml");
- OutputStream os = new FileOutputStream(new File(agentDir, "agent/conf/logback.xml"));) {
+ var is = Agent.class.getClassLoader().getResourceAsStream("agent/conf/logback.xml");
+ var os = new FileOutputStream(new File(agentDir, "agent/conf/logback.xml"));) {
IOUtils.copy(is, os);
}
FileUtils.touchFile(new File(agentDir, "agent/conf/attributes.properties"));
@@ -102,9 +102,11 @@ public class AgentResource extends AbstractResource {
ZipUtils.zip(agentDir, packageFile, "agent/boot/wrapper-*, agent/bin/*.sh");
IOUtils.copy(packageFile, attributes.getResponse().getOutputStream());
} else {
- TarUtils.tar(agentDir, Sets.newHashSet("**"), Sets.newHashSet(),
- Sets.newHashSet("agent/boot/wrapper-*", "agent/bin/*.sh"),
- attributes.getResponse().getOutputStream(), true);
+ try (var os = attributes.getResponse().getOutputStream()) {
+ TarUtils.tar(agentDir, Sets.newHashSet("**"), Sets.newHashSet(),
+ Sets.newHashSet("agent/boot/wrapper-*", "agent/bin/*.sh"),
+ os, true);
+ }
}
} finally {
FileUtils.deleteDir(tempDir);
diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
index ca8a9a5fd9..c59842d116 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultCommitInfoManager.java
@@ -1183,9 +1183,11 @@ public class DefaultCommitInfoManager extends AbstractMultiEnvironmentManager
KubernetesHelper.BEARER + " " + clusterManager.getCredential());
try (Response response = builder.get()) {
KubernetesHelper.checkStatus(response);
- TarUtils.untar(
- response.readEntity(InputStream.class),
- getEnvDir(targetProjectId.toString()), false);
+ try (var is = response.readEntity(InputStream.class)) {
+ TarUtils.untar(is, getEnvDir(targetProjectId.toString()), false);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
} finally {
client.close();
diff --git a/server-core/src/main/java/io/onedev/server/xodus/DefaultVisitInfoManager.java b/server-core/src/main/java/io/onedev/server/xodus/DefaultVisitInfoManager.java
index e2e182c1c6..a66c385504 100644
--- a/server-core/src/main/java/io/onedev/server/xodus/DefaultVisitInfoManager.java
+++ b/server-core/src/main/java/io/onedev/server/xodus/DefaultVisitInfoManager.java
@@ -29,10 +29,7 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
-import java.io.File;
-import java.io.InputStream;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
+import java.io.*;
import java.util.Date;
import static java.lang.Long.valueOf;
@@ -250,9 +247,11 @@ public class DefaultVisitInfoManager extends AbstractMultiEnvironmentManager
KubernetesHelper.BEARER + " " + clusterManager.getCredential());
try (Response response = builder.get()) {
KubernetesHelper.checkStatus(response);
- TarUtils.untar(
- response.readEntity(InputStream.class),
- getEnvDir(projectId.toString()), false);
+ try (var is = response.readEntity(InputStream.class)) {
+ TarUtils.untar(is, getEnvDir(projectId.toString()), false);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
} finally {
client.close();
diff --git a/server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java b/server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java
index ad749f1fa6..40bb7fe301 100644
--- a/server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java
+++ b/server-plugin/server-plugin-buildspec-dotnet/src/main/java/io/onedev/server/plugin/buildspec/dotnet/DotnetJobSuggestion.java
@@ -1,5 +1,6 @@
package io.onedev.server.plugin.buildspec.dotnet;
+import com.google.common.collect.Lists;
import io.onedev.k8shelper.ExecuteCondition;
import io.onedev.server.buildspec.job.Job;
import io.onedev.server.buildspec.job.JobSuggestion;
@@ -45,7 +46,7 @@ public class DotnetJobSuggestion implements JobSuggestion {
var setupCache = new SetupCacheStep();
setupCache.setName("set up nuget cache");
setupCache.setKey("nuget_packages_@file:checksum@");
- setupCache.setPath("/root/.nuget/packages");
+ setupCache.setPaths(Lists.newArrayList("/root/.nuget/packages"));
setupCache.getLoadKeys().add("nuget_packages");
job.getSteps().add(setupCache);
diff --git a/server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleJobSuggestion.java b/server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleJobSuggestion.java
index 4e92cf9f17..48e1033f1c 100644
--- a/server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleJobSuggestion.java
+++ b/server-plugin/server-plugin-buildspec-gradle/src/main/java/io/onedev/server/plugin/buildspec/gradle/GradleJobSuggestion.java
@@ -1,5 +1,6 @@
package io.onedev.server.plugin.buildspec.gradle;
+import com.google.common.collect.Lists;
import io.onedev.k8shelper.ExecuteCondition;
import io.onedev.server.buildspec.job.Job;
import io.onedev.server.buildspec.job.JobSuggestion;
@@ -41,7 +42,7 @@ public class GradleJobSuggestion implements JobSuggestion {
var setupCache = new SetupCacheStep();
setupCache.setName("set up gradle cache");
setupCache.setKey("gradle_@file:checksum@");
- setupCache.setPath("/home/gradle/.gradle/caches");
+ setupCache.setPaths(Lists.newArrayList("/home/gradle/.gradle/caches"));
setupCache.getLoadKeys().add("gradle");
job.getSteps().add(setupCache);
diff --git a/server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenJobSuggestion.java b/server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenJobSuggestion.java
index 4a86c9c72a..8b1378cfd3 100644
--- a/server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenJobSuggestion.java
+++ b/server-plugin/server-plugin-buildspec-maven/src/main/java/io/onedev/server/plugin/buildspec/maven/MavenJobSuggestion.java
@@ -44,7 +44,7 @@ public class MavenJobSuggestion implements JobSuggestion {
var setupCache = new SetupCacheStep();
setupCache.setName("set up maven cache");
setupCache.setKey("maven_repository_@file:checksum@");
- setupCache.setPath("/root/.m2/repository");
+ setupCache.setPaths(Lists.newArrayList("/root/.m2/repository"));
setupCache.getLoadKeys().add("maven_repository");
job.getSteps().add(setupCache);
diff --git a/server-plugin/server-plugin-buildspec-node/src/main/java/io/onedev/server/plugin/buildspec/node/NodeJobSuggestion.java b/server-plugin/server-plugin-buildspec-node/src/main/java/io/onedev/server/plugin/buildspec/node/NodeJobSuggestion.java
index b8864e3c7a..e41bab87c1 100644
--- a/server-plugin/server-plugin-buildspec-node/src/main/java/io/onedev/server/plugin/buildspec/node/NodeJobSuggestion.java
+++ b/server-plugin/server-plugin-buildspec-node/src/main/java/io/onedev/server/plugin/buildspec/node/NodeJobSuggestion.java
@@ -292,7 +292,7 @@ public class NodeJobSuggestion implements JobSuggestion {
var setupCache = new SetupCacheStep();
setupCache.setName("set up npm cache");
setupCache.setKey("node_modules_@file:checksum@");
- setupCache.setPath("node_modules");
+ setupCache.setPaths(Lists.newArrayList("node_modules"));
setupCache.getLoadKeys().add("node_modules");
return Lists.newArrayList(generateChecksum, setupCache);
diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
index 98207b65a4..fe0c8eb24a 100644
--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesExecutor.java
@@ -1078,11 +1078,8 @@ public class KubernetesExecutor extends JobExecutor implements RegistryLoginAwar
throw new ExplicitException("This step can only be executed by server docker executor or " +
"remote docker executor");
} else {
- if (facade instanceof SetupCacheFacade) {
- var cachePath = ((SetupCacheFacade) facade).getPath();
- if (!cachePaths.add(cachePath))
- throw new ExplicitException("Duplicate cache path: " + cachePath);
- }
+ if (facade instanceof SetupCacheFacade)
+ cachePaths.addAll(((SetupCacheFacade) facade).getPaths());
stepContainerSpec = newHashMap(
"name", containerName,
"image", helperImage);
diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
index 5b2ea791e9..4cd0396add 100644
--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
@@ -144,7 +144,6 @@ public class KubernetesResource {
try {
jobManager.copyDependencies(jobContext, tempDir);
TarUtils.tar(tempDir, os, false);
- os.flush();
} finally {
sessionManager.openSession();
FileUtils.deleteDir(tempDir);
@@ -159,16 +158,16 @@ public class KubernetesResource {
@QueryParam("jobToken") String jobToken,
@QueryParam("cacheKey") @Nullable String cacheKey,
@QueryParam("cacheLoadKeys") @Nullable String joinedCacheLoadKeys,
- @QueryParam("cachePath") String cachePath) {
+ @QueryParam("cachePaths") String cachePaths) {
return os -> {
sessionManager.closeSession();
try {
var jobContext = jobManager.getJobContext(jobToken, true);
if (cacheKey != null) {
- jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, cachePath, os);
+ jobCacheManager.downloadCache(jobContext.getProjectId(), cacheKey, Splitter.on('\n').splitToList(cachePaths), os);
} else {
var cacheLoadKeys = Splitter.on('\n').splitToList(joinedCacheLoadKeys);
- jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, cachePath, os);
+ jobCacheManager.downloadCache(jobContext.getProjectId(), cacheLoadKeys, Splitter.on('\n').splitToList(cachePaths), os);
}
} finally {
sessionManager.openSession();
@@ -181,7 +180,7 @@ public class KubernetesResource {
public Response checkUploadCache(
@QueryParam("jobToken") String jobToken,
@QueryParam("cacheKey") String cacheKey,
- @QueryParam("cachePath") String cachePath) {
+ @QueryParam("cachePaths") String cachePaths) {
var jobContext = jobManager.getJobContext(jobToken, true);
var project = projectManager.load(jobContext.getProjectId());
if (project.isCommitOnBranch(jobContext.getCommitId(), project.getDefaultBranch())
@@ -198,13 +197,13 @@ public class KubernetesResource {
public Response uploadCache(
@QueryParam("jobToken") String jobToken,
@QueryParam("cacheKey") String cacheKey,
- @QueryParam("cachePath") String cachePath,
+ @QueryParam("cachePaths") String cachePaths,
InputStream is) {
- checkUploadCache(jobToken, cacheKey, cachePath);
+ checkUploadCache(jobToken, cacheKey, cachePaths);
var jobContext = jobManager.getJobContext(jobToken, true);
sessionManager.closeSession();
try {
- jobCacheManager.uploadCache(jobContext.getProjectId(), cacheKey, cachePath, is);
+ jobCacheManager.uploadCache(jobContext.getProjectId(), cacheKey, Splitter.on('\n').splitToList(cachePaths), is);
return Response.ok().build();
} finally {
sessionManager.openSession();
diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
index 8e5b126dcf..fdf742b2cc 100644
--- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
+++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
@@ -39,7 +39,6 @@ import static io.onedev.agent.ExecutorUtils.newInfoLogger;
import static io.onedev.agent.ExecutorUtils.newWarningLogger;
import static io.onedev.agent.ShellExecutorUtils.testCommands;
import static io.onedev.k8shelper.KubernetesHelper.*;
-import static java.nio.charset.StandardCharsets.UTF_8;
@Editable(order=ServerShellExecutor.ORDER, name="Server Shell Executor", description="" +
"This executor runs build jobs with OneDev server's shell facility.<br>" +
@@ -220,9 +219,10 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
}
} else if (facade instanceof SetupCacheFacade) {
SetupCacheFacade setupCacheFacade = (SetupCacheFacade) facade;
- var cachePath = setupCacheFacade.getPath();
- if (new File(cachePath).isAbsolute())
- throw new ExplicitException("Shell executor does not allow absolute cache path: " + cachePath);
+ for (var cachePath: setupCacheFacade.getPaths()) {
+ if (new File(cachePath).isAbsolute())
+ throw new ExplicitException("Shell executor does not allow absolute cache path: " + cachePath);
+ }
cacheHelper.setupCache(setupCacheFacade);
} else if (facade instanceof ServerSideFacade) {
ServerSideFacade serverSideFacade = (ServerSideFacade) facade;