Skip to content

The Unified and Autonomous Development Platform

Fix issue #661 - Able to downlad artifacts to specified sub directory under job workspace

Robin Shen52ee4222a10d
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
index 9a08e67d8f..4fe945500f 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/DefaultJobManager.java
@@ -55,6 +55,7 @@ import io.onedev.commons.utils.LockUtils;
import io.onedev.commons.utils.StringUtils;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.k8shelper.Action;
+import io.onedev.k8shelper.CacheAllocationRequest;
import io.onedev.k8shelper.CacheInstance;
import io.onedev.k8shelper.CompositeFacade;
import io.onedev.k8shelper.LeafFacade;
@@ -346,6 +347,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
dependence.setDependent(build);
dependence.setRequireSuccessful(interpolated.isRequireSuccessful());
dependence.setArtifacts(interpolated.getArtifacts());
+ dependence.setDestinationPath(interpolated.getDestinationPath());
build.getDependencies().add(dependence);
}
@@ -389,6 +391,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
dependence.setDependency(dependencyBuild);
dependence.setDependent(build);
dependence.setArtifacts(dependency.getArtifacts());
+ dependence.setDestinationPath(dependency.getDestinationPath());
build.getDependencies().add(dependence);
}
@@ -668,11 +671,14 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
public Void call() throws Exception {
File artifactsDir = dependency.getArtifactsDir();
if (artifactsDir.exists()) {
+ File destDir = targetDir;
+ if (dependence.getDestinationPath() != null)
+ destDir = new File(destDir, dependence.getDestinationPath());
PatternSet patternSet = PatternSet.parse(dependence.getArtifacts());
int baseLen = artifactsDir.getAbsolutePath().length() + 1;
for (File file: patternSet.listFiles(artifactsDir)) {
try {
- FileUtils.copyFile(file, new File(targetDir, file.getAbsolutePath().substring(baseLen)));
+ FileUtils.copyFile(file, new File(destDir, file.getAbsolutePath().substring(baseLen)));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -1296,17 +1302,16 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
}
@Override
- public Map<CacheInstance, String> allocateJobCaches(String jobToken, Date currentTime,
- Map<CacheInstance, Date> cacheInstances) {
+ public Map<CacheInstance, String> allocateJobCaches(String jobToken, CacheAllocationRequest request) {
synchronized (jobContexts) {
JobContext jobContext = getJobContext(jobToken, true);
- List<CacheInstance> sortedInstances = new ArrayList<>(cacheInstances.keySet());
+ List<CacheInstance> sortedInstances = new ArrayList<>(request.getInstances().keySet());
sortedInstances.sort(new Comparator<CacheInstance>() {
@Override
public int compare(CacheInstance o1, CacheInstance o2) {
- return cacheInstances.get(o2).compareTo(cacheInstances.get(o1));
+ return request.getInstances().get(o2).compareTo(request.getInstances().get(o1));
}
});
@@ -1335,7 +1340,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
@Override
public void accept(CacheInstance instance) {
- long ellapsed = currentTime.getTime() - cacheInstances.get(instance).getTime();
+ long ellapsed = request.getCurrentTime().getTime() - request.getInstances().get(instance).getTime();
if (ellapsed > jobContext.getCacheTTL() * 24L * 3600L * 1000L) {
allocations.put(instance, null);
jobContext.getAllocatedCaches().add(instance.getName());
@@ -1345,7 +1350,7 @@ public class DefaultJobManager implements JobManager, Runnable, CodePullAuthoriz
};
- cacheInstances.keySet()
+ request.getInstances().keySet()
.stream()
.filter(it->!allAllocated.contains(it.getName()))
.forEach(deletionMarker);
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/JobDependency.java b/server-core/src/main/java/io/onedev/server/buildspec/job/JobDependency.java
index 7dd32c79d7..2c1116f4c6 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/JobDependency.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/JobDependency.java
@@ -36,6 +36,8 @@ public class JobDependency implements Serializable {
private String artifacts = "**";
+ private String destinationPath;
+
// change Named("jobName") also if change name of this property
@Editable(order=100, name="Job")
@ChoiceProvider("getJobChoices")
@@ -84,6 +86,18 @@ public class JobDependency implements Serializable {
this.artifacts = artifacts;
}
+ @Editable(order=400, placeholder="Job workspace", description=""
+ + "Optionally specify a path relative to <a href='$docRoot/pages/concepts.md#job-workspace'>job workspace</a> "
+ + "to put retrieved artifacts. Leave empty to use job workspace itself")
+ @Interpolative(variableSuggester="suggestVariables")
+ public String getDestinationPath() {
+ return destinationPath;
+ }
+
+ public void setDestinationPath(String destinationPath) {
+ this.destinationPath = destinationPath;
+ }
+
@SuppressWarnings("unused")
private static List<InputSuggestion> suggestVariables(String matchWith) {
return BuildSpec.suggestVariables(matchWith, false, false);
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java b/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java
index 19c6b68cb1..7d6ccb932c 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/JobManager.java
@@ -1,7 +1,6 @@
package io.onedev.server.buildspec.job;
import java.io.File;
-import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -10,6 +9,7 @@ import javax.annotation.Nullable;
import org.eclipse.jgit.lib.ObjectId;
import io.onedev.commons.utils.TaskLogger;
+import io.onedev.k8shelper.CacheAllocationRequest;
import io.onedev.k8shelper.CacheInstance;
import io.onedev.server.model.Build;
import io.onedev.server.model.Project;
@@ -27,8 +27,7 @@ public interface JobManager {
JobContext getJobContext(String jobToken, boolean mustExist);
- Map<CacheInstance, String> allocateJobCaches(String jobToken, Date currentTime,
- Map<CacheInstance, Date> cacheInstances);
+ Map<CacheInstance, String> allocateJobCaches(String jobToken, CacheAllocationRequest request);
@Nullable
Map<String, byte[]> runServerStep(String jobToken, List<Integer> stepPosition,
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/job/projectdependency/ProjectDependency.java b/server-core/src/main/java/io/onedev/server/buildspec/job/projectdependency/ProjectDependency.java
index 202b2c4356..07b0bb0e96 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/job/projectdependency/ProjectDependency.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/job/projectdependency/ProjectDependency.java
@@ -37,6 +37,8 @@ public class ProjectDependency implements Serializable {
private String artifacts = "**";
+ private String destinationPath;
+
private String accessTokenSecret;
// change Named("projectPath") also if change name of this property
@@ -99,6 +101,18 @@ public class ProjectDependency implements Serializable {
this.artifacts = artifacts;
}
+ @Editable(order=500, placeholder="Job workspace", description=""
+ + "Optionally specify a path relative to <a href='$docRoot/pages/concepts.md#job-workspace'>job workspace</a> "
+ + "to put retrieved artifacts. Leave empty to use job workspace itself")
+ @Interpolative(variableSuggester="suggestVariables")
+ public String getDestinationPath() {
+ return destinationPath;
+ }
+
+ public void setDestinationPath(String destinationPath) {
+ this.destinationPath = destinationPath;
+ }
+
@SuppressWarnings("unused")
private static List<InputSuggestion> suggestVariables(String matchWith) {
return BuildSpec.suggestVariables(matchWith, false, false);
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
index fa8688f35b..67f6214ab3 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CheckoutStep.java
@@ -71,7 +71,8 @@ public class CheckoutStep extends Step {
}
@Editable(order=300, placeholder="Job workspace", description="Optionally specify path relative to "
- + "job workspace to clone code into. Leave empty to use job workspace itself")
+ + "<a href='$docRoot/pages/concepts.md#job-workspace'>job workspace</a> to clone code into. "
+ + "Leave empty to use job workspace itself")
@Interpolative(variableSuggester="suggestVariables")
public String getCheckoutPath() {
return checkoutPath;
diff --git a/server-core/src/main/java/io/onedev/server/model/BuildDependence.java b/server-core/src/main/java/io/onedev/server/model/BuildDependence.java
index f9a10062bd..2a4a8281e8 100644
--- a/server-core/src/main/java/io/onedev/server/model/BuildDependence.java
+++ b/server-core/src/main/java/io/onedev/server/model/BuildDependence.java
@@ -37,6 +37,8 @@ public class BuildDependence extends AbstractEntity {
private String artifacts;
+ private String destinationPath;
+
public Build getDependent() {
return dependent;
}
@@ -70,6 +72,15 @@ public class BuildDependence extends AbstractEntity {
this.artifacts = artifacts;
}
+ @Nullable
+ public String getDestinationPath() {
+ return destinationPath;
+ }
+
+ public void setDestinationPath(String destinationPath) {
+ this.destinationPath = destinationPath;
+ }
+
@SuppressWarnings("unused")
private static Build getDependentExample() {
Build build = new Build();
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java
index fb592ee9c6..e7b068bcde 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/BuildDetailPage.java
@@ -386,9 +386,6 @@ public abstract class BuildDetailPage extends ProjectPage
protected List<Tab> load() {
List<Tab> tabs = new ArrayList<>();
- if (SecurityUtils.canReadCode(getProject()))
- tabs.add(new BuildTab("In Pipeline", BuildPipelinePage.class));
-
if (SecurityUtils.canAccessLog(getBuild())) {
tabs.add(new BuildTab("Log", BuildLogPage.class) {
@@ -401,6 +398,9 @@ public abstract class BuildDetailPage extends ProjectPage
});
}
+ if (SecurityUtils.canReadCode(getProject()))
+ tabs.add(new BuildTab("In Pipeline", BuildPipelinePage.class));
+
LockUtils.read(getBuild().getArtifactsLockKey(), new Callable<Void>() {
@Override
diff --git a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java
index dba650d609..0c63667bab 100644
--- a/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java
+++ b/server-core/src/main/java/io/onedev/server/web/page/project/builds/detail/dashboard/BuildDashboardPage.java
@@ -19,10 +19,10 @@ public class BuildDashboardPage extends BuildDetailPage {
super(params);
PageProvider pageProvider;
- if (SecurityUtils.canReadCode(getProject()))
- pageProvider = new PageProvider(BuildPipelinePage.class, BuildPipelinePage.paramsOf(getBuild()));
- else if (SecurityUtils.canAccessLog(getBuild()))
+ if (SecurityUtils.canAccessLog(getBuild()))
pageProvider = new PageProvider(BuildLogPage.class, BuildLogPage.paramsOf(getBuild()));
+ else if (SecurityUtils.canReadCode(getProject()))
+ pageProvider = new PageProvider(BuildPipelinePage.class, BuildPipelinePage.paramsOf(getBuild()));
else if (getBuild().getArtifactsDir().exists())
pageProvider = new PageProvider(BuildArtifactsPage.class, BuildArtifactsPage.paramsOf(getBuild()));
else
diff --git a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
index cc802e39cd..6612d7ede7 100644
--- a/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
+++ b/server-plugin/server-plugin-executor-kubernetes/src/main/java/io/onedev/server/plugin/executor/kubernetes/KubernetesResource.java
@@ -74,10 +74,10 @@ public class KubernetesResource {
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@POST
- public byte[] allocateJobCaches(String cacheAllocationRequestString) {
- CacheAllocationRequest cacheAllocationRequest = CacheAllocationRequest.fromString(cacheAllocationRequestString);
+ public byte[] allocateJobCaches(String requestString) {
+ CacheAllocationRequest request = CacheAllocationRequest.fromString(requestString);
return SerializationUtils.serialize((Serializable) jobManager.allocateJobCaches(
- getJobToken(), cacheAllocationRequest.getCurrentTime(), cacheAllocationRequest.getInstances()));
+ getJobToken(), request));
}
@Path("/run-server-step")
diff --git a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
index f5120d9915..23beeb7e7c 100644
--- a/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
+++ b/server-plugin/server-plugin-executor-serverdocker/src/main/java/io/onedev/server/plugin/executor/serverdocker/ServerDockerExecutor.java
@@ -6,9 +6,7 @@ import static io.onedev.agent.DockerExecutorUtils.deleteNetwork;
import static io.onedev.agent.DockerExecutorUtils.isUseProcessIsolation;
import static io.onedev.agent.DockerExecutorUtils.newDockerKiller;
import static io.onedev.agent.DockerExecutorUtils.startService;
-import static io.onedev.k8shelper.KubernetesHelper.checkCacheAllocations;
import static io.onedev.k8shelper.KubernetesHelper.cloneRepository;
-import static io.onedev.k8shelper.KubernetesHelper.getCacheInstances;
import static io.onedev.k8shelper.KubernetesHelper.installGitCert;
import static io.onedev.k8shelper.KubernetesHelper.readPlaceholderValues;
import static io.onedev.k8shelper.KubernetesHelper.replacePlaceholders;
@@ -20,7 +18,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -28,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
import javax.annotation.Nullable;
import javax.validation.ConstraintValidatorContext;
@@ -53,11 +49,13 @@ import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.ExecutionResult;
import io.onedev.commons.utils.command.LineConsumer;
import io.onedev.k8shelper.BuildImageFacade;
+import io.onedev.k8shelper.CacheAllocationRequest;
import io.onedev.k8shelper.CacheInstance;
import io.onedev.k8shelper.CheckoutFacade;
import io.onedev.k8shelper.CloneInfo;
import io.onedev.k8shelper.CommandFacade;
import io.onedev.k8shelper.CompositeFacade;
+import io.onedev.k8shelper.JobCache;
import io.onedev.k8shelper.KubernetesHelper;
import io.onedev.k8shelper.LeafFacade;
import io.onedev.k8shelper.LeafHandler;
@@ -167,18 +165,22 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
JobManager jobManager = OneDev.getInstance(JobManager.class);
File hostCacheHome = getCacheHome();
- jobLogger.log("Allocating job caches...") ;
- Map<CacheInstance, Date> cacheInstances = getCacheInstances(hostCacheHome);
- Map<CacheInstance, String> cacheAllocations = jobManager.allocateJobCaches(jobToken, new Date(), cacheInstances);
- checkCacheAllocations(hostCacheHome, cacheAllocations, new Consumer<File>() {
-
+ jobLogger.log("Setting up job cache...") ;
+ JobCache cache = new JobCache(hostCacheHome) {
+
@Override
- public void accept(File directory) {
- cleanDirAsRoot(directory, newDocker(), Bootstrap.isInDocker());
+ protected Map<CacheInstance, String> allocate(CacheAllocationRequest request) {
+ return jobManager.allocateJobCaches(jobToken, request);
+ }
+
+ @Override
+ protected void clean(File cacheDir) {
+ cleanDirAsRoot(cacheDir, newDocker(), Bootstrap.isInDocker());
}
- });
-
+ };
+ cache.init(false);
+
login(jobLogger);
createNetwork(newDocker(), network, jobLogger);
@@ -195,9 +197,11 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
AtomicReference<File> hostAuthInfoHome = new AtomicReference<>(null);
try {
+ cache.installSymbolinks(hostWorkspace);
+
jobLogger.log("Copying job dependencies...");
jobContext.copyDependencies(hostWorkspace);
-
+
String containerBuildHome;
String containerWorkspace;
if (SystemUtils.IS_OS_WINDOWS) {
@@ -222,64 +226,66 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
if (getRunOptions() != null)
docker.addArgs(StringUtils.parseQuoteTokens(getRunOptions()));
- docker.addArgs("-v", getHostPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);
-
- for (Map.Entry<String, String> entry: volumeMounts.entrySet()) {
- String hostPath = getHostPath(new File(hostWorkspace, entry.getKey()).getAbsolutePath());
- docker.addArgs("-v", hostPath + ":" + entry.getValue());
- }
-
- if (entrypoint != null)
- docker.addArgs("-w", containerWorkspace);
- else if (workingDir != null)
- docker.addArgs("-w", workingDir);
-
- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
- if (!PathUtils.isCurrent(entry.getValue())) {
+ // Uninstall symbol links as docker can not process it well
+ cache.uninstallSymbolinks(hostWorkspace);
+ try {
+ docker.addArgs("-v", getHostPath(hostBuildHome.getAbsolutePath()) + ":" + containerBuildHome);
+
+ for (Map.Entry<String, String> entry: volumeMounts.entrySet()) {
+ String hostPath = getHostPath(new File(hostWorkspace, entry.getKey()).getAbsolutePath());
+ docker.addArgs("-v", hostPath + ":" + entry.getValue());
+ }
+
+ if (entrypoint != null)
+ docker.addArgs("-w", containerWorkspace);
+ else if (workingDir != null)
+ docker.addArgs("-w", workingDir);
+
+ for (Map.Entry<CacheInstance, String> entry: cache.getAllocations().entrySet()) {
String hostCachePath = entry.getKey().getDirectory(hostCacheHome).getAbsolutePath();
String containerCachePath = PathUtils.resolve(containerWorkspace, entry.getValue());
docker.addArgs("-v", getHostPath(hostCachePath) + ":" + containerCachePath);
- } else {
- throw new ExplicitException("Invalid cache path: " + entry.getValue());
}
- }
-
- if (SystemUtils.IS_OS_WINDOWS)
- docker.addArgs("-v", "//./pipe/docker_engine://./pipe/docker_engine");
- else
- docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");
-
- if (hostAuthInfoHome.get() != null) {
- String hostPath = getHostPath(hostAuthInfoHome.get().getAbsolutePath());
- if (SystemUtils.IS_OS_WINDOWS) {
- docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerAdministrator\\auth-info");
- docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerUser\\auth-info");
- } else {
- docker.addArgs("-v", hostPath + ":/root/auth-info");
+
+ if (SystemUtils.IS_OS_WINDOWS)
+ docker.addArgs("-v", "//./pipe/docker_engine://./pipe/docker_engine");
+ else
+ docker.addArgs("-v", "/var/run/docker.sock:/var/run/docker.sock");
+
+ if (hostAuthInfoHome.get() != null) {
+ String hostPath = getHostPath(hostAuthInfoHome.get().getAbsolutePath());
+ if (SystemUtils.IS_OS_WINDOWS) {
+ docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerAdministrator\\auth-info");
+ docker.addArgs("-v", hostPath + ":C:\\Users\\ContainerUser\\auth-info");
+ } else {
+ docker.addArgs("-v", hostPath + ":/root/auth-info");
+ }
}
+
+ for (Map.Entry<String, String> entry: environments.entrySet())
+ docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());
+
+ docker.addArgs("-e", "ONEDEV_WORKSPACE=" + containerWorkspace);
+
+ if (useTTY)
+ docker.addArgs("-t");
+
+ if (entrypoint != null)
+ docker.addArgs("--entrypoint=" + entrypoint);
+
+ if (isUseProcessIsolation(newDocker(), image, osInfo, jobLogger))
+ docker.addArgs("--isolation=process");
+
+ docker.addArgs(image);
+ docker.addArgs(arguments.toArray(new String[arguments.size()]));
+
+ ExecutionResult result = docker.execute(ExecutorUtils.newInfoLogger(jobLogger),
+ ExecutorUtils.newWarningLogger(jobLogger), null, newDockerKiller(newDocker(),
+ containerName, jobLogger));
+ return result.getReturnCode();
+ } finally {
+ cache.installSymbolinks(hostWorkspace);
}
-
- for (Map.Entry<String, String> entry: environments.entrySet())
- docker.addArgs("-e", entry.getKey() + "=" + entry.getValue());
-
- docker.addArgs("-e", "ONEDEV_WORKSPACE=" + containerWorkspace);
-
- if (useTTY)
- docker.addArgs("-t");
-
- if (entrypoint != null)
- docker.addArgs("--entrypoint=" + entrypoint);
-
- if (isUseProcessIsolation(newDocker(), image, osInfo, jobLogger))
- docker.addArgs("--isolation=process");
-
- docker.addArgs(image);
- docker.addArgs(arguments.toArray(new String[arguments.size()]));
-
- ExecutionResult result = docker.execute(ExecutorUtils.newInfoLogger(jobLogger),
- ExecutorUtils.newWarningLogger(jobLogger), null, newDockerKiller(newDocker(),
- containerName, jobLogger));
- return result.getReturnCode();
}
@Override
@@ -330,7 +336,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
hostAuthInfoHome.set(FileUtils.createTempDir());
Commandline git = new Commandline(AppLoader.getInstance(GitConfig.class).getExecutable());
- checkoutFacade.setupWorkingDir(git, hostWorkspace, hostCacheHome, cacheAllocations);
+ checkoutFacade.setupWorkingDir(git, hostWorkspace);
git.environments().put("HOME", hostAuthInfoHome.get().getAbsolutePath());
CloneInfo cloneInfo = checkoutFacade.getCloneInfo();
@@ -362,7 +368,6 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
}
} else {
ServerSideFacade serverSideFacade = (ServerSideFacade) facade;
-
File filesDir = FileUtils.createTempDir();
try {
Collection<String> placeholders = serverSideFacade.getPlaceholders();
@@ -410,6 +415,7 @@ public class ServerDockerExecutor extends JobExecutor implements Testable<TestDa
if (!successful)
throw new FailedException();
} finally {
+ cache.uninstallSymbolinks(hostWorkspace);
// Fix https://code.onedev.io/projects/160/issues/597
if (SystemUtils.IS_OS_WINDOWS)
FileUtils.deleteDir(hostWorkspace);
diff --git a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
index 6c396b6b07..3981e2c5a7 100644
--- a/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
+++ b/server-plugin/server-plugin-executor-servershell/src/main/java/io/onedev/server/plugin/executor/servershell/ServerShellExecutor.java
@@ -1,10 +1,7 @@
package io.onedev.server.plugin.executor.servershell;
-import static io.onedev.agent.ShellExecutorUtils.resolveCachePath;
import static io.onedev.agent.ShellExecutorUtils.testCommands;
-import static io.onedev.k8shelper.KubernetesHelper.checkCacheAllocations;
import static io.onedev.k8shelper.KubernetesHelper.cloneRepository;
-import static io.onedev.k8shelper.KubernetesHelper.getCacheInstances;
import static io.onedev.k8shelper.KubernetesHelper.installGitCert;
import static io.onedev.k8shelper.KubernetesHelper.readPlaceholderValues;
import static io.onedev.k8shelper.KubernetesHelper.replacePlaceholders;
@@ -12,15 +9,12 @@ import static io.onedev.k8shelper.KubernetesHelper.replacePlaceholders;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.function.Consumer;
import javax.validation.constraints.Size;
@@ -32,24 +26,24 @@ import io.onedev.commons.bootstrap.Bootstrap;
import io.onedev.commons.loader.AppLoader;
import io.onedev.commons.utils.ExplicitException;
import io.onedev.commons.utils.FileUtils;
-import io.onedev.commons.utils.PathUtils;
import io.onedev.commons.utils.TaskLogger;
import io.onedev.commons.utils.command.Commandline;
import io.onedev.commons.utils.command.ExecutionResult;
import io.onedev.k8shelper.BuildImageFacade;
+import io.onedev.k8shelper.CacheAllocationRequest;
import io.onedev.k8shelper.CacheInstance;
import io.onedev.k8shelper.CheckoutFacade;
import io.onedev.k8shelper.CloneInfo;
import io.onedev.k8shelper.CommandFacade;
import io.onedev.k8shelper.CompositeFacade;
-import io.onedev.k8shelper.RunContainerFacade;
+import io.onedev.k8shelper.JobCache;
import io.onedev.k8shelper.LeafFacade;
import io.onedev.k8shelper.LeafHandler;
import io.onedev.k8shelper.OsExecution;
import io.onedev.k8shelper.OsInfo;
+import io.onedev.k8shelper.RunContainerFacade;
import io.onedev.k8shelper.ServerSideFacade;
import io.onedev.server.OneDev;
-import io.onedev.server.buildspec.job.CacheSpec;
import io.onedev.server.buildspec.job.JobContext;
import io.onedev.server.buildspec.job.JobManager;
import io.onedev.server.git.config.GitConfig;
@@ -102,26 +96,25 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
JobManager jobManager = OneDev.getInstance(JobManager.class);
File cacheHomeDir = getCacheHome();
- for (CacheSpec cacheSpec: jobContext.getCacheSpecs()) {
- if (new File(cacheSpec.getPath()).isAbsolute()) {
- throw new ExplicitException("Shell executor does not support "
- + "absolute cache path: " + cacheSpec.getPath());
+ jobLogger.log("Setting up job cache...") ;
+ JobCache cache = new JobCache(cacheHomeDir) {
+
+ @Override
+ protected Map<CacheInstance, String> allocate(CacheAllocationRequest request) {
+ return jobManager.allocateJobCaches(jobToken, request);
}
- }
- jobLogger.log("Allocating job caches...") ;
- Map<CacheInstance, Date> cacheInstances = getCacheInstances(cacheHomeDir);
- Map<CacheInstance, String> cacheAllocations = jobManager.allocateJobCaches(jobToken, new Date(), cacheInstances);
- checkCacheAllocations(cacheHomeDir, cacheAllocations, new Consumer<File>() {
-
+
@Override
- public void accept(File directory) {
- FileUtils.cleanDir(directory);
+ protected void clean(File cacheDir) {
+ FileUtils.cleanDir(cacheDir);
}
- });
-
+ };
+ cache.init(true);
FileUtils.createDir(workspaceDir);
+ cache.installSymbolinks(workspaceDir);
+
jobLogger.log("Copying job dependencies...");
jobContext.copyDependencies(workspaceDir);
@@ -158,24 +151,6 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
throw new RuntimeException(e);
}
- for (Map.Entry<CacheInstance, String> entry: cacheAllocations.entrySet()) {
- if (!PathUtils.isCurrent(entry.getValue())) {
- File sourceDir = entry.getKey().getDirectory(cacheHomeDir);
- File destDir = resolveCachePath(workspaceDir, entry.getValue());
- if (destDir.exists())
- FileUtils.deleteDir(destDir);
- else
- FileUtils.createDir(destDir.getParentFile());
- try {
- Files.createSymbolicLink(destDir.toPath(), sourceDir.toPath());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- throw new ExplicitException("Invalid cache path: " + entry.getValue());
- }
- }
-
Commandline interpreter = commandFacade.getInterpreter();
Map<String, String> environments = new HashMap<>();
environments.put("GIT_HOME", userDir.getAbsolutePath());
@@ -196,7 +171,7 @@ public class ServerShellExecutor extends JobExecutor implements Testable<TestDat
jobLogger.log("Checking out code...");
Commandline git = new Commandline(AppLoader.getInstance(GitConfig.class).getExecutable());
- checkoutFacade.setupWorkingDir(git, workspaceDir, cacheHomeDir, cacheAllocations);
+ checkoutFacade.setupWorkingDir(git, workspaceDir);
Map<String, String> environments = new HashMap<>();
environments.put("HOME", userDir.getAbsolutePath());
diff --git a/server-product/src/test/java/io/onedev/server/product/Test.java b/server-product/src/test/java/io/onedev/server/product/Test.java
index a96bf27a34..3d0dead201 100644
--- a/server-product/src/test/java/io/onedev/server/product/Test.java
+++ b/server-product/src/test/java/io/onedev/server/product/Test.java
@@ -1,9 +1,11 @@
package io.onedev.server.product;
+import java.io.IOException;
+
public class Test {
@org.junit.Test
- public void test() {
+ public void test() throws IOException {
}
}
\ No newline at end of file