Skip to content

The Unified and Autonomous Development Platform

refactor: Introduce CacheEntry class and update cache configuration structure

Robin Shenf9eee98d0d0f
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
index 735ce10473..c2b5a68e02 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java
@@ -2760,4 +2760,47 @@ public class BuildSpec implements Serializable, Validatable {
});
}
+ private static boolean isSetupCacheStep(MappingNode stepNode) {
+ var stepType = getStepType(stepNode);
+ return "SetupCacheStep".equals(stepType)
+ || "RenovateCacheStep".equals(stepType)
+ || stepNode.getTag().getValue().equals("!SetupCacheStep")
+ || stepNode.getTag().getValue().equals("!RenovateCacheStep");
+ }
+
+ @SuppressWarnings("unused")
+ private void migrate52(VersionedYamlDoc doc, Stack<Integer> versions) {
+ migrateSteps(doc, versions, stepsNode -> {
+ for (var stepNodeItem : stepsNode.getValue()) {
+ var stepNode = (MappingNode) stepNodeItem;
+ if (!isSetupCacheStep(stepNode))
+ continue;
+ SequenceNode pathsNode = null;
+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {
+ var stepTuple = itStepTuple.next();
+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();
+ if (propName.equals("paths")) {
+ pathsNode = (SequenceNode) stepTuple.getValueNode();
+ itStepTuple.remove();
+ } else if (propName.equals("changeDetectionExcludes")) {
+ itStepTuple.remove();
+ }
+ }
+ if (pathsNode != null) {
+ List<Node> entryNodes = new ArrayList<>();
+ for (var pathNode : pathsNode.getValue()) {
+ List<NodeTuple> entryTuples = new ArrayList<>();
+ entryTuples.add(new NodeTuple(
+ new ScalarNode(Tag.STR, "path"),
+ new ScalarNode(Tag.STR, ((ScalarNode) pathNode).getValue())));
+ entryNodes.add(new MappingNode(Tag.MAP, entryTuples, FlowStyle.BLOCK));
+ }
+ stepNode.getValue().add(new NodeTuple(
+ new ScalarNode(Tag.STR, "entries"),
+ new SequenceNode(Tag.SEQ, entryNodes, FlowStyle.BLOCK)));
+ }
+ }
+ });
+ }
+
}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.java
new file mode 100644
index 0000000000..939caa4026
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.java
@@ -0,0 +1,61 @@
+package io.onedev.server.buildspec.step;
+
+import java.io.Serializable;
+import java.util.List;
+
+import javax.validation.constraints.NotEmpty;
+
+import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.k8shelper.CacheEntryFacade;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.annotation.Path;
+import io.onedev.server.buildspec.BuildSpec;
+
+@Editable
+public class CacheEntry implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String path;
+
+ private String excludes;
+
+ @Editable(order=100, description = """
+ Specify cache path. Non-absolute path is considered to be relative to
+ <a href='https://docs.onedev.io/concepts#job-workdir' target='_blank'>job working directory</a>.
+ Note that shell related executors runs directly on host machine, and only accept relative paths""")
+ @Path
+ @Interpolative(variableSuggester="suggestVariables")
+ @NotEmpty
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ @Editable(order=200, placeholder = "None", description = """
+ Optionally specify files or directories relative to cache path to exclude.
+ Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.
+ Multiple excludes should be separated by space, and single exclude containing space should be quoted""")
+ @Interpolative(variableSuggester="suggestVariables")
+ public String getExcludes() {
+ return excludes;
+ }
+
+ public void setExcludes(String excludes) {
+ this.excludes = excludes;
+ }
+
+ public CacheEntryFacade getFacade() {
+ return new CacheEntryFacade(path, excludes);
+ }
+
+ @SuppressWarnings("unused")
+ private static List<InputSuggestion> suggestVariables(String matchWith) {
+ return BuildSpec.suggestVariables(matchWith, true, true, false);
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
index 9d95af6985..0ba38e8105 100644
--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java
@@ -6,7 +6,9 @@ import static java.util.stream.Collectors.toList;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import javax.validation.Valid;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
@@ -19,7 +21,6 @@ import io.onedev.k8shelper.SetupCacheFacade;
import io.onedev.k8shelper.StepFacade;
import io.onedev.k8shelper.UploadStrategy;
import io.onedev.server.annotation.ChoiceProvider;
-import io.onedev.server.annotation.DependsOn;
import io.onedev.server.annotation.Editable;
import io.onedev.server.annotation.Interpolative;
import io.onedev.server.annotation.ProjectChoice;
@@ -42,12 +43,10 @@ public class SetupCacheStep extends Step {
private String checksumFiles;
- private List<String> paths = new ArrayList<>();
+ private List<CacheEntry> entries = new ArrayList<>();
private UploadStrategy uploadStrategy = UPLOAD_IF_NOT_EXACT_MATCH;
- private String changeDetectionExcludes;
-
private String uploadProjectPath;
private String uploadAccessTokenSecret;
@@ -85,17 +84,27 @@ public class SetupCacheStep extends Step {
this.checksumFiles = checksumFiles;
}
- @Editable(order=300, name="Cache Paths", description = """
- Non-absolute path is considered to be relative to <a href='https://docs.onedev.io/concepts#job-workdir' target='_blank'>job working directory</a>.
- Note that shell related executors runs directly on host machine, and only accept relative paths""")
- @Interpolative(variableSuggester="suggestVariables")
+ @Editable(order=300, name="Cache Entries", description = "Specify cache entries")
+ @Valid
@Size(min=1, max=100)
+ public List<CacheEntry> getEntries() {
+ return entries;
+ }
+
+ public void setEntries(List<CacheEntry> entries) {
+ this.entries = entries;
+ }
+
public List<String> getPaths() {
- return paths;
+ return getEntries().stream().map(CacheEntry::getPath).collect(Collectors.toList());
}
public void setPaths(List<String> paths) {
- this.paths = paths;
+ entries = paths.stream().map(it -> {
+ var entry = new CacheEntry();
+ entry.setPath(it);
+ return entry;
+ }).collect(Collectors.toList());
}
@Editable(order=400, description = """
@@ -110,20 +119,6 @@ public class SetupCacheStep extends Step {
public void setUploadStrategy(UploadStrategy uploadStrategy) {
this.uploadStrategy = uploadStrategy;
}
-
- @Editable(order=425, description = """
- Optionally specify files relative to cache path to ignore when detect cache changes.
- Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.
- Multiple files should be separated by space, and single file containing space should be quoted""")
- @Interpolative(variableSuggester="suggestVariables")
- @DependsOn(property="uploadStrategy", value="UPLOAD_IF_CHANGED")
- public String getChangeDetectionExcludes() {
- return changeDetectionExcludes;
- }
-
- public void setChangeDetectionExcludes(String changeDetectionExcludes) {
- this.changeDetectionExcludes = changeDetectionExcludes;
- }
@Editable(order=450, name="Upload to Project", placeholder = "Current project", description = """
In case cache needs to be uploaded, this property specifies target project for the upload.
@@ -175,7 +170,8 @@ public class SetupCacheStep extends Step {
var patterns = PatternSet.parse(checksumFiles);
parsedChecksumFiles = Pair.of(patterns.getIncludes(), patterns.getExcludes());
}
- var cacheConfig = new CacheConfigFacade(key, parsedChecksumFiles, getPaths(), uploadStrategy, changeDetectionExcludes,
+ var cacheConfig = new CacheConfigFacade(key, parsedChecksumFiles,
+ getEntries().stream().map(CacheEntry::getFacade).collect(toList()), uploadStrategy,
uploadProjectPath, uploadAccessToken);
return new SetupCacheFacade(cacheConfig);
}
diff --git a/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java b/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java
index 4bf23cea7e..d052a2fff6 100644
--- a/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java
+++ b/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java
@@ -3,6 +3,7 @@ package io.onedev.server.cache;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.util.List;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
@@ -83,7 +84,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {
}
@Override
- protected boolean upload(CacheConfigFacade config, String path, File pathDir) {
+ protected boolean upload(CacheConfigFacade config, String path, File pathDir, List<String> excludes) {
var key = config.getKey();
var checksum = config.getChecksum();
@@ -114,7 +115,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {
var activeServer = getProjectService().getActiveServer(projectId, true);
if (activeServer.equals(getClusterService().getLocalServerAddress())) {
getCacheService().uploadCache(projectId, key, checksum, path,
- os -> TarUtils.tar(pathDir, os, false));
+ os -> TarUtils.tar(pathDir, excludes, os, false));
} else {
Client client = ClientBuilder.newClient();
client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");
@@ -129,7 +130,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {
Invocation.Builder builder = target.request();
builder.header(HttpHeaders.AUTHORIZATION,
KubernetesHelper.BEARER + " " + getClusterService().getCredential());
- StreamingOutput output = os -> TarUtils.tar(pathDir, os, false);
+ StreamingOutput output = os -> TarUtils.tar(pathDir, excludes, os, false);
try (Response response = builder.post(Entity.entity(output, MediaType.APPLICATION_OCTET_STREAM))) {
KubernetesHelper.checkStatus(response);
}
diff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
index d3b353da60..4735aca29a 100644
--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
@@ -8934,6 +8934,24 @@ public class DataMigrator {
pathElement.addElement("path").setText(path);
}
}
+ var cacheConfigsElement = workspaceSpecElement.element("cacheConfigs");
+ if (cacheConfigsElement != null) {
+ for (Element cacheConfigElement : cacheConfigsElement.elements()) {
+ var pathsElement = cacheConfigElement.element("paths");
+ var changeDetectionExcludesElement = cacheConfigElement.element("changeDetectionExcludes");
+ if (changeDetectionExcludesElement != null)
+ changeDetectionExcludesElement.detach();
+ if (pathsElement != null) {
+ pathsElement.setName("entries");
+ for (Element pathElement : pathsElement.elements()) {
+ var path = pathElement.getText();
+ pathElement.setName("io.onedev.server.model.support.workspace.spec.CacheEntry");
+ pathElement.setText("");
+ pathElement.addElement("path").setText(path);
+ }
+ }
+ }
+ }
}
}
}
diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java
index dd8b4ee8d8..586a4ba876 100644
--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java
+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java
@@ -6,7 +6,9 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import javax.validation.Valid;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
@@ -16,7 +18,6 @@ import org.apache.commons.lang3.tuple.Pair;
import io.onedev.commons.codeassist.InputSuggestion;
import io.onedev.k8shelper.CacheConfigFacade;
import io.onedev.k8shelper.UploadStrategy;
-import io.onedev.server.annotation.DependsOn;
import io.onedev.server.annotation.Editable;
import io.onedev.server.annotation.Interpolative;
import io.onedev.server.annotation.ProjectChoice;
@@ -32,12 +33,10 @@ public class CacheConfig implements Serializable {
private String checksumFiles;
- private List<String> paths = new ArrayList<>();
+ private List<CacheEntry> entries = new ArrayList<>();
private UploadStrategy uploadStrategy = UPLOAD_IF_NOT_EXACT_MATCH;
- private String changeDetectionExcludes;
-
private String uploadProjectPath;
private String uploadAccessToken;
@@ -75,19 +74,29 @@ public class CacheConfig implements Serializable {
this.checksumFiles = checksumFiles;
}
- @Editable(order=300, name="Cache Paths", description = """
- Non-absolute path is considered to be relative to workspace working directory.
- Note that only relative path is accepted if not running in container""")
- @Interpolative(variableSuggester="suggestVariables")
+ @Editable(order=300, name="Cache Entries", description = "Specify cache entries")
+ @Valid
@Size(min=1, max=100)
+ public List<CacheEntry> getEntries() {
+ return entries;
+ }
+
+ public void setEntries(List<CacheEntry> entries) {
+ this.entries = entries;
+ }
+
public List<String> getPaths() {
- return paths;
+ return getEntries().stream().map(CacheEntry::getPath).collect(Collectors.toList());
}
public void setPaths(List<String> paths) {
- this.paths = paths;
+ entries = paths.stream().map(it -> {
+ var entry = new CacheEntry();
+ entry.setPath(it);
+ return entry;
+ }).collect(Collectors.toList());
}
-
+
@Editable(order=400, description = """
Specify cache upload strategy after build successful. <i>Upload If Not Exact Match</i>
means to upload when no cache found with matching key and checksum , and
@@ -101,20 +110,6 @@ public class CacheConfig implements Serializable {
this.uploadStrategy = uploadStrategy;
}
- @Editable(order=425, description = """
- Optionally specify files relative to cache path to ignore when detect cache changes.
- Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.
- Multiple files should be separated by space, and single file containing space should be quoted""")
- @Interpolative(variableSuggester="suggestVariables")
- @DependsOn(property="uploadStrategy", value="UPLOAD_IF_CHANGED")
- public String getChangeDetectionExcludes() {
- return changeDetectionExcludes;
- }
-
- public void setChangeDetectionExcludes(String changeDetectionExcludes) {
- this.changeDetectionExcludes = changeDetectionExcludes;
- }
-
@Editable(order=450, name="Upload to Project", placeholder = "Current project", description = """
In case cache needs to be uploaded, this property specifies target project for the upload.
Leave empty for current project""")
@@ -144,8 +139,9 @@ public class CacheConfig implements Serializable {
var patterns = PatternSet.parse(checksumFiles);
parsedChecksumFiles = Pair.of(patterns.getIncludes(), patterns.getExcludes());
}
- return new CacheConfigFacade(key, parsedChecksumFiles, getPaths(), uploadStrategy,
- changeDetectionExcludes, uploadProjectPath, uploadAccessToken);
+ return new CacheConfigFacade(key, parsedChecksumFiles,
+ getEntries().stream().map(CacheEntry::getFacade).collect(Collectors.toList()), uploadStrategy,
+ uploadProjectPath, uploadAccessToken);
}
@SuppressWarnings("unused")
diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.java
new file mode 100644
index 0000000000..d4d40d4efc
--- /dev/null
+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.java
@@ -0,0 +1,60 @@
+package io.onedev.server.model.support.workspace.spec;
+
+import java.io.Serializable;
+import java.util.List;
+
+import javax.validation.constraints.NotEmpty;
+
+import io.onedev.commons.codeassist.InputSuggestion;
+import io.onedev.k8shelper.CacheEntryFacade;
+import io.onedev.server.annotation.Editable;
+import io.onedev.server.annotation.Interpolative;
+import io.onedev.server.annotation.Path;
+import io.onedev.server.web.util.SuggestionUtils;
+
+@Editable
+public class CacheEntry implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String path;
+
+ private String excludes;
+
+ @Editable(order=100, description = """
+ Specify cache path. Non-absolute path is considered to be relative to working directory.
+ Note that shell related executors only accept relative paths""")
+ @Path
+ @Interpolative(variableSuggester="suggestVariables")
+ @NotEmpty
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ @Editable(order=200, placeholder = "None", description = """
+ Optionally specify directories or files relative to cache path to exclude.
+ Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.
+ Multiple excludes should be separated by space, and single exclude containing space should be quoted""")
+ @Interpolative(variableSuggester="suggestVariables")
+ public String getExcludes() {
+ return excludes;
+ }
+
+ public void setExcludes(String excludes) {
+ this.excludes = excludes;
+ }
+
+ public CacheEntryFacade getFacade() {
+ return new CacheEntryFacade(path, excludes);
+ }
+
+ @SuppressWarnings("unused")
+ private static List<InputSuggestion> suggestVariables(String matchWith) {
+ return SuggestionUtils.suggestWorkspaceVariables(matchWith);
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java
index 9f1518c826..a421e7d323 100644
--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java
+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java
@@ -311,8 +311,8 @@ public class WorkspaceSpec implements Serializable, Validatable {
Set<String> seenCachePaths = new HashSet<>();
for (int i = 0; i < cacheConfigs.size(); i++) {
var cacheConfig = cacheConfigs.get(i);
- for (int j = 0; j < cacheConfig.getPaths().size(); j++) {
- var path = cacheConfig.getPaths().get(j);
+ for (int j = 0; j < cacheConfig.getEntries().size(); j++) {
+ var path = cacheConfig.getEntries().get(j).getPath();
if (path == null || path.isEmpty())
continue;
if (!path.contains("@")) {
diff --git a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java
index 415e2d671b..a35b73126b 100644
--- a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java
@@ -60,7 +60,7 @@ class CacheConfigListEditPanel extends DrawCardBeanListEditPanel<CacheConfig> {
@Override
public void populateItem(Item<ICellPopulator<CacheConfig>> cellItem, String componentId, IModel<CacheConfig> rowModel) {
- cellItem.add(new Label(componentId, rowModel.getObject().getPaths().size()));
+ cellItem.add(new Label(componentId, rowModel.getObject().getEntries().size()));
}
});
diff --git a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java
index fce0c04bab..a1b2621a5c 100644
--- a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java
+++ b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java
@@ -52,7 +52,7 @@ class CacheConfigListViewPanel extends DrawCardBeanListViewPanel<CacheConfig> {
@Override
public void populateItem(Item<ICellPopulator<CacheConfig>> cellItem, String componentId, IModel<CacheConfig> rowModel) {
- cellItem.add(new Label(componentId, rowModel.getObject().getPaths().size()));
+ cellItem.add(new Label(componentId, rowModel.getObject().getEntries().size()));
}
});
diff --git a/server-ee b/server-ee
index 1d71851597..040e574e60 160000
--- a/server-ee
+++ b/server-ee
@@ -1 +1 @@
-Subproject commit 1d7185159767dc5bb18d75c13350cbe618849f98
+Subproject commit 040e574e605ad4a99afdf79a1ac92758f74c6f89