The Unified and Autonomous Development Platform
refactor: Introduce CacheEntry class and update cache configuration structure
Robin Shenf9eee98d0d0f
diff --git a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.javaindex 735ce10473..c2b5a68e02 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/BuildSpec.java@@ -2760,4 +2760,47 @@ public class BuildSpec implements Serializable, Validatable {});}+ private static boolean isSetupCacheStep(MappingNode stepNode) {+ var stepType = getStepType(stepNode);+ return "SetupCacheStep".equals(stepType)+ || "RenovateCacheStep".equals(stepType)+ || stepNode.getTag().getValue().equals("!SetupCacheStep")+ || stepNode.getTag().getValue().equals("!RenovateCacheStep");+ }++ @SuppressWarnings("unused")+ private void migrate52(VersionedYamlDoc doc, Stack<Integer> versions) {+ migrateSteps(doc, versions, stepsNode -> {+ for (var stepNodeItem : stepsNode.getValue()) {+ var stepNode = (MappingNode) stepNodeItem;+ if (!isSetupCacheStep(stepNode))+ continue;+ SequenceNode pathsNode = null;+ for (var itStepTuple = stepNode.getValue().iterator(); itStepTuple.hasNext();) {+ var stepTuple = itStepTuple.next();+ var propName = ((ScalarNode) stepTuple.getKeyNode()).getValue();+ if (propName.equals("paths")) {+ pathsNode = (SequenceNode) stepTuple.getValueNode();+ itStepTuple.remove();+ } else if (propName.equals("changeDetectionExcludes")) {+ itStepTuple.remove();+ }+ }+ if (pathsNode != null) {+ List<Node> entryNodes = new ArrayList<>();+ for (var pathNode : pathsNode.getValue()) {+ List<NodeTuple> entryTuples = new ArrayList<>();+ entryTuples.add(new NodeTuple(+ new ScalarNode(Tag.STR, "path"),+ new ScalarNode(Tag.STR, ((ScalarNode) pathNode).getValue())));+ entryNodes.add(new MappingNode(Tag.MAP, entryTuples, FlowStyle.BLOCK));+ }+ stepNode.getValue().add(new NodeTuple(+ new ScalarNode(Tag.STR, "entries"),+ new SequenceNode(Tag.SEQ, entryNodes, FlowStyle.BLOCK)));+ }+ }+ });+ }+}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.java b/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.javanew file mode 100644index 0000000000..939caa4026--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/CacheEntry.java@@ -0,0 +1,61 @@+package io.onedev.server.buildspec.step;++import java.io.Serializable;+import java.util.List;++import javax.validation.constraints.NotEmpty;++import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.k8shelper.CacheEntryFacade;+import io.onedev.server.annotation.Editable;+import io.onedev.server.annotation.Interpolative;+import io.onedev.server.annotation.Path;+import io.onedev.server.buildspec.BuildSpec;++@Editable+public class CacheEntry implements Serializable {++ private static final long serialVersionUID = 1L;++ private String path;++ private String excludes;++ @Editable(order=100, description = """+ Specify cache path. Non-absolute path is considered to be relative to+ <a href='https://docs.onedev.io/concepts#job-workdir' target='_blank'>job working directory</a>.+ Note that shell related executors runs directly on host machine, and only accept relative paths""")+ @Path+ @Interpolative(variableSuggester="suggestVariables")+ @NotEmpty+ public String getPath() {+ return path;+ }++ public void setPath(String path) {+ this.path = path;+ }++ @Editable(order=200, placeholder = "None", description = """+ Optionally specify files or directories relative to cache path to exclude.+ Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.+ Multiple excludes should be separated by space, and single exclude containing space should be quoted""")+ @Interpolative(variableSuggester="suggestVariables")+ public String getExcludes() {+ return excludes;+ }++ public void setExcludes(String excludes) {+ this.excludes = excludes;+ }++ public CacheEntryFacade getFacade() {+ return new CacheEntryFacade(path, excludes);+ }++ @SuppressWarnings("unused")+ private static List<InputSuggestion> suggestVariables(String matchWith) {+ return BuildSpec.suggestVariables(matchWith, true, true, false);+ }++}diff --git a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.javaindex 9d95af6985..0ba38e8105 100644--- a/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java+++ b/server-core/src/main/java/io/onedev/server/buildspec/step/SetupCacheStep.java@@ -6,7 +6,9 @@ import static java.util.stream.Collectors.toList;import java.util.ArrayList;import java.util.List;import java.util.Set;+import java.util.stream.Collectors;+import javax.validation.Valid;import javax.validation.constraints.NotEmpty;import javax.validation.constraints.NotNull;import javax.validation.constraints.Size;@@ -19,7 +21,6 @@ import io.onedev.k8shelper.SetupCacheFacade;import io.onedev.k8shelper.StepFacade;import io.onedev.k8shelper.UploadStrategy;import io.onedev.server.annotation.ChoiceProvider;-import io.onedev.server.annotation.DependsOn;import io.onedev.server.annotation.Editable;import io.onedev.server.annotation.Interpolative;import io.onedev.server.annotation.ProjectChoice;@@ -42,12 +43,10 @@ public class SetupCacheStep extends Step {private String checksumFiles;- private List<String> paths = new ArrayList<>();+ private List<CacheEntry> entries = new ArrayList<>();private UploadStrategy uploadStrategy = UPLOAD_IF_NOT_EXACT_MATCH;- private String changeDetectionExcludes;-private String uploadProjectPath;private String uploadAccessTokenSecret;@@ -85,17 +84,27 @@ public class SetupCacheStep extends Step {this.checksumFiles = checksumFiles;}- @Editable(order=300, name="Cache Paths", description = """- Non-absolute path is considered to be relative to <a href='https://docs.onedev.io/concepts#job-workdir' target='_blank'>job working directory</a>.- Note that shell related executors runs directly on host machine, and only accept relative paths""")- @Interpolative(variableSuggester="suggestVariables")+ @Editable(order=300, name="Cache Entries", description = "Specify cache entries")+ @Valid@Size(min=1, max=100)+ public List<CacheEntry> getEntries() {+ return entries;+ }++ public void setEntries(List<CacheEntry> entries) {+ this.entries = entries;+ }+public List<String> getPaths() {- return paths;+ return getEntries().stream().map(CacheEntry::getPath).collect(Collectors.toList());}public void setPaths(List<String> paths) {- this.paths = paths;+ entries = paths.stream().map(it -> {+ var entry = new CacheEntry();+ entry.setPath(it);+ return entry;+ }).collect(Collectors.toList());}@Editable(order=400, description = """@@ -110,20 +119,6 @@ public class SetupCacheStep extends Step {public void setUploadStrategy(UploadStrategy uploadStrategy) {this.uploadStrategy = uploadStrategy;}-- @Editable(order=425, description = """- Optionally specify files relative to cache path to ignore when detect cache changes.- Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.- Multiple files should be separated by space, and single file containing space should be quoted""")- @Interpolative(variableSuggester="suggestVariables")- @DependsOn(property="uploadStrategy", value="UPLOAD_IF_CHANGED")- public String getChangeDetectionExcludes() {- return changeDetectionExcludes;- }-- public void setChangeDetectionExcludes(String changeDetectionExcludes) {- this.changeDetectionExcludes = changeDetectionExcludes;- }@Editable(order=450, name="Upload to Project", placeholder = "Current project", description = """In case cache needs to be uploaded, this property specifies target project for the upload.@@ -175,7 +170,8 @@ public class SetupCacheStep extends Step {var patterns = PatternSet.parse(checksumFiles);parsedChecksumFiles = Pair.of(patterns.getIncludes(), patterns.getExcludes());}- var cacheConfig = new CacheConfigFacade(key, parsedChecksumFiles, getPaths(), uploadStrategy, changeDetectionExcludes,+ var cacheConfig = new CacheConfigFacade(key, parsedChecksumFiles,+ getEntries().stream().map(CacheEntry::getFacade).collect(toList()), uploadStrategy,uploadProjectPath, uploadAccessToken);return new SetupCacheFacade(cacheConfig);}diff --git a/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java b/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.javaindex 4bf23cea7e..d052a2fff6 100644--- a/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java+++ b/server-core/src/main/java/io/onedev/server/cache/ServerCacheProvisioner.java@@ -3,6 +3,7 @@ package io.onedev.server.cache;import java.io.File;import java.io.IOException;import java.io.InputStream;+import java.util.List;import javax.ws.rs.client.Client;import javax.ws.rs.client.ClientBuilder;@@ -83,7 +84,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {}@Override- protected boolean upload(CacheConfigFacade config, String path, File pathDir) {+ protected boolean upload(CacheConfigFacade config, String path, File pathDir, List<String> excludes) {var key = config.getKey();var checksum = config.getChecksum();@@ -114,7 +115,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {var activeServer = getProjectService().getActiveServer(projectId, true);if (activeServer.equals(getClusterService().getLocalServerAddress())) {getCacheService().uploadCache(projectId, key, checksum, path,- os -> TarUtils.tar(pathDir, os, false));+ os -> TarUtils.tar(pathDir, excludes, os, false));} else {Client client = ClientBuilder.newClient();client.property(ClientProperties.REQUEST_ENTITY_PROCESSING, "CHUNKED");@@ -129,7 +130,7 @@ public abstract class ServerCacheProvisioner extends CacheProvisioner {Invocation.Builder builder = target.request();builder.header(HttpHeaders.AUTHORIZATION,KubernetesHelper.BEARER + " " + getClusterService().getCredential());- StreamingOutput output = os -> TarUtils.tar(pathDir, os, false);+ StreamingOutput output = os -> TarUtils.tar(pathDir, excludes, os, false);try (Response response = builder.post(Entity.entity(output, MediaType.APPLICATION_OCTET_STREAM))) {KubernetesHelper.checkStatus(response);}diff --git a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.javaindex d3b353da60..4735aca29a 100644--- a/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java+++ b/server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java@@ -8934,6 +8934,24 @@ public class DataMigrator {pathElement.addElement("path").setText(path);}}+ var cacheConfigsElement = workspaceSpecElement.element("cacheConfigs");+ if (cacheConfigsElement != null) {+ for (Element cacheConfigElement : cacheConfigsElement.elements()) {+ var pathsElement = cacheConfigElement.element("paths");+ var changeDetectionExcludesElement = cacheConfigElement.element("changeDetectionExcludes");+ if (changeDetectionExcludesElement != null)+ changeDetectionExcludesElement.detach();+ if (pathsElement != null) {+ pathsElement.setName("entries");+ for (Element pathElement : pathsElement.elements()) {+ var path = pathElement.getText();+ pathElement.setName("io.onedev.server.model.support.workspace.spec.CacheEntry");+ pathElement.setText("");+ pathElement.addElement("path").setText(path);+ }+ }+ }+ }}}}diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.javaindex dd8b4ee8d8..586a4ba876 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheConfig.java@@ -6,7 +6,9 @@ import java.io.Serializable;import java.util.ArrayList;import java.util.List;import java.util.Set;+import java.util.stream.Collectors;+import javax.validation.Valid;import javax.validation.constraints.NotEmpty;import javax.validation.constraints.NotNull;import javax.validation.constraints.Size;@@ -16,7 +18,6 @@ import org.apache.commons.lang3.tuple.Pair;import io.onedev.commons.codeassist.InputSuggestion;import io.onedev.k8shelper.CacheConfigFacade;import io.onedev.k8shelper.UploadStrategy;-import io.onedev.server.annotation.DependsOn;import io.onedev.server.annotation.Editable;import io.onedev.server.annotation.Interpolative;import io.onedev.server.annotation.ProjectChoice;@@ -32,12 +33,10 @@ public class CacheConfig implements Serializable {private String checksumFiles;- private List<String> paths = new ArrayList<>();+ private List<CacheEntry> entries = new ArrayList<>();private UploadStrategy uploadStrategy = UPLOAD_IF_NOT_EXACT_MATCH;- private String changeDetectionExcludes;-private String uploadProjectPath;private String uploadAccessToken;@@ -75,19 +74,29 @@ public class CacheConfig implements Serializable {this.checksumFiles = checksumFiles;}- @Editable(order=300, name="Cache Paths", description = """- Non-absolute path is considered to be relative to workspace working directory.- Note that only relative path is accepted if not running in container""")- @Interpolative(variableSuggester="suggestVariables")+ @Editable(order=300, name="Cache Entries", description = "Specify cache entries")+ @Valid@Size(min=1, max=100)+ public List<CacheEntry> getEntries() {+ return entries;+ }++ public void setEntries(List<CacheEntry> entries) {+ this.entries = entries;+ }+public List<String> getPaths() {- return paths;+ return getEntries().stream().map(CacheEntry::getPath).collect(Collectors.toList());}public void setPaths(List<String> paths) {- this.paths = paths;+ entries = paths.stream().map(it -> {+ var entry = new CacheEntry();+ entry.setPath(it);+ return entry;+ }).collect(Collectors.toList());}-+@Editable(order=400, description = """Specify cache upload strategy after build successful. <i>Upload If Not Exact Match</i>means to upload when no cache found with matching key and checksum , and@@ -101,20 +110,6 @@ public class CacheConfig implements Serializable {this.uploadStrategy = uploadStrategy;}- @Editable(order=425, description = """- Optionally specify files relative to cache path to ignore when detect cache changes.- Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.- Multiple files should be separated by space, and single file containing space should be quoted""")- @Interpolative(variableSuggester="suggestVariables")- @DependsOn(property="uploadStrategy", value="UPLOAD_IF_CHANGED")- public String getChangeDetectionExcludes() {- return changeDetectionExcludes;- }-- public void setChangeDetectionExcludes(String changeDetectionExcludes) {- this.changeDetectionExcludes = changeDetectionExcludes;- }-@Editable(order=450, name="Upload to Project", placeholder = "Current project", description = """In case cache needs to be uploaded, this property specifies target project for the upload.Leave empty for current project""")@@ -144,8 +139,9 @@ public class CacheConfig implements Serializable {var patterns = PatternSet.parse(checksumFiles);parsedChecksumFiles = Pair.of(patterns.getIncludes(), patterns.getExcludes());}- return new CacheConfigFacade(key, parsedChecksumFiles, getPaths(), uploadStrategy,- changeDetectionExcludes, uploadProjectPath, uploadAccessToken);+ return new CacheConfigFacade(key, parsedChecksumFiles,+ getEntries().stream().map(CacheEntry::getFacade).collect(Collectors.toList()), uploadStrategy,+ uploadProjectPath, uploadAccessToken);}@SuppressWarnings("unused")diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.javanew file mode 100644index 0000000000..d4d40d4efc--- /dev/null+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/CacheEntry.java@@ -0,0 +1,60 @@+package io.onedev.server.model.support.workspace.spec;++import java.io.Serializable;+import java.util.List;++import javax.validation.constraints.NotEmpty;++import io.onedev.commons.codeassist.InputSuggestion;+import io.onedev.k8shelper.CacheEntryFacade;+import io.onedev.server.annotation.Editable;+import io.onedev.server.annotation.Interpolative;+import io.onedev.server.annotation.Path;+import io.onedev.server.web.util.SuggestionUtils;++@Editable+public class CacheEntry implements Serializable {++ private static final long serialVersionUID = 1L;++ private String path;++ private String excludes;++ @Editable(order=100, description = """+ Specify cache path. Non-absolute path is considered to be relative to working directory.+ Note that shell related executors only accept relative paths""")+ @Path+ @Interpolative(variableSuggester="suggestVariables")+ @NotEmpty+ public String getPath() {+ return path;+ }++ public void setPath(String path) {+ this.path = path;+ }++ @Editable(order=200, placeholder = "None", description = """+ Optionally specify directories or files relative to cache path to exclude.+ Use '**', '*' or '?' for <a href='https://docs.onedev.io/appendix/path-wildcard' target='_blank'>path wildcard match</a>.+ Multiple excludes should be separated by space, and single exclude containing space should be quoted""")+ @Interpolative(variableSuggester="suggestVariables")+ public String getExcludes() {+ return excludes;+ }++ public void setExcludes(String excludes) {+ this.excludes = excludes;+ }++ public CacheEntryFacade getFacade() {+ return new CacheEntryFacade(path, excludes);+ }++ @SuppressWarnings("unused")+ private static List<InputSuggestion> suggestVariables(String matchWith) {+ return SuggestionUtils.suggestWorkspaceVariables(matchWith);+ }++}diff --git a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.javaindex 9f1518c826..a421e7d323 100644--- a/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java+++ b/server-core/src/main/java/io/onedev/server/model/support/workspace/spec/WorkspaceSpec.java@@ -311,8 +311,8 @@ public class WorkspaceSpec implements Serializable, Validatable {Set<String> seenCachePaths = new HashSet<>();for (int i = 0; i < cacheConfigs.size(); i++) {var cacheConfig = cacheConfigs.get(i);- for (int j = 0; j < cacheConfig.getPaths().size(); j++) {- var path = cacheConfig.getPaths().get(j);+ for (int j = 0; j < cacheConfig.getEntries().size(); j++) {+ var path = cacheConfig.getEntries().get(j).getPath();if (path == null || path.isEmpty())continue;if (!path.contains("@")) {diff --git a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.javaindex 415e2d671b..a35b73126b 100644--- a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListEditPanel.java@@ -60,7 +60,7 @@ class CacheConfigListEditPanel extends DrawCardBeanListEditPanel<CacheConfig> {@Overridepublic void populateItem(Item<ICellPopulator<CacheConfig>> cellItem, String componentId, IModel<CacheConfig> rowModel) {- cellItem.add(new Label(componentId, rowModel.getObject().getPaths().size()));+ cellItem.add(new Label(componentId, rowModel.getObject().getEntries().size()));}});diff --git a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.javaindex fce0c04bab..a1b2621a5c 100644--- a/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java+++ b/server-core/src/main/java/io/onedev/server/web/editable/workspacespec/cacheconfig/CacheConfigListViewPanel.java@@ -52,7 +52,7 @@ class CacheConfigListViewPanel extends DrawCardBeanListViewPanel<CacheConfig> {@Overridepublic void populateItem(Item<ICellPopulator<CacheConfig>> cellItem, String componentId, IModel<CacheConfig> rowModel) {- cellItem.add(new Label(componentId, rowModel.getObject().getPaths().size()));+ cellItem.add(new Label(componentId, rowModel.getObject().getEntries().size()));}});diff --git a/server-ee b/server-eeindex 1d71851597..040e574e60 160000--- a/server-ee+++ b/server-ee@@ -1 +1 @@-Subproject commit 1d7185159767dc5bb18d75c13350cbe618849f98+Subproject commit 040e574e605ad4a99afdf79a1ac92758f74c6f89