Apache Lucene open-source search software
Use git-ls-files as the source of file paths for license checks. (#15200)
Dawid Weiss5227fd72bd0c
diff --git a/.hgignore b/.hgignoredeleted file mode 100644index 1e17ec787de..00000000000--- a/.hgignore+++ /dev/null@@ -1,4 +0,0 @@-syntax: glob-*/build/*-*.class-diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.javanew file mode 100644index 00000000000..d00a28afa73--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitFileListValueSource.java@@ -0,0 +1,84 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.gitinfo;++import java.io.BufferedOutputStream;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.List;+import javax.inject.Inject;+import org.gradle.api.GradleException;+import org.gradle.api.provider.ValueSource;+import org.gradle.process.ExecOperations;+import org.slf4j.LoggerFactory;++/** Returns a list of versioned and non-versioned (but not ignored) git files. */+public abstract class GitFileListValueSource+ implements ValueSource<List<String>, GitValueSourceParameters> {++ @Inject+ public abstract ExecOperations getExecOps();++ @Override+ public List<String> obtain() {+ try (var baos = new ByteArrayOutputStream();+ var out = new BufferedOutputStream(baos)) {+ var result =+ getExecOps()+ .exec(+ spec -> {+ spec.setStandardOutput(out);+ spec.setErrorOutput(out);+ spec.setIgnoreExitValue(true);++ getParameters().configure(spec);++ spec.setExecutable(getParameters().getGitExec().get());+ spec.args(+ "ls-files",+ // show cached+ "-c",+ // show others+ "-o",+ // apply .gitignore exclusions+ "--exclude-standard",+ // don't quote paths, 0-terminate strings.+ "-z");+ });+ out.flush();++ // Assume the output is UTF-8, even if it has 0 eols.+ String gitOutput = baos.toString(StandardCharsets.UTF_8);++ if (result.getExitValue() != 0) {+ // Something went wrong. Assume this isn't a git checkout and+ // return an empty list of files.+ LoggerFactory.getLogger(getClass())+ .warn(+ "Failed executing git (exit status: {}). An empty list of versioned files will be used (run 'git init'?).",+ result.getExitValue());++ return List.of();+ }++ return List.of(gitOutput.split("\u0000"));+ } catch (IOException e) {+ throw new GradleException("Errors calling git to fetch local repository status.", e);+ }+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.javaindex f6b3df2ca33..84832c9530c 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoExtension.java@@ -17,6 +17,7 @@package org.apache.lucene.gradle.plugins.gitinfo;import org.gradle.api.file.FileSystemLocation;+import org.gradle.api.provider.ListProperty;import org.gradle.api.provider.MapProperty;import org.gradle.api.provider.Property;@@ -32,4 +33,10 @@ public abstract class GitInfoExtension {* @return Return the location of {@code .git} directory (or file, if worktrees are used).*/public abstract Property<FileSystemLocation> getDotGitDir();++ /**+ * @return Return a set of all versioned and non-versioned files (that are not ignored by {@code+ * .gitignore}).+ */+ public abstract ListProperty<String> getAllNonIgnoredProjectFiles();}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.javaindex a7f64593d77..e38a8d72df4 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoPlugin.java@@ -19,55 +19,68 @@ package org.apache.lucene.gradle.plugins.gitinfo;import java.nio.file.Files;import java.nio.file.Path;import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;+import org.gradle.api.Action;import org.gradle.api.GradleException;import org.gradle.api.Project;import org.gradle.api.file.Directory;+import org.gradle.api.file.FileSystemLocation;+import org.gradle.api.provider.Property;+import org.gradle.api.provider.ValueSourceSpec;public class GitInfoPlugin extends LuceneGradlePlugin {@Overridepublic void apply(Project project) {applicableToRootProjectOnly(project);- var gitInfoProvider =- project- .getProviders()- .of(- GitInfoValueSource.class,- spec -> {- spec.getParameters().getRootProjectDir().set(project.getProjectDir());- });-var gitInfoExtension =project.getExtensions().create(GitInfoExtension.NAME, GitInfoExtension.class);+ var providers = project.getProviders();- gitInfoExtension.getGitInfo().value(gitInfoProvider).finalizeValueOnRead();-- gitInfoExtension- .getDotGitDir()+ Property<FileSystemLocation> dotGitDir = gitInfoExtension.getDotGitDir();+ dotGitDir.convention(- project- .getProviders()- .provider(- () -> {- Directory projectDirectory =- project.getRootProject().getLayout().getProjectDirectory();- Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");- if (!Files.exists(gitLocation)) {- // don't return anything from the provider if we can't locate the .git- // folder. This will result in the property returning false from isPresent.- return null;- }+ providers.provider(+ () -> {+ Directory projectDirectory =+ project.getRootProject().getLayout().getProjectDirectory();+ Path gitLocation = projectDirectory.getAsFile().toPath().resolve(".git");+ if (!Files.exists(gitLocation)) {+ // don't return anything from the provider if we can't locate the .git+ // folder. This will result in the property returning false from isPresent.+ return null;+ }++ if (Files.isDirectory(gitLocation)) {+ return projectDirectory.dir(".git");+ } else if (Files.isRegularFile(gitLocation)) {+ return projectDirectory.file(".git");+ } else {+ throw new GradleException(+ "Panic, .git location not a directory or file: "+ + gitLocation.toAbsolutePath());+ }+ }))+ .finalizeValueOnRead();- if (Files.isDirectory(gitLocation)) {- return projectDirectory.dir(".git");- } else if (Files.isRegularFile(gitLocation)) {- return projectDirectory.file(".git");- } else {- throw new GradleException(- "Panic, .git location not a directory or file: "- + gitLocation.toAbsolutePath());- }- }))+ var gitExec =+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class).externalTool("git");++ Action<ValueSourceSpec<GitValueSourceParameters>> configureGitParams =+ spec -> {+ var params = spec.getParameters();+ params.getRootProjectDir().set(project.getProjectDir());+ params.getGitExec().set(gitExec);+ params.getDotDir().set(dotGitDir);+ };++ gitInfoExtension+ .getGitInfo()+ .value(providers.of(GitInfoValueSource.class, configureGitParams)).finalizeValueOnRead();++ gitInfoExtension+ .getAllNonIgnoredProjectFiles()+ .value(providers.of(GitFileListValueSource.class, configureGitParams));}}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.javaindex cd1045765b6..0f1cae5a2c7 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitInfoValueSource.java@@ -25,19 +25,12 @@ import java.util.TreeMap;import java.util.regex.Matcher;import java.util.regex.Pattern;import javax.inject.Inject;-import org.apache.tools.ant.taskdefs.condition.Os;import org.gradle.api.GradleException;-import org.gradle.api.file.DirectoryProperty;import org.gradle.api.provider.ValueSource;-import org.gradle.api.provider.ValueSourceParameters;-import org.gradle.api.services.BuildServiceParameters;import org.gradle.process.ExecOperations;public abstract class GitInfoValueSource- implements ValueSource<Map<String, String>, GitInfoValueSource.Parameters> {- public abstract static class Parameters implements BuildServiceParameters, ValueSourceParameters {- public abstract DirectoryProperty getRootProjectDir();- }+ implements ValueSource<Map<String, String>, GitValueSourceParameters> {@Injectpublic abstract ExecOperations getExecOps();@@ -54,9 +47,9 @@ public abstract class GitInfoValueSourcespec.setErrorOutput(out);spec.setIgnoreExitValue(true);- spec.setWorkingDir(- getParameters().getRootProjectDir().getAsFile().get().getAbsolutePath());- spec.setExecutable(Os.isFamily(Os.FAMILY_WINDOWS) ? "git.exe" : "git");+ getParameters().configure(spec);++ spec.setExecutable(getParameters().getGitExec().get());spec.args("status", "--porcelain=v2", "--branch");});out.flush();diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.javanew file mode 100644index 00000000000..49b57265f8e--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/gitinfo/GitValueSourceParameters.java@@ -0,0 +1,39 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.gitinfo;++import org.gradle.api.file.DirectoryProperty;+import org.gradle.api.file.FileSystemLocation;+import org.gradle.api.provider.Property;+import org.gradle.api.provider.ValueSourceParameters;+import org.gradle.api.services.BuildServiceParameters;+import org.gradle.process.ExecSpec;++public abstract class GitValueSourceParameters+ implements BuildServiceParameters, ValueSourceParameters {+ public abstract DirectoryProperty getRootProjectDir();++ public abstract Property<String> getGitExec();++ public abstract Property<FileSystemLocation> getDotDir();++ void configure(ExecSpec spec) {+ String workDir = getRootProjectDir().getAsFile().get().getAbsolutePath();+ spec.setWorkingDir(workDir);+ spec.environment("GIT_DIR", getDotDir().get().getAsFile().getAbsolutePath());+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.javaindex 628c4904cda..224e343ab89 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java@@ -16,8 +16,13 @@*/package org.apache.lucene.gradle.plugins.licenses;+import java.io.File;+import java.util.List;import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtension;+import org.apache.tools.ant.types.selectors.SelectorUtils;import org.gradle.api.Project;+import org.gradle.api.specs.Spec;/** This configures ASL and other license checks. */public class CheckLicensesPlugin extends LuceneGradlePlugin {@@ -48,79 +53,96 @@ public class CheckLicensesPlugin extends LuceneGradlePlugin {private void configureCheckLicenses(CheckLicensesTask task) {Project project = task.getProject();- task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));- task.getFiles()- .from(- project.fileTree(- ".",- tree -> {- // Exclude build outputs, ide files, .git.- tree.exclude(".git");- tree.exclude(".idea");- tree.exclude(".muse");- tree.exclude("**/build/**");- tree.exclude("**/.gradle");+ assert project.getRootProject() == project;+ var allNonIgnoredFiles =+ project.getExtensions().getByType(GitInfoExtension.class).getAllNonIgnoredProjectFiles();- // Exclude generated stuff.- tree.exclude("**/src/generated/**");+ task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));- // Exclude github stuff (templates, workflows).- tree.exclude(".github");+ // Build a list of files excluded from the license check. Just reuse ant's glob patterns.+ var rootDir = getProjectRootPath(project);+ var excludedPaths =+ List.of(+ // Ignore binary files. Previously we used apache rat, which had a 'binary guesser'+ // but it's faster to just exclude by name (rather than scan the file).+ "**/*.adoc",+ "**/*.bin",+ "**/*.brk",+ "**/*.bz2",+ "**/*.dat",+ "**/*.gif",+ "**/*.gz",+ "**/*.png",+ "**/*.svg",+ "**/*.xls",+ "**/*.zip",- // do not let RAT attempt to scan a python venv, it gets lost and confused...- tree.exclude("**/.venv/**");+ // Ignore build infrastructure and misc utility files.+ ".asf.yaml",+ ".dir-locals.el",+ ".editorconfig",+ ".git-blame-ignore-revs",+ ".gitattributes",+ ".github/**",+ ".gitignore",+ ".lift.toml",+ ".vscode/**",+ "LICENSE.txt",+ "NOTICE.txt",+ "build-options.properties",+ "dev-tools/**",+ "gradle/**",+ "help/*.txt",+ "lucene/licenses/*",+ "versions.lock",- // apache rat has a 'binary guesser'... I don't think this needs to be done at all- // -- // just exclude binaries here.- tree.exclude("**/*.dat");- tree.exclude("**/*.brk");- tree.exclude("**/*.gz");- tree.exclude("**/*.bin");- tree.exclude("**/*.bz2");- tree.exclude("**/*.gif");- tree.exclude("**/*.svg");- tree.exclude("lucene/analysis/smartcn/src/**/*.mem");+ // Ignore resources in source folders, also generated resources.+ "**/src/**/*.txt",+ "**/src/**/*.properties",+ "**/src/**/*.utf8",+ "**/src/generated/**",- // Only check these selected file patterns as folks have various .gitignore-d- // resources generated by IDEs, etc.- tree.include("**/*.gradle");- tree.include("**/*.xml");- tree.include("**/*.md");- tree.include("**/*.py");- tree.include("**/*.sh");- tree.include("**/*.bat");+ // Ignore other binary resources within sources. Try to be+ // specific here.+ "build-tools/build-infra-shadow/src/java/keep.me",+ "lucene/CHANGES.txt",+ "lucene/analysis.tests/src/**/*.aff",+ "lucene/analysis.tests/src/**/*.dic",+ "lucene/analysis/common/src/**/*.aff",+ "lucene/analysis/common/src/**/*.dic",+ "lucene/analysis/common/src/**/*.good",+ "lucene/analysis/common/src/**/*.htm*",+ "lucene/analysis/common/src/**/*.rslp",+ "lucene/analysis/common/src/**/*.sug",+ "lucene/analysis/common/src/**/*.wrong",+ "lucene/analysis/icu/src/**/utr30.nrm",+ "lucene/analysis/kuromoji/src/**/bocchan.utf-8",+ "lucene/analysis/morfologik/src/**/*.dict",+ "lucene/analysis/morfologik/src/**/*.info",+ "lucene/analysis/morfologik/src/**/*.input",+ "lucene/analysis/opennlp/src/**/en-test-lemmas.dict",+ "lucene/analysis/smartcn/src/**/*.mem",+ "lucene/analysis/stempel/src/**/*.tbl",+ "lucene/benchmark/.gitignore",+ "lucene/demo/src/**/knn-token-vectors",+ "lucene/luke/src/**/ElegantIcons.ttf",+ "lucene/test-framework/src/**/europarl.lines.txt.seek",- // Include selected patterns from any source folders.- tree.include("**/src/**");- tree.exclude("**/src/**/*.png");- tree.exclude("**/src/**/*.txt");- tree.exclude("**/src/**/*.zip");- tree.exclude("**/src/**/*.properties");- tree.exclude("**/src/**/*.utf8");+ // these may require a review, actually?+ "lucene/queryparser/docs/**",+ "lucene/benchmark/conf/*",+ "lucene/benchmark/README.enwiki");- // project-specific exclusions.- tree.exclude("build-tools/build-infra-shadow/src/java/keep.me");- tree.exclude("lucene/analysis/icu/src/**/utr30.nrm");- tree.exclude("lucene/analysis/kuromoji/src/**/bocchan.utf-8");- tree.exclude("lucene/analysis/morfologik/src/**/*.info");- tree.exclude("lucene/analysis/morfologik/src/**/*.input");- tree.exclude("lucene/analysis/morfologik/src/**/*.dict");- tree.exclude("lucene/analysis/stempel/src/**/*.tbl");- tree.exclude("lucene/analysis/opennlp/src/**/en-test-lemmas.dict");- tree.exclude("lucene/demo/src/**/knn-token-vectors");- tree.exclude("lucene/test-framework/src/**/europarl.lines.txt.seek");- tree.exclude("lucene/analysis/common/src/**/*.aff");- tree.exclude("lucene/analysis/common/src/**/*.dic");- tree.exclude("lucene/analysis/common/src/**/*.good");- tree.exclude("lucene/analysis/common/src/**/*.sug");- tree.exclude("lucene/analysis/common/src/**/*.wrong");- tree.exclude("lucene/analysis/common/src/**/*.rslp");- tree.exclude("lucene/analysis/common/src/**/*.htm*");- tree.exclude("lucene/analysis.tests/src/**/*.aff");- tree.exclude("lucene/analysis.tests/src/**/*.dic");- // Luke has an embedded ElegantIcons font (MIT licensed).- tree.exclude("lucene/luke/src/**/ElegantIcons.ttf");- }));+ // I thought it'd be possible to somehow precompile those glob filters but apparently not.+ // I guess it's fine if the list of patterns and files is of reasonable size.+ Spec<File> maybeExcludeKnownExceptions =+ file -> {+ // relativize from root project directory. No need to replace path separators as ant is+ // os-agnostic here.+ var filePath = rootDir.relativize(file.toPath()).toString();+ return excludedPaths.stream()+ .noneMatch(pattern -> SelectorUtils.match(pattern, filePath));+ };+ task.getFiles().from(project.files(allNonIgnoredFiles).filter(maybeExcludeKnownExceptions));}}diff --git a/build.gradle b/build.gradleindex 926c584bee7..b483f83f1db 100644--- a/build.gradle+++ b/build.gradle@@ -1,3 +1,4 @@+import org.apache.lucene.gradle.plugins.gitinfo.GitInfoExtensionimport org.apache.lucene.gradle.plugins.licenses.CheckLicensesTask/*