Apache Lucene open-source search software
Move more gradle plugins to java. Remove Apache Rat dependency. (#15195)
Dawid Weiss79a98b4c8fe5
diff --git a/build-tools/build-infra-shadow/build.gradle b/build-tools/build-infra-shadow/build.gradleindex 845a7c51d43..c6291c69624 100644--- a/build-tools/build-infra-shadow/build.gradle+++ b/build-tools/build-infra-shadow/build.gradle@@ -44,7 +44,6 @@ dependencies {implementation localGroovy()implementation deps.commons.codecimplementation deps.randomizedtesting.runner- implementation deps.ratimplementation deps.zstdimplementation deps.gjfdiff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradleindex 6d2a390976a..e6a4c450411 100644--- a/build-tools/build-infra/build.gradle+++ b/build-tools/build-infra/build.gradle@@ -75,7 +75,6 @@ dependencies {implementation localGroovy()implementation deps.commons.codecimplementation deps.randomizedtesting.runner- implementation deps.ratimplementation deps.zstdimplementation deps.flexmark.core@@ -91,7 +90,6 @@ dependencies {implementation plugin(deps.plugins.carrotsearch.dependencychecks)implementation plugin(deps.plugins.forbiddenapis)implementation plugin(deps.plugins.spotless)- implementation plugin(deps.plugins.owasp.dependencycheck)implementation plugin(deps.plugins.undercouch.download)implementation plugin(deps.plugins.errorprone)implementation plugin(deps.plugins.jacocolog)diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradledeleted file mode 100644index 4de4f552071..00000000000--- a/build-tools/build-infra/src/main/groovy/lucene.java.show-failed-tests-at-end.gradle+++ /dev/null@@ -1,93 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import java.util.regex.Pattern-import org.apache.lucene.gradle.plugins.java.ErrorReportingTestListener--// Display all failed tests at the end of the build.--if (project != project.rootProject) {- throw new GradleException("Applicable to rootProject only: " + project.path)-}--def failedTests = []--allprojects {- plugins.withType(JavaPlugin).configureEach {- tasks.withType(Test).configureEach { Test task ->- def testOutputsDir = task.ext.testOutputsDir- def reproLine = ErrorReportingTestListener.getReproLineOptions(task)-- afterTest { desc, result ->- if (result.resultType == TestResult.ResultType.FAILURE) {- // check if it's a constructor or a before/after class hook that failed.- def qTestName- if (desc.name == "classMethod") {- qTestName = desc.className- } else {- qTestName = "${desc.className}.${desc.name}"- }-- def randomizationParameters = ""- def p = Pattern.compile(/.+ (?<params>[{].*[}])$/)- def matcher = p.matcher(qTestName)- if (matcher.matches()) {- randomizationParameters = matcher.group("params")- qTestName = qTestName.replace(randomizationParameters, "").trim()- }-- failedTests << [- "name" : qTestName,- "randomizationParameters": randomizationParameters,- "project" : "${test.project.path}",- "output" : file("${testOutputsDir}/${ErrorReportingTestListener.getOutputLogName(desc.parent)}"),- "reproduce": "gradlew ${project.path}:test --tests \"${qTestName}\" ${reproLine}"- ]- }- }-- afterSuite { desc, result ->- if (result.exceptions) {- failedTests << [- "name" : "${desc.name}",- "project" : "${test.project.path}",- "output" : file("${testOutputsDir}/${ErrorReportingTestListener.getOutputLogName(desc)}"),- "reproduce": "gradlew ${project.path}:test --tests \"${desc.name}\" ${reproLine}"- ]- }- }- }- }-}--gradle.buildFinished { result ->- if (failedTests) {- def formatted = failedTests- .sort { a, b -> b.project.compareTo(a.project) }- .collect { e ->- String.format(Locale.ROOT,- " - %s (%s)%s\n Test output: %s\n Reproduce with: %s\n",- e.name, e.project,- e.containsKey("randomizationParameters") &&- !e.randomizationParameters.isBlank() ? "\n Context parameters: ${e.randomizationParameters}" : "",- e.output, e.reproduce)- }- .join("\n")-- logger.error("\nERROR: The following {} failed:\n\n{}", failedTests.size() == 1 ? "test has" : "tests have", formatted)- }-}diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.slowest-tests-at-end.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.slowest-tests-at-end.gradledeleted file mode 100644index a6f1750a425..00000000000--- a/build-tools/build-infra/src/main/groovy/lucene.java.slowest-tests-at-end.gradle+++ /dev/null@@ -1,89 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--// Add test duration summary at the end of the build.--if (project != project.rootProject) {- throw new GradleException("Applicable to rootProject only: " + project.path)-}--def allTests = []-def allSuites = []--allprojects {- plugins.withType(JavaPlugin).configureEach {- Provider<Boolean> slowestTestsOption = buildOptions.addBooleanOption("tests.slowestTests", "Print the summary of the slowest tests.", true)- int slowestTestsMinTime = buildOptions.addIntOption("tests.slowestTests.minTime", "Minimum test time to consider a test slow (millis).", 500).get()- Provider<Boolean> slowestSuitesOption = buildOptions.addBooleanOption("tests.slowestSuites", "Print the summary of the slowest suites.", true)- int slowestSuitesMinTime = buildOptions.addIntOption("tests.slowestSuites.minTime", "Minimum suite time to consider a suite slow (millis).", 1000).get()-- tasks.withType(Test).configureEach { task ->- if (slowestTestsOption.get()) {- afterTest { desc, result ->- def duration = (result.getEndTime() - result.getStartTime())- if (duration >= slowestTestsMinTime) {- allTests << [- name : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.path})",- duration: duration- ]- }- }- }-- if (slowestSuitesOption.get()) {- afterSuite { desc, result ->- // Gradle reports runner times as well, omit anything that isn't attached to a concrete class.- if (desc.className != null) {- def duration = (result.getEndTime() - result.getStartTime())- if (duration >= slowestSuitesMinTime) {- allSuites << [- name : "${desc.className.replaceAll('.+\\.', "")} (${project.path})",- duration: duration- ]- }- }- }- }- }- }-}--gradle.buildFinished { result ->- if (result.getFailure() == null) {- if (allTests) {- def slowest = allTests- .sort { a, b -> b.duration.compareTo(a.duration) }- .take(10)- .collect { e -> String.format(Locale.ROOT, "%5.2fs %s", e.duration / 1000d, e.name) }-- if (slowest) {- logger.lifecycle("The slowest tests during this run:\n " + slowest.join("\n "))- }- }-- if (allSuites) {- def slowest = allSuites- .sort { a, b -> b.duration.compareTo(a.duration) }- .take(10)- .collect { e -> String.format(Locale.ROOT, "%5.2fs %s", e.duration / 1000d, e.name) }-- if (slowest) {- logger.lifecycle("The slowest suites during this run:\n " + slowest.join("\n "))- }- }- }-}diff --git a/build-tools/build-infra/src/main/groovy/lucene.validation.owasp.gradle b/build-tools/build-infra/src/main/groovy/lucene.validation.owasp.gradledeleted file mode 100644index 32c7c628366..00000000000--- a/build-tools/build-infra/src/main/groovy/lucene.validation.owasp.gradle+++ /dev/null@@ -1,64 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */---// This adds OWASP vulnerability validation of project dependencies--if (project != project.rootProject) {- throw new GradleException("Applicable to rootProject only: " + project.path)-}--// If -Pvalidation.owasp=true is set the validation will also run as part of the check task.-Provider<Boolean> owaspOption = buildOptions.addBooleanOption("validation.owasp",- "Enable OWASP vulnerability check for dependencies.", false)--Provider<Integer> owaspThresholdOption = buildOptions.addIntOption("validation.owasp.threshold",- "Owasp validation threshold to fail the build, if exceeded.", 7)--Provider<String> owaspApiKey = buildOptions.addOption("validation.owasp.apikey",- "Owasp validation API key (https://nvd.nist.gov/developers/request-an-api-key).")--if (owaspOption.get() || gradle.startParameter.taskNames.contains("owasp")) {- plugins.apply(deps.plugins.owasp.dependencycheck.get().pluginId)-- dependencyCheck {- failBuildOnCVSS = owaspThresholdOption.get()- formats = ['HTML', 'JSON']- skipProjects = []- skipConfigurations = ['unifiedClasspath']- suppressionFile = rootProject.layout.projectDirectory.file("gradle/validation/owasp/exclusions.xml")-- if (owaspApiKey.isPresent()) {- nvd.apiKey = owaspApiKey.get()- }- }-- def owaspTask = tasks.register("owasp", {- group = "Verification"- description = "Check project dependencies against OWASP vulnerability database."- dependsOn "dependencyCheckAggregate"- })-- // Unless explicitly enabled, do not attach owasp to check. It has a large download- // footprint and takes a significant amount of time. This should be enabled for- // nightly CI runs only, I think.- if (owaspOption.get()) {- tasks.named("check").configure {- dependsOn owaspTask- }- }-}diff --git a/build-tools/build-infra/src/main/groovy/lucene.validation.rat-sources.gradle b/build-tools/build-infra/src/main/groovy/lucene.validation.rat-sources.gradledeleted file mode 100644index c9817a054ce..00000000000--- a/build-tools/build-infra/src/main/groovy/lucene.validation.rat-sources.gradle+++ /dev/null@@ -1,256 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import groovy.xml.NamespaceBuilder--if (project != project.rootProject) {- throw new GradleException("Applicable to rootProject only: " + project.path)-}--// Configure the rat validation task and all scanned directories.-allprojects {- configurations {- ratDeps- }-- dependencies {- ratDeps deps.rat- }-- def ratTask = tasks.register("rat", RatTask, {- group = 'Verification'- description = 'Runs Apache Rat checks.'-- dependsOn configurations.ratDeps-- def defaultScanFileTree = project.fileTree(projectDir, {- // Don't check under the project's build folder.- exclude project.layout.buildDirectory.asFile.get().name-- // Exclude any generated stuff.- exclude "src/generated"-- // Don't check any of the subprojects - they have their own rat tasks.- if (project.path == ":") {- exclude "lucene"- exclude "**/build"- include "build-tools/build-infra/**"- } else {- exclude subprojects.collect { it.projectDir.name }- }-- // At the module scope we only check selected file patterns as folks have various .gitignore-d resources- // generated by IDEs, etc.- include "**/*.gradle"- include "**/*.xml"- include "**/*.md"- include "**/*.py"- include "**/*.sh"- include "**/*.bat"-- // Any .gradle folder in composite or top-level project.- exclude "**/.gradle/**"-- // Include selected patterns from any source folders. We could make this- // relative to source sets but it seems to be of little value - all our source sets- // live under 'src' anyway.- include "src/**"- exclude "src/**/*.png"- exclude "src/**/*.txt"- exclude "src/**/*.zip"- exclude "src/**/*.properties"- exclude "src/**/*.utf8"-- // Conditionally apply module-specific patterns. We do it here instead- // of reconfiguring each project because the provider can be made lazy- // and it's easier to manage this way.- switch (project.path) {- case ":":- include "gradlew"- include "gradlew.bat"- exclude ".gradle"- exclude ".idea"- exclude ".muse"- exclude ".git"-- // Exclude github stuff (templates, workflows).- exclude ".github"-- // do not let RAT attempt to scan a python venv, it gets lost and confused...- exclude "dev-tools/scripts/.venv/**"- break-- case ":lucene:analysis:morfologik":- exclude "src/**/*.info"- exclude "src/**/*.input"- break-- case ":lucene:analysis:opennlp":- exclude "src/**/en-test-lemmas.dict"- break-- case ":lucene:demo":- exclude "src/**/knn-token-vectors"- break-- case ":lucene:test-framework":- exclude "src/**/europarl.lines.txt.seek"- break-- case ":lucene:analysis:common":- case ":lucene:analysis.tests":- exclude "src/**/*.aff"- exclude "src/**/*.dic"- exclude "src/**/*.good"- exclude "src/**/*.sug"- exclude "src/**/*.wrong"- exclude "src/**/charfilter/*.htm*"- exclude "src/**/*LuceneResourcesWikiPage.html"- exclude "src/**/*.rslp"- break-- case ":lucene:build-tools:build-infra-shadow":- exclude "src/**/keep.me"- break-- case ":lucene:benchmark":- exclude "data/"- break- }- })- inputFileTrees.add(defaultScanFileTree)- })-- tasks.named("check").configure {- dependsOn ratTask- }-}--/**- * An Apache RAT adapter that validates whether files contain acceptable licenses.- */-@CacheableTask-class RatTask extends DefaultTask {-- @InputFiles- @PathSensitive(PathSensitivity.RELATIVE)- @IgnoreEmptyDirectories- final ListProperty<ConfigurableFileTree> inputFileTrees = project.objects.listProperty(ConfigurableFileTree)-- @OutputFile- final RegularFileProperty xmlReport = project.objects.fileProperty().convention(- project.layout.buildDirectory.file("rat/rat-report.xml"))-- def generateReport(File reportFile) {- // Set up ant rat task.- def ratClasspath = project.configurations.ratDeps.asPath- ant.setLifecycleLogLevel(AntBuilder.AntMessagePriority.ERROR)- ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', classpath: ratClasspath)-- // Collect all output files for debugging.- String inputFileList = inputFileTrees.get().collectMany { fileTree ->- fileTree.asList()- }.sort().join("\n")- project.file(reportFile.path.replaceAll('.xml$', '-filelist.txt')).setText(inputFileList, "UTF-8")-- // Run rat via ant.- ant.report(format: 'xml', reportFile: reportFile, addDefaultLicenseMatchers: true) {- // Pass all gradle file trees to the ant task (Gradle's internal adapters are used).- inputFileTrees.get().each { fileTree ->- fileTree.addToAntBuilder(ant, 'resources', FileCollection.AntType.ResourceCollection)- }-- // BSD 4-clause stuff (is disallowed below)- substringMatcher(licenseFamilyCategory: "BSD4 ", licenseFamilyName: "Original BSD License (with advertising clause)") {- pattern(substring: "All advertising materials")- }-- // BSD-like stuff- substringMatcher(licenseFamilyCategory: "BSD ", licenseFamilyName: "Modified BSD License") {- // brics automaton- pattern(substring: "Copyright (c) 2001-2009 Anders Moeller")- // snowball- pattern(substring: "Copyright (c) 2001, Dr Martin Porter")- // UMASS kstem- pattern(substring: "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS")- // Egothor- pattern(substring: "Egothor Software License version 1.00")- // JaSpell- pattern(substring: "Copyright (c) 2005 Bruno Martins")- // d3.js- pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS")- // highlight.js- pattern(substring: "THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS")- }-- // MIT-like- substringMatcher(licenseFamilyCategory: "MIT ", licenseFamilyName:"Modified BSD License") {- // ICU license- pattern(substring: "Permission is hereby granted, free of charge, to any person obtaining a copy")- // ui-grid- pattern(substring: " ; License: MIT")- }-- // Apache- substringMatcher(licenseFamilyCategory: "AL ", licenseFamilyName: "Apache") {- pattern(substring: "Licensed to the Apache Software Foundation (ASF) under")- // this is the old - school one under some files- pattern(substring: 'Licensed under the Apache License, Version 2.0 (the "License")')- }-- substringMatcher(licenseFamilyCategory: "GEN ", licenseFamilyName: "Generated") {- // svg files generated by gnuplot- pattern(substring: "Produced by GNUPLOT")- // snowball stemmers generated by snowball compiler- pattern(substring: "Generated by Snowball")- }-- approvedLicense(familyName: "Apache")- approvedLicense(familyName: "The MIT License")- approvedLicense(familyName: "Modified BSD License")- approvedLicense(familyName: "Generated")- }- }-- def printUnknownFiles(File reportFile) {- def ratXml = new XmlParser().parse(reportFile)- def errors = []- ratXml.resource.each { resource ->- if (resource.'license-approval'.@name[0] == "false") {- errors << "Unknown license: ${resource.@name}"- }- }- if (errors) {- throw new GradleException("Found " + errors.size() + " file(s) with errors:\n" +- errors.collect{ msg -> " - ${msg}" }.join("\n"))- }- }-- @TaskAction- def execute() {- def origEncoding = System.getProperty("file.encoding")- try {- File reportFile = xmlReport.get().asFile- generateReport(reportFile)- printUnknownFiles(reportFile)- } finally {- if (System.getProperty("file.encoding") != origEncoding) {- throw new GradleException("Something is wrong: Apache RAT changed file.encoding to ${System.getProperty('file.encoding')}?")- }- }- }-}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.javaindex 89967f68ac8..b2d374bff5a 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/HacksPlugin.java@@ -24,7 +24,6 @@ import java.nio.file.Files;import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;import org.apache.lucene.gradle.plugins.java.EcjLintPlugin;import org.apache.lucene.gradle.plugins.misc.CheckGradlewScriptsTweakedPlugin;-import org.apache.lucene.gradle.plugins.spotless.ValidateSourcePatternsPlugin;import org.gradle.api.Project;/** This applies various odd hacks that we probably should not need. */@@ -56,8 +55,7 @@ public class HacksPlugin extends LuceneGradlePlugin {task -> {var taskName = task.getName();return taskName.startsWith(EcjLintPlugin.TASK_PREFIX)- || taskName.equals(CheckGradlewScriptsTweakedPlugin.TASK_NAME)- || taskName.equals(ValidateSourcePatternsPlugin.TASK_NAME);+ || taskName.equals(CheckGradlewScriptsTweakedPlugin.TASK_NAME);}).configureEach(task -> {diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.javaindex 27b13c88447..26b1de13a78 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/help/BuildOptionGroupsPlugin.java@@ -53,9 +53,6 @@ public class BuildOptionGroupsPlugin implements Plugin<Project> {"hunspell.corpora","hunspell.dictionaries","hunspell.repo.path",- "validation.owasp",- "validation.owasp.apikey",- "validation.owasp.threshold","tests.linedocsfile","tests.LUCENE_VERSION","tests.bwcdir"));diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.javaindex 0d5b50cc691..dedb8499df4 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/JavaProjectConventionsPlugin.java@@ -54,5 +54,7 @@ public class JavaProjectConventionsPlugin extends LuceneGradlePlugin {plugins.apply(CodeProfilingPlugin.class);plugins.apply(FailOnNoMatchingFilteredTestsPlugin.class);plugins.apply(CodeCoveragePlugin.class);+ plugins.apply(ShowSlowestTestsAtEndPlugin.class);+ plugins.apply(ShowFailedTestsAtEndPlugin.class);}}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ProfileResults.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ProfileResults.javaindex e4073c9e56f..802cef5f6d5 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ProfileResults.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ProfileResults.java@@ -85,7 +85,8 @@ public class ProfileResults {* Driver method, for testing standalone.** <pre>- * java -Dtests.profile.count=5 buildSrc/src/main/java/org/apache/lucene/gradle/ProfileResults.java \+ * java -Dtests.profile.count=5 \+ * build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ProfileResults.java \* ./lucene/core/build/tmp/tests-cwd/somefile.jfr ...* </pre>*/diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.javanew file mode 100644index 00000000000..a44e56bcad3--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowFailedTestsAtEndPlugin.java@@ -0,0 +1,211 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.java;++import java.io.File;+import java.util.Locale;+import java.util.concurrent.ConcurrentLinkedQueue;+import java.util.regex.Pattern;+import java.util.stream.Collectors;+import javax.inject.Inject;+import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.gradle.api.Project;+import org.gradle.api.file.DirectoryProperty;+import org.gradle.api.logging.Logger;+import org.gradle.api.logging.Logging;+import org.gradle.api.plugins.JavaPlugin;+import org.gradle.api.provider.Provider;+import org.gradle.api.services.BuildService;+import org.gradle.api.services.BuildServiceParameters;+import org.gradle.api.tasks.testing.Test;+import org.gradle.api.tasks.testing.TestDescriptor;+import org.gradle.api.tasks.testing.TestListener;+import org.gradle.api.tasks.testing.TestResult;++/** Display any failing tests, along with their reproduce-line at the end of the build. */+public class ShowFailedTestsAtEndPlugin extends LuceneGradlePlugin {+ private static final String SERVICE_NAME = "failingTestsTrackingService";++ public abstract static class RootExtPlugin extends LuceneGradlePlugin {+ @Override+ public void apply(Project project) {+ applicableToRootProjectOnly(project);++ // Register the shared build service that will do the end-of-build check+ project+ .getGradle()+ .getSharedServices()+ .registerIfAbsent(SERVICE_NAME, FailingTestsReportingService.class, _ -> {});+ }+ }++ public record Entry(+ String name,+ String randomizationParameters,+ String projectPath,+ String reproLine,+ File testOutput) {}++ @Override+ public void apply(Project project) {+ requiresAppliedPlugin(project, JavaPlugin.class);++ if (project != project.getRootProject()) {+ project.getRootProject().getPlugins().apply(RootExtPlugin.class);+ }++ project+ .getTasks()+ .withType(Test.class)+ .configureEach(+ task -> {+ @SuppressWarnings("unchecked")+ Provider<FailingTestsReportingService> service =+ (Provider<FailingTestsReportingService>)+ project+ .getGradle()+ .getSharedServices()+ .getRegistrations()+ .getByName(SERVICE_NAME)+ .getService();+ task.usesService(service);++ String projectPath = project.getPath();+ String reproLine = ErrorReportingTestListener.getReproLineOptions(task);+ DirectoryProperty testOutputsDir =+ task.getExtensions()+ .getByType(TestsAndRandomizationPlugin.TestOutputsExtension.class)+ .getTestOutputsDir();++ task.addTestListener(+ new TestListener() {+ @Override+ public void beforeTest(TestDescriptor testDescriptor) {}++ @Override+ public void afterTest(TestDescriptor desc, TestResult result) {+ if (result.getResultType() == TestResult.ResultType.FAILURE) {+ // check if it's a constructor or a before/after class hook that failed.+ String qTestName;+ if (desc.getName().equals("classMethod")) {+ qTestName = desc.getClassName();+ } else {+ qTestName = desc.getClassName() + "." + desc.getName();+ }++ var randomizationParameters = "";+ var p = Pattern.compile(".+ (?<params>[{].*[}])$");+ var matcher = p.matcher(qTestName);+ if (matcher.matches()) {+ randomizationParameters = matcher.group("params");+ qTestName = qTestName.replace(randomizationParameters, "").trim();+ }++ service+ .get()+ .addFailedTest(+ new Entry(+ qTestName,+ randomizationParameters,+ projectPath,+ "gradlew "+ + (projectPath + ":test")+ + (" --tests \"" + qTestName + "\" ")+ + reproLine,+ testOutputsDir+ .file(+ ErrorReportingTestListener.getOutputLogName(+ desc.getParent()))+ .get()+ .getAsFile()));+ }+ }++ @Override+ public void beforeSuite(TestDescriptor suite) {}++ @Override+ public void afterSuite(TestDescriptor desc, TestResult result) {+ if (result.getExceptions() != null && !result.getExceptions().isEmpty()) {+ service+ .get()+ .addFailedTest(+ new Entry(+ desc.getName(),+ null,+ projectPath,+ "gradlew "+ + (projectPath + ":test")+ + (" --tests \"" + desc.getName() + "\" ")+ + reproLine,+ testOutputsDir+ .file(ErrorReportingTestListener.getOutputLogName(desc))+ .get()+ .getAsFile()));+ }+ }+ });+ });+ }++ /** Build service that keeps track of test and suite times. */+ public abstract static class FailingTestsReportingService+ implements BuildService<BuildServiceParameters.None>, AutoCloseable {+ private static final Logger LOGGER = Logging.getLogger(FailingTestsReportingService.class);++ private final ConcurrentLinkedQueue<Entry> failedTests = new ConcurrentLinkedQueue<>();++ @Inject+ public FailingTestsReportingService() {}++ public void addFailedTest(Entry entry) {+ failedTests.add(entry);+ }++ @Override+ public void close() {+ if (!failedTests.isEmpty()) {+ var limit = 10;+ var formatted =+ failedTests.stream()+ .sorted((a, b) -> b.projectPath.compareTo(a.projectPath))+ .map(+ e ->+ String.format(+ Locale.ROOT,+ " - %s (%s)%s\n Test output: %s\n Reproduce with: %s\n",+ e.name,+ e.projectPath,+ e.randomizationParameters != null+ && !e.randomizationParameters.isBlank()+ ? "\n Context parameters: " + e.randomizationParameters+ : "",+ e.testOutput,+ e.reproLine))+ .limit(limit)+ .collect(Collectors.joining("\n"));++ LOGGER.error(+ "\nERROR: {} {} failed{}:\n\n{}",+ failedTests.size(),+ failedTests.size() == 1 ? "test has" : failedTests.size() + " tests have",+ failedTests.size() > limit ? " (top " + limit + " shown)" : "",+ formatted);+ }+ }+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowSlowestTestsAtEndPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowSlowestTestsAtEndPlugin.javanew file mode 100644index 00000000000..c8cebccf325--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ShowSlowestTestsAtEndPlugin.java@@ -0,0 +1,227 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.java;++import java.util.Collection;+import java.util.Locale;+import java.util.concurrent.ConcurrentLinkedQueue;+import java.util.stream.Collectors;+import javax.inject.Inject;+import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.gradle.api.Project;+import org.gradle.api.logging.Logger;+import org.gradle.api.logging.Logging;+import org.gradle.api.plugins.JavaPlugin;+import org.gradle.api.provider.Provider;+import org.gradle.api.services.BuildService;+import org.gradle.api.services.BuildServiceParameters;+import org.gradle.api.tasks.testing.Test;+import org.gradle.api.tasks.testing.TestDescriptor;+import org.gradle.api.tasks.testing.TestListener;+import org.gradle.api.tasks.testing.TestResult;+import org.gradle.build.event.BuildEventsListenerRegistry;+import org.gradle.tooling.events.FinishEvent;+import org.gradle.tooling.events.OperationCompletionListener;+import org.gradle.tooling.events.task.TaskFailureResult;++/** Display the slowest tests at the end of the build. */+public class ShowSlowestTestsAtEndPlugin extends LuceneGradlePlugin {+ private static final String SERVICE_NAME = "slowestTestsTrackingService";++ public abstract static class RootExtPlugin extends LuceneGradlePlugin {+ /**+ * @return Returns the injected build events listener registry.+ */+ @Inject+ protected abstract BuildEventsListenerRegistry getListenerRegistry();++ @Override+ public void apply(Project project) {+ applicableToRootProjectOnly(project);++ // Register the shared build service that will do the end-of-build check+ var service =+ project+ .getGradle()+ .getSharedServices()+ .registerIfAbsent(SERVICE_NAME, TestStatsService.class, _ -> {});++ getListenerRegistry().onTaskCompletion(service);+ }+ }++ public record Entry(String name, long duration) {}++ @Override+ public void apply(Project project) {+ requiresAppliedPlugin(project, JavaPlugin.class);++ var buildOptions = getBuildOptions(project);+ Provider<Boolean> slowestTestsOption =+ buildOptions.addBooleanOption(+ "tests.slowestTests", "Print the summary of the slowest tests.", true);+ int slowestTestsMinTime =+ buildOptions+ .addIntOption(+ "tests.slowestTests.minTime",+ "Minimum test time to consider a test slow (millis).",+ 500)+ .get();+ Provider<Boolean> slowestSuitesOption =+ buildOptions.addBooleanOption(+ "tests.slowestSuites", "Print the summary of the slowest suites.", true);+ int slowestSuitesMinTime =+ buildOptions+ .addIntOption(+ "tests.slowestSuites.minTime",+ "Minimum suite time to consider a suite slow (millis).",+ 1000)+ .get();++ boolean collectSlowestTests = slowestTestsOption.get();+ boolean collectSlowestSuites = slowestSuitesOption.get();++ if (!collectSlowestTests && !collectSlowestSuites) {+ // Nothing to do.+ return;+ }++ if (project != project.getRootProject()) {+ project.getRootProject().getPlugins().apply(RootExtPlugin.class);+ }++ project+ .getTasks()+ .withType(Test.class)+ .configureEach(+ task -> {+ @SuppressWarnings("unchecked")+ Provider<TestStatsService> service =+ (Provider<TestStatsService>)+ project+ .getGradle()+ .getSharedServices()+ .getRegistrations()+ .getByName(SERVICE_NAME)+ .getService();+ task.usesService(service);++ var projectPath = project.getPath();++ task.addTestListener(+ new TestListener() {+ @Override+ public void beforeSuite(TestDescriptor suite) {}++ @Override+ public void afterSuite(TestDescriptor suite, TestResult result) {+ // Gradle reports runner times as well, omit anything that isn't attached to a+ // concrete class.+ if (collectSlowestSuites && suite.getClassName() != null) {+ long duration = (result.getEndTime() - result.getStartTime());+ if (duration >= slowestTestsMinTime) {+ service+ .get()+ .addSuiteEntry(+ new Entry(+ lastNameComponent(suite) + (" (" + projectPath + ")"),+ duration));+ }+ }+ }++ @Override+ public void beforeTest(TestDescriptor testDescriptor) {}++ @Override+ public void afterTest(TestDescriptor testDescriptor, TestResult result) {+ if (collectSlowestTests) {+ long duration = (result.getEndTime() - result.getStartTime());+ if (duration >= slowestSuitesMinTime) {+ service+ .get()+ .addTestEntry(+ new Entry(+ lastNameComponent(testDescriptor)+ + ("." + testDescriptor.getName())+ + (" (" + projectPath + ")"),+ duration));+ }+ }+ }+ });+ });+ }++ private static String lastNameComponent(TestDescriptor suite) {+ return suite.getClassName().replaceAll(".+\\.", "");+ }++ /** Build service that keeps track of test and suite times. */+ public abstract static class TestStatsService+ implements BuildService<BuildServiceParameters.None>,+ OperationCompletionListener,+ AutoCloseable {+ private static final Logger LOGGER = Logging.getLogger(TestStatsService.class);++ private final ConcurrentLinkedQueue<Entry> allTests = new ConcurrentLinkedQueue<>();+ private final ConcurrentLinkedQueue<Entry> allSuites = new ConcurrentLinkedQueue<>();++ private volatile boolean hadFailedTask;++ @Inject+ public TestStatsService() {}++ public void addTestEntry(Entry entry) {+ allTests.add(entry);+ }++ public void addSuiteEntry(Entry entry) {+ allSuites.add(entry);+ }++ @Override+ public void onFinish(FinishEvent event) {+ if (event.getResult() instanceof TaskFailureResult) {+ hadFailedTask = true;+ }+ }++ @Override+ public void close() {+ if (hadFailedTask || (allTests.isEmpty() && allSuites.isEmpty())) {+ return;+ }++ if (!allTests.isEmpty()) {+ LOGGER.lifecycle("The slowest tests during this run:\n " + toString(allTests));+ }++ if (!allSuites.isEmpty()) {+ LOGGER.lifecycle("The slowest suites during this run:\n " + toString(allSuites));+ }+ }++ private static String toString(Collection<Entry> entries) {+ return entries.stream()+ .sorted((a, b) -> Long.compare(b.duration, a.duration))+ .limit(10)+ .map(e -> String.format(Locale.ROOT, "%5.2fs %s", e.duration / 1000d, e.name))+ .collect(Collectors.joining("\n "));+ }+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.javaindex 865c0ac2fad..9b434b4af59 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java@@ -36,7 +36,6 @@ import org.gradle.api.file.Directory;import org.gradle.api.file.DirectoryProperty;import org.gradle.api.file.RegularFileProperty;import org.gradle.api.invocation.Gradle;-import org.gradle.api.plugins.ExtraPropertiesExtension;import org.gradle.api.plugins.JavaPlugin;import org.gradle.api.provider.Provider;import org.gradle.api.tasks.InputFile;@@ -365,8 +364,9 @@ public class TestsAndRandomizationPlugin extends LuceneGradlePlugin {.getAsFile();task.getExtensions()- .getByType(ExtraPropertiesExtension.class)- .set("testOutputsDir", testOutputsDir);+ .create("testOutputsExtension", TestOutputsExtension.class)+ .getTestOutputsDir()+ .set(testOutputsDir);// LUCENE-9660: Make it possible to always rerun tests, even if they're incrementally// up-to-date.@@ -513,6 +513,10 @@ public class TestsAndRandomizationPlugin extends LuceneGradlePlugin {});}+ public abstract static class TestOutputsExtension {+ abstract DirectoryProperty getTestOutputsDir();+ }+public abstract static class LoggingFileArgumentProvider implements CommandLineArgumentProvider {@InputFile@PathSensitive(PathSensitivity.RELATIVE)diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.javanew file mode 100644index 00000000000..628c4904cda--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesPlugin.java@@ -0,0 +1,126 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.licenses;++import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.gradle.api.Project;++/** This configures ASL and other license checks. */+public class CheckLicensesPlugin extends LuceneGradlePlugin {+ @Override+ public void apply(Project project) {+ applicableToRootProjectOnly(project);++ // Add check licenses task to the top-level project, configure it to scan all files, including+ // those from subprojects. It's fast and simple.+ var checkLicensesTask =+ project+ .getTasks()+ .register("checkLicenses", CheckLicensesTask.class, this::configureCheckLicenses);++ // Link any 'check' task from any subproject to this top-level task.+ project.subprojects(+ subproject -> {+ subproject+ .getTasks()+ .named("check")+ .configure(+ checkTask -> {+ checkTask.dependsOn(checkLicensesTask);+ });+ });+ }++ private void configureCheckLicenses(CheckLicensesTask task) {+ Project project = task.getProject();++ task.getReportFile().set(project.getLayout().getBuildDirectory().file("licenses-report.txt"));+ task.getFiles()+ .from(+ project.fileTree(+ ".",+ tree -> {+ // Exclude build outputs, ide files, .git.+ tree.exclude(".git");+ tree.exclude(".idea");+ tree.exclude(".muse");+ tree.exclude("**/build/**");+ tree.exclude("**/.gradle");++ // Exclude generated stuff.+ tree.exclude("**/src/generated/**");++ // Exclude github stuff (templates, workflows).+ tree.exclude(".github");++ // do not let RAT attempt to scan a python venv, it gets lost and confused...+ tree.exclude("**/.venv/**");++ // apache rat has a 'binary guesser'... I don't think this needs to be done at all+ // -+ // just exclude binaries here.+ tree.exclude("**/*.dat");+ tree.exclude("**/*.brk");+ tree.exclude("**/*.gz");+ tree.exclude("**/*.bin");+ tree.exclude("**/*.bz2");+ tree.exclude("**/*.gif");+ tree.exclude("**/*.svg");+ tree.exclude("lucene/analysis/smartcn/src/**/*.mem");++ // Only check these selected file patterns as folks have various .gitignore-d+ // resources generated by IDEs, etc.+ tree.include("**/*.gradle");+ tree.include("**/*.xml");+ tree.include("**/*.md");+ tree.include("**/*.py");+ tree.include("**/*.sh");+ tree.include("**/*.bat");++ // Include selected patterns from any source folders.+ tree.include("**/src/**");+ tree.exclude("**/src/**/*.png");+ tree.exclude("**/src/**/*.txt");+ tree.exclude("**/src/**/*.zip");+ tree.exclude("**/src/**/*.properties");+ tree.exclude("**/src/**/*.utf8");++ // project-specific exclusions.+ tree.exclude("build-tools/build-infra-shadow/src/java/keep.me");+ tree.exclude("lucene/analysis/icu/src/**/utr30.nrm");+ tree.exclude("lucene/analysis/kuromoji/src/**/bocchan.utf-8");+ tree.exclude("lucene/analysis/morfologik/src/**/*.info");+ tree.exclude("lucene/analysis/morfologik/src/**/*.input");+ tree.exclude("lucene/analysis/morfologik/src/**/*.dict");+ tree.exclude("lucene/analysis/stempel/src/**/*.tbl");+ tree.exclude("lucene/analysis/opennlp/src/**/en-test-lemmas.dict");+ tree.exclude("lucene/demo/src/**/knn-token-vectors");+ tree.exclude("lucene/test-framework/src/**/europarl.lines.txt.seek");+ tree.exclude("lucene/analysis/common/src/**/*.aff");+ tree.exclude("lucene/analysis/common/src/**/*.dic");+ tree.exclude("lucene/analysis/common/src/**/*.good");+ tree.exclude("lucene/analysis/common/src/**/*.sug");+ tree.exclude("lucene/analysis/common/src/**/*.wrong");+ tree.exclude("lucene/analysis/common/src/**/*.rslp");+ tree.exclude("lucene/analysis/common/src/**/*.htm*");+ tree.exclude("lucene/analysis.tests/src/**/*.aff");+ tree.exclude("lucene/analysis.tests/src/**/*.dic");+ // Luke has an embedded ElegantIcons font (MIT licensed).+ tree.exclude("lucene/luke/src/**/ElegantIcons.ttf");+ }));+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesTask.javanew file mode 100644index 00000000000..77c928e67c3--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckLicensesTask.java@@ -0,0 +1,297 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.gradle.plugins.licenses;++import java.io.BufferedWriter;+import java.io.File;+import java.io.IOException;+import java.io.InputStreamReader;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Set;+import java.util.TreeMap;+import java.util.function.Predicate;+import java.util.stream.Collectors;+import java.util.stream.Stream;+import org.gradle.api.DefaultTask;+import org.gradle.api.GradleException;+import org.gradle.api.file.ConfigurableFileCollection;+import org.gradle.api.file.RegularFileProperty;+import org.gradle.api.provider.Property;+import org.gradle.api.tasks.CacheableTask;+import org.gradle.api.tasks.Input;+import org.gradle.api.tasks.InputFiles;+import org.gradle.api.tasks.OutputFile;+import org.gradle.api.tasks.PathSensitive;+import org.gradle.api.tasks.PathSensitivity;+import org.gradle.api.tasks.TaskAction;+import org.gradle.work.FileChange;+import org.gradle.work.Incremental;+import org.gradle.work.InputChanges;++/**+ * This task takes a set of files as input and verifies if their header contains any of the known+ * license patterns. Files that don't have any permitted licenses will trigger an error.+ */+@CacheableTask+public abstract class CheckLicensesTask extends DefaultTask {+ /** The default number of leading characters scanned in each file. */+ private static final Integer DEFAULT_SCANNED_HEADER = 1024;++ private record LicenseFamily(String code, String name, Predicate<String> matcherPredicate) {}++ static final List<LicenseFamily> LUCENE_ACCEPTED_LICENSES =+ List.of(+ new LicenseFamily(+ "ASL",+ "Apache Software License 2.0",+ fixedSubstrings(+ "http://www.apache.org/licenses/LICENSE-2.0",+ "https://www.apache.org/licenses/LICENSE-2.0")),+ new LicenseFamily(+ "MIT",+ "The MIT License",+ // ICU license (ScriptIterator.java)+ fixedSubstring(+ "Permission is hereby granted, free of charge, to any person obtaining a copy")+ .and(+ fixedSubstring(+ "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR"))),+ new LicenseFamily(+ "BSD",+ "Modified BSD License",+ fixedSubstrings(+ // brics automaton+ "Copyright (c) 2001-2009 Anders Moeller",+ // snowball+ "Copyright (c) 2001, Dr Martin Porter",+ // UMASS kstem+ "THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS",+ // Egothor+ "Egothor Software License version 1.00",+ // JaSpell+ "Copyright (c) 2005 Bruno Martins",+ // lz4+ "Copyright (c) 2011-2016, Yann Collet")),+ new LicenseFamily(+ "GENERATED",+ "Generated files",+ fixedSubstrings(+ // snowball+ "Generated by Snowball",+ // javacc+ "Generated By:JavaCC")));++ /** The number of leading characters scanned in each file. */+ @Input+ public abstract Property<Integer> getMaxScannedHeaderSize();++ @InputFiles+ @Incremental+ @PathSensitive(PathSensitivity.RELATIVE)+ public abstract ConfigurableFileCollection getFiles();++ /**+ * An output report file to keep up-to-date checks sane. The output file is a sorted list of all+ * input files and their license codes.+ */+ @OutputFile+ public abstract RegularFileProperty getReportFile();++ public CheckLicensesTask() {+ getMaxScannedHeaderSize().convention(DEFAULT_SCANNED_HEADER);+ }++ @TaskAction+ public void run(InputChanges changes) throws IOException {+ getLogger()+ .info("Checking licenses {}", changes.isIncremental() ? "(incremental run)" : "(full run)");++ // load the current report (if any) into a sorted map+ File reportFile = getReportFile().getAsFile().get();+ TreeMap<String, String> report = readExistingReport(reportFile);++ // update the report with changes.+ List<File> missingLicense = new ArrayList<>();+ char[] scratch = new char[1024];+ StringBuilder buffer = new StringBuilder();+ int count = 0;+ for (FileChange fc : changes.getFileChanges(getFiles())) {+ count++;+ File file = fc.getFile();+ if (file.isDirectory()) {+ continue;+ }++ String key = toRootRelative(file);++ switch (fc.getChangeType()) {+ case REMOVED -> report.remove(key);+ case ADDED, MODIFIED -> {+ LicenseFamily licenseFamily = detectLicense(file, buffer, scratch);+ if (licenseFamily == null) {+ missingLicense.add(file);+ } else {+ report.put(key, licenseFamily.code);+ }+ }+ default -> throw new IOException("Unexpected change type: " + fc.getChangeType());+ }+ }++ getLogger().info("Checked {} {}", count, count == 1 ? "file" : "files");++ if (!missingLicense.isEmpty()) {+ throw new GradleException(+ String.format(+ Locale.ROOT,+ "The following files have no (acceptable) license header or the license header is not within the first %s header characters:\n%s",+ getMaxScannedHeaderSize().get(),+ missingLicense.stream()+ .map(file -> " - " + toRootRelative(file))+ .collect(Collectors.joining("\n"))));+ }++ // Also, in case the input set of files has itself changed, ensure the report only+ // contains files that still exist under the current sources.+ Set<String> current = new HashSet<>();+ for (File f : getFiles().getFiles()) {+ if (f.isFile()) {+ current.add(toRootRelative(f));+ }+ }+ report.keySet().removeIf(k -> !current.contains(k));++ var logger = getLogger();+ if (logger.isInfoEnabled()) {+ var counts =+ report.entrySet().stream()+ .collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.counting()));+ logger.info(+ "License type counts:\n{}",+ counts.entrySet().stream()+ .sorted((a, b) -> Long.compare(b.getValue().longValue(), a.getValue().longValue()))+ .map(+ e ->+ String.format(+ Locale.ROOT,+ " - %s: %,d %s",+ e.getKey(),+ e.getValue(),+ e.getValue() == 1 ? "file" : "files"))+ .collect(Collectors.joining("\n")));+ }++ writeReport(report, reportFile);+ }++ private TreeMap<String, String> readExistingReport(File reportFile) throws IOException {+ TreeMap<String, String> report = new TreeMap<>();+ if (!reportFile.exists()) {+ return report;+ }++ try (var lineStream = Files.lines(reportFile.toPath())) {+ lineStream.forEach(+ line -> {+ int idx = line.lastIndexOf(':');+ String path = line.substring(0, idx).trim();+ String state = line.substring(idx + 1).trim();+ if (!path.isEmpty() && !state.isEmpty()) {+ report.put(path, state);+ }+ });+ }++ return report;+ }++ private void writeReport(TreeMap<String, String> report, File reportFile) throws IOException {+ Files.createDirectories(reportFile.toPath().getParent());++ try (BufferedWriter bw = Files.newBufferedWriter(reportFile.toPath())) {+ for (Map.Entry<String, String> e : report.entrySet()) {+ bw.write(e.getKey());+ bw.write(": ");+ bw.write(e.getValue());+ bw.write("\n");+ }+ }+ }++ /** Convert the given path to a root-project-relative path with unix path separators. */+ private String toRootRelative(File file) {+ Path root = getProject().getRootDir().toPath().normalize();+ Path p = file.toPath().toAbsolutePath().normalize();+ String key = root.relativize(p).toString();+ if (File.separatorChar != '/') {+ key = key.replace(File.separatorChar, '/');+ }+ return key;+ }++ private LicenseFamily detectLicense(File file, StringBuilder buffer, char[] scratch)+ throws IOException {+ // I assume all files are in UTF8... This is verified elsewhere (eclint).++ buffer.setLength(0);+ try (var reader =+ new InputStreamReader(Files.newInputStream(file.toPath()), StandardCharsets.UTF_8)) {++ int maxChars = getMaxScannedHeaderSize().get();+ while (maxChars > 0) {+ int readChars = reader.read(scratch);+ if (readChars < 0) {+ break;+ }+ buffer.append(scratch, 0, readChars);+ maxChars -= readChars;+ }++ String header = buffer.toString();++ for (var licenseFamily : LUCENE_ACCEPTED_LICENSES) {+ if (licenseFamily.matcherPredicate.test(header)) {+ return licenseFamily;+ }+ }++ return null;+ }+ }++ /** Matches a fixed substring. */+ private static Predicate<String> fixedSubstring(String substring) {+ return header -> header.contains(substring);+ }++ /** Any of the provided substrings. */+ private static Predicate<String> fixedSubstrings(String... otherSubstrings) {+ return anyOf(Stream.of(otherSubstrings).map(CheckLicensesTask::fixedSubstring).toList());+ }++ private static Predicate<String> anyOf(List<Predicate<String>> list) {+ return list.stream().reduce(_ -> false, Predicate::or);+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.javaindex 3e44576a7d7..08349c1a247 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java@@ -31,9 +31,9 @@ import org.apache.lucene.gradle.plugins.hacks.WipeGradleTempPlugin;import org.apache.lucene.gradle.plugins.help.BuildOptionGroupsPlugin;import org.apache.lucene.gradle.plugins.ide.EclipseSupportPlugin;import org.apache.lucene.gradle.plugins.ide.IdeaSupportPlugin;+import org.apache.lucene.gradle.plugins.licenses.CheckLicensesPlugin;import org.apache.lucene.gradle.plugins.regenerate.RegenerateTasksSupportPlugin;import org.apache.lucene.gradle.plugins.spotless.GradleGroovyFormatPlugin;-import org.apache.lucene.gradle.plugins.spotless.ValidateSourcePatternsPlugin;import org.gradle.api.Project;import org.gradle.api.file.DuplicatesStrategy;import org.gradle.api.initialization.IncludedBuild;@@ -76,7 +76,7 @@ public class RootProjectSetupPlugin extends LuceneGradlePlugin {plugins.apply(HacksPlugin.class);plugins.apply(WipeGradleTempPlugin.class);plugins.apply(GradleGroovyFormatPlugin.class);- plugins.apply(ValidateSourcePatternsPlugin.class);+ plugins.apply(CheckLicensesPlugin.class);plugins.apply(ConfigureLockFilePlugin.class);plugins.apply(CheckGradlewScriptsTweakedPlugin.class);diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.javadeleted file mode 100644index 1eb57eeb7a7..00000000000--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/spotless/ValidateSourcePatternsPlugin.java+++ /dev/null@@ -1,236 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.gradle.plugins.spotless;--import com.google.common.base.Splitter;-import java.io.File;-import java.io.IOException;-import java.io.InputStreamReader;-import java.io.StringWriter;-import java.nio.charset.CharsetDecoder;-import java.nio.charset.CodingErrorAction;-import java.nio.charset.StandardCharsets;-import java.nio.file.Files;-import java.util.Locale;-import java.util.Set;-import java.util.TreeSet;-import java.util.regex.Matcher;-import java.util.regex.Pattern;-import java.util.stream.Collectors;-import javax.inject.Inject;-import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;-import org.apache.rat.Defaults;-import org.apache.rat.analysis.RatHeaderAnalysisException;-import org.apache.rat.document.impl.FileDocument;-import org.gradle.api.DefaultTask;-import org.gradle.api.GradleException;-import org.gradle.api.Project;-import org.gradle.api.file.ConfigurableFileTree;-import org.gradle.api.tasks.CacheableTask;-import org.gradle.api.tasks.IgnoreEmptyDirectories;-import org.gradle.api.tasks.InputFiles;-import org.gradle.api.tasks.PathSensitive;-import org.gradle.api.tasks.PathSensitivity;-import org.gradle.api.tasks.TaskAction;-import org.gradle.api.tasks.compile.JavaCompile;-import org.gradle.internal.logging.progress.ProgressLogger;-import org.gradle.internal.logging.progress.ProgressLoggerFactory;--/** Checks for invalid usage patterns in source files. */-public class ValidateSourcePatternsPlugin extends LuceneGradlePlugin {- public static final String TASK_NAME = "validateSourcePatterns";-- @Override- public void apply(Project project) {- applicableToRootProjectOnly(project);-- project.allprojects(this::configureProject);- }-- private void configureProject(Project project) {- var validateSourcePatternsTask =- project- .getTasks()- .register(- TASK_NAME,- ValidateSourcePatternsTask.class,- task -> {- task.setGroup("Verification");- task.setDescription("Validate Source Patterns");-- ConfigurableFileTree sourceFiles = task.getSourceFiles();- sourceFiles.setDir(project.getLayout().getProjectDirectory());-- // it seems we only scan XML files - everything else has been moved- // to rat scanning or elsewhere.- sourceFiles.include("**/*.xml");-- // default excludes.- sourceFiles.exclude("**/build/**");- sourceFiles.exclude("**/.idea/**");- sourceFiles.exclude("**/.gradle/**");- sourceFiles.exclude("**/.git/**");-- // Don't go into subproject folders (each is scanned individually).- sourceFiles.exclude(- project.getChildProjects().keySet().stream()- .map(name -> name + "/**")- .toList());- });-- // Add to all checks.- project.getTasks().named("check").configure(task -> task.dependsOn(validateSourcePatternsTask));-- // Ensure validation runs prior to any compilation task.- project- .getTasks()- .withType(JavaCompile.class)- .configureEach(- task -> {- task.mustRunAfter(validateSourcePatternsTask);- });-- // project-specific tuning.- project- .project(":lucene:benchmark")- .getTasks()- .withType(ValidateSourcePatternsTask.class)- .configureEach(- task -> {- task.getSourceFiles().exclude("data/**", "work/**");- });- }-- @CacheableTask- public abstract static class ValidateSourcePatternsTask extends DefaultTask {- @InputFiles- @PathSensitive(PathSensitivity.RELATIVE)- @IgnoreEmptyDirectories- public abstract ConfigurableFileTree getSourceFiles();-- @Inject- protected abstract ProgressLoggerFactory getProgressLoggerFactory();-- @TaskAction- public void check() {- Set<File> files = getSourceFiles().getFiles();- getLogger()- .info(- "Input files for scanning:\n{}",- files.stream().map(f -> " - " + f).collect(Collectors.joining("\n")));-- var xmlCommentPattern = Pattern.compile("(?sm)\\Q<!--\\E(.*?)\\Q-->\\E");- var xmlTagPattern = Pattern.compile("(?m)\\s*<[a-zA-Z].*");- var violations = new TreeSet<String>();-- ProgressLogger progress = getProgressLoggerFactory().newOperation(this.getClass());- progress.start(this.getName(), this.getName());- for (var file : files) {- progress.progress("Scanning " + file.getName());-- String fileText = readUtf8WithValidation(file);-- if (file.getName().endsWith(".xml")) {- var ratDocument = new FileDocument(file);- checkLicenseHeaderPrecedes(- file, "<tag>", xmlTagPattern, xmlCommentPattern, fileText, ratDocument, violations);- }- }- progress.completed();-- if (!violations.isEmpty()) {- throw new GradleException(- String.format(- Locale.ROOT,- "Found %d source violation(s):\n %s",- violations.size(),- String.join("\n ", violations)));- }- }-- private void checkLicenseHeaderPrecedes(- File file,- String description,- Pattern contentPattern,- Pattern commentPattern,- String fileText,- FileDocument ratDocument,- TreeSet<String> violations) {- Matcher contentMatcher = contentPattern.matcher(fileText);- if (contentMatcher.find()) {- int contentStartPos = contentMatcher.start();- Matcher commentMatcher = commentPattern.matcher(fileText);- while (commentMatcher.find()) {- if (isLicense(file, commentMatcher.group(1), ratDocument)) {- if (commentMatcher.start() < contentStartPos) {- // This file is all good, so break the loop:- // license header precedes 'description' definition- break;- } else {- reportViolation(- violations, file, description + " declaration precedes license header");- }- }- }- }- }-- private void reportViolation(TreeSet<String> violations, File file, String name) {- String msg = String.format(Locale.ROOT, "%s: %s", file, name);- getLogger().error(msg);- violations.add(msg);- }-- // See LUCENE-10419 - rat is not thread safe.- private static final Object ratLockBug = new Object();- private static final Splitter lineSplitter = Splitter.on(Pattern.compile("[\\r\\n]+"));-- private boolean isLicense(File file, String text, FileDocument ratDocument) {- synchronized (ratLockBug) {- var licenseMatcher = Defaults.createDefaultMatcher();- licenseMatcher.reset();- return lineSplitter.splitToList(text).stream()- .anyMatch(- it -> {- try {- return licenseMatcher.match(ratDocument, it);- } catch (RatHeaderAnalysisException e) {- throw new GradleException("Could not scan this file with rat: " + file, e);- }- });- }- }-- private static String readUtf8WithValidation(File file) {- String fileText;- CharsetDecoder validatingDecoder =- StandardCharsets.UTF_8- .newDecoder()- .onMalformedInput(CodingErrorAction.REPORT)- .onUnmappableCharacter(CodingErrorAction.REPORT);- try (var is = Files.newInputStream(file.toPath());- var sw = new StringWriter();- var reader = new InputStreamReader(is, validatingDecoder)) {- reader.transferTo(sw);- fileText = sw.toString();- } catch (IOException e) {- throw new GradleException("Could not read: " + file, e);- }- return fileText;- }- }-}diff --git a/build.gradle b/build.gradleindex 05b3d5425e2..926c584bee7 100644--- a/build.gradle+++ b/build.gradle@@ -1,5 +1,4 @@-import org.apache.lucene.gradle.plugins.misc.CheckEnvironmentPlugin-import org.apache.lucene.gradle.plugins.spotless.GoogleJavaFormatPlugin+import org.apache.lucene.gradle.plugins.licenses.CheckLicensesTask/** Licensed to the Apache Software Foundation (ASF) under one or more@@ -27,15 +26,11 @@ plugins {id "lucene.root-project.setup"id "lucene.java-projects.conventions"- id "lucene.java.slowest-tests-at-end"- id "lucene.java.show-failed-tests-at-end"id "lucene.java.modules"id "lucene.validation.gradle-versions-cleanup"id "lucene.validation.error-prone"id "lucene.validation.jar-checks"- id "lucene.validation.rat-sources"- id "lucene.validation.owasp"id "lucene.publications.maven"id "lucene.publications.maven-to-nexus-releases"diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.tomlindex 3ec3fbbc760..5d77347ab90 100644--- a/gradle/libs.versions.toml+++ b/gradle/libs.versions.toml@@ -50,8 +50,6 @@ opennlp = "2.5.5"procfork = "1.0.6"# unit testsrandomizedtesting = "2.8.3"-# license checks-rat = "0.15"# spatial-extras/ supports2-geometry = "1.0.0"# spatial-extras/ support@@ -92,7 +90,6 @@ nekohtml = { module = "net.sourceforge.nekohtml:nekohtml", version.ref = "nekohtopennlp-tools = { module = "org.apache.opennlp:opennlp-tools", version.ref = "opennlp" }procfork = { module = "com.carrotsearch:procfork", version.ref = "procfork" }randomizedtesting-runner = { module = "com.carrotsearch.randomizedtesting:randomizedtesting-runner", version.ref = "randomizedtesting" }-rat = { module = "org.apache.rat:apache-rat", version.ref = "rat" }s2-geometry = { module = "io.sgr:s2-geometry-library-java", version.ref = "s2-geometry" }spatial4j = { module = "org.locationtech.spatial4j:spatial4j", version.ref = "spatial4j" }xerces = { module = "xerces:xercesImpl", version.ref = "xerces" }@@ -105,7 +102,6 @@ carrotsearch-dependencychecks = "com.carrotsearch.gradle.dependencychecks:0.1.0"errorprone = "net.ltgt.errorprone:4.3.0"forbiddenapis = "de.thetaphi.forbiddenapis:3.9"jacocolog = "org.barfuin.gradle.jacocolog:3.1.0"-owasp-dependencycheck = "org.owasp.dependencycheck:12.1.3"randomizedtesting = "com.carrotsearch.gradle.randomizedtesting:0.0.6"spotless = "com.diffplug.spotless:7.2.1"undercouch-download = "de.undercouch.download:5.6.0"diff --git a/help/workflow.txt b/help/workflow.txtindex 8b35247fcfe..f69966d5abc 100644--- a/help/workflow.txt+++ b/help/workflow.txt@@ -40,11 +40,3 @@ ls lucene/core/build/docsAssemble entire documentation (including javadocs):gradlew documentationls lucene/documentation/build/site---Other validation and checks-===========================--Generate a report of dependencies with known OWASP vulnerabilities:-gradlew :dependencyCheckAnalyze-open ./build/reports/dependency-check-report.html