Apache Lucene open-source search software
Run ecj in-process (#14915)
Dawid Weiss015b3bbf9c3a
diff --git a/build-tools/build-infra-shadow/build.gradle b/build-tools/build-infra-shadow/build.gradleindex 7d8043837f9..d7768878140 100644--- a/build-tools/build-infra-shadow/build.gradle+++ b/build-tools/build-infra-shadow/build.gradle@@ -46,6 +46,7 @@ dependencies {implementation deps.randomizedtesting.runnerimplementation deps.gjfimplementation deps.jgit+ implementation deps.ecjimplementation plugin(deps.plugins.carrotsearch.buildopts)implementation plugin(deps.plugins.carrotsearch.dependencychecks)diff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradleindex 84ed5c64b37..a0d0dc79e0c 100644--- a/build-tools/build-infra/build.gradle+++ b/build-tools/build-infra/build.gradle@@ -68,6 +68,7 @@ dependencies {implementation deps.flexmark.ext.tablesimplementation deps.gjfimplementation deps.jgit+ implementation deps.ecjimplementation plugin(deps.plugins.carrotsearch.buildopts)implementation plugin(deps.plugins.carrotsearch.dependencychecks)diff --git a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradleindex 7fa69f63aec..6f0171078df 100644--- a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle+++ b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle@@ -22,6 +22,7 @@ import org.apache.lucene.gradle.plugins.java.JarManifestConfigurationPluginimport org.apache.lucene.gradle.plugins.java.TestsAndRandomizationPluginimport org.apache.lucene.gradle.plugins.java.TestsBeastingPluginimport org.apache.lucene.gradle.plugins.java.AlternativeJdkSupportPlugin+import org.apache.lucene.gradle.plugins.java.EcjLintPluginallprojects { project ->if (['src/java', 'src/test'].any {path ->@@ -38,7 +39,7 @@ allprojects { project ->plugins.apply("lucene.java.per-project-test-summary")plugins.apply("lucene.validation.forbidden-apis")- plugins.apply("lucene.validation.ecj-lint")+ plugins.apply(EcjLintPlugin)plugins.apply(GoogleJavaFormatPlugin)}}diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradleindex daa29d3211d..839c9cb4c82 100644--- a/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle+++ b/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle@@ -19,6 +19,7 @@import java.nio.charset.StandardCharsetsimport java.nio.file.Filesimport java.nio.file.Path+import org.apache.lucene.gradle.plugins.java.ModularPathsExtensionApi// Configures miscellaneous aspects required to support the java module system layer.@@ -274,7 +275,7 @@ allprojects {// moduleImplementation - 'requires'// moduleCompileOnly - 'requires static'//-class ModularPathsExtension implements Cloneable, Iterable<Object> {+class ModularPathsExtension implements Cloneable, Iterable<Object>, ModularPathsExtensionApi {/*** Determines how paths are split between module path and classpath.*/diff --git a/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradle b/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradledeleted file mode 100644index 9f6bec29a64..00000000000--- a/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradle+++ /dev/null@@ -1,195 +0,0 @@-import org.apache.lucene.gradle.plugins.spotless.ApplyGoogleJavaFormatTask--/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--// This adds 'ecjLint' task to Java projects.--// Check if we need to use Lucene's storage space for interim "drop" versions of ecj-// that are ahead of the official Maven publications.-//-// Drop versions are at:-// https://www.eclipse.org/downloads/download.php?r=1&file=/eclipse/downloads/drops4/S-4.36M1-202504031800/ecj-${ecjVersion}.jar-//-// dweiss has access to Lucene's AWS storage space (requested via Infra).-//-boolean useDropMirror = deps.versions.ecj.get() in ["4.36M1"]-if (useDropMirror) {- repositories {- exclusiveContent {- forRepository {- ivy {- url = 'https://s3.amazonaws.com/lucene-testdata/temp-repo/ecj/'- patternLayout {- artifact "[artifact]-[revision](.[ext])"- m2compatible = true- }- metadataSources {- it.artifact()- }- }- }- filter {- includeGroup "org.eclipse.jdt"- }- }- }-}--configurations {- ecjDeps-}--dependencies {- ecjDeps deps.ecj-}--Configuration ecjConfiguration = configurations.ecjDeps-String javadocPrefsPath = rootProject.layout.projectDirectory.file("gradle/validation/ecj-lint/ecj.javadocs.prefs").asFile.absolutePath--// Create a [sourceSetName]EcjLint task for each source set-// with a non-empty java.srcDirs. These tasks are then-// attached to project's "ecjLint" task.-def lintTasks = sourceSets.collect { SourceSet sourceSet ->- def srcDirs = sourceSet.java.sourceDirectories- .filter { dir -> dir.exists() }-- tasks.register(sourceSet.getTaskName("ecjLint", null), JavaExec, { JavaExec task ->- // This dependency is on a configuration; technically it causes- // all dependencies to be resolved before this task executes- // (this includes scheduling tasks that compile the- // sources from other projects for example).- dependsOn sourceSet.compileClasspath- dependsOn ecjConfiguration-- mustRunAfter tasks.withType(ApplyGoogleJavaFormatTask)-- // The inputs are all source files from the sourceSet.- inputs.files sourceSet.allSource.asFileTree-- // We create a task for all source sets but ignore those- // that don't have any Java source directories.- enabled = !srcDirs.isEmpty()-- // Ignore MR-JAR sourceSets.- if (sourceSet.name ==~ /main\d+/) {- enabled = false- }-- classpath = ecjConfiguration- mainClass = "org.eclipse.jdt.internal.compiler.batch.Main"-- // DOCS: https://help.eclipse.org/latest/index.jsp?topic=%2Forg.eclipse.jdt.doc.user%2Ftasks%2Ftask-using_batch_compiler.htm-- // Don't emit any .class files.- // Hack around "-d none" still emitting package-info.class- // by running in a temporary directory.- def tmpDst = getTemporaryDir()- workingDir tmpDst-- args += ["-d", "none"]-- assert project.java.sourceCompatibility == project.java.targetCompatibility-- // Compilation environment.- // we use -source/-target as it is significantly faster than --release- args += [- "-source",- project.java.sourceCompatibility- ]- args += [- "-target",- project.java.targetCompatibility- ]- args += ["-encoding", "UTF-8"]- args += ["-proc:none"]- args += ["-nowarn"]- args += ["-enableJavadoc"]- args += [- "-properties",- javadocPrefsPath- ]-- // We depend on modular paths.- def modularPaths = sourceSet.modularPathsForEcj- dependsOn modularPaths-- // Add modular dependencies and their transitive dependencies to module path.- task.argumentProviders.add(modularPaths.compilationArguments)-- // Add classpath, if needed.- task.argumentProviders.add((CommandLineArgumentProvider) {- // Add classpath locations in a lazy provider (can't resolve the- // configuration at evaluation time). Filter out non-existing entries- // (output folders for non-existing input source dirs like resources).- FileCollection cpath = modularPaths.compilationClasspath.filter { p -> p.exists() }- if (!cpath.isEmpty()) {- return [- "-classpath",- cpath.join(File.pathSeparator)- ]- } else {- return []- }- })-- // Place input files in an external file to dodge command line argument- // limits. We could pass a directory but ecj seems to be buggy: when it- // encounters a module-info.java file it no longer compiles other source files.- def inputsFile = file("${tmpDst}/ecj-inputs.txt")- task.argumentProviders.add((CommandLineArgumentProvider) {- return ["@" + inputsFile.absolutePath]- })-- doFirst {- modularPaths.logCompilationPaths(logger)-- tmpDst.mkdirs()-- // escape filename accoring to ECJ's rules:- // https://github.com/eclipse/aspectj.eclipse.jdt.core/blob/a05312e746b9bc2b48b4b039f6e7b5e061b5b393/org.eclipse.jdt.core/batch/org/eclipse/jdt/internal/compiler/batch/Main.java#L1533-L1537- // Basically surround all whitespace by quotes:- def escapeFileName = { String s -> s.replaceAll(/ +/, /"$0"/) }- inputsFile.setText(- srcDirs.collectMany { dir ->- project.fileTree(- dir: dir,- include: "**/*.java",- // Exclude the benchmark class with dependencies on nekohtml, which causes module-classpath conflicts and breaks ecj.- exclude: "**/DemoHTMLParser.java"- ).files- }- // Try to sort all input files; a side-effect of this should be that module-info.java- // is placed first on the list, which works around ECJ bug:- // https://bugs.eclipse.org/bugs/show_bug.cgi?id=569833- .sort()- .collect {file -> escapeFileName(file.absolutePath.toString())}.join("\n"), "UTF-8")- }- })-}--def ecjLint = tasks.register("ecjLint", {- description = "Lint Java sources using ECJ."- group = "Verification"-- dependsOn lintTasks-})--// Attach ecjLint to check.-tasks.named("check").configure {- dependsOn ecjLint-}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.javanew file mode 100644index 00000000000..8c378f979e0--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.java@@ -0,0 +1,209 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.java;++import java.io.File;+import java.io.IOException;+import java.io.PrintWriter;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import java.util.stream.Collectors;+import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;+import org.apache.lucene.gradle.plugins.spotless.ApplyGoogleJavaFormatTask;+import org.eclipse.jdt.internal.compiler.batch.Main;+import org.gradle.api.GradleException;+import org.gradle.api.Project;+import org.gradle.api.Task;+import org.gradle.api.file.FileCollection;+import org.gradle.api.plugins.JavaPlugin;+import org.gradle.api.plugins.JavaPluginExtension;+import org.gradle.api.tasks.SourceSet;+import org.gradle.api.tasks.TaskProvider;+import org.gradle.process.CommandLineArgumentProvider;++/**+ * This adds additional linting of Java sources using ECJ (eclipse's compiler).+ *+ * @see+ * "https://help.eclipse.org/latest/index.jsp?topic=%2Forg.eclipse.jdt.doc.user%2Ftasks%2Ftask-using_batch_compiler.htm"+ */+public class EcjLintPlugin extends LuceneGradlePlugin {++ /*+ * Using an explicit dependency means we can't be as flexible in using interim "drops" of ecj,+ * as previously implemented in lucene.validation.ecj-lint.gradle. Defer the solution+ * until it really becomes a problem.+ */++ @Override+ public void apply(Project project) {+ requiresAppliedPlugin(project, JavaPlugin.class);++ Path javadocPrefsPath = gradlePluginResource(project, "validation/ecj-lint/ecj.javadocs.prefs");++ // Create a [sourceSetName]EcjLint task for each source set+ // with a non-empty java.srcDirs. These tasks are then+ // attached to project's "ecjLint" task.+ var javaExt = project.getExtensions().getByType(JavaPluginExtension.class);+ assert javaExt.getSourceCompatibility().equals(javaExt.getTargetCompatibility());++ var sourceSets = javaExt.getSourceSets();+ var tasks = project.getTasks();+ List<? extends TaskProvider<?>> lintTasks =+ sourceSets.stream()+ .map(+ sourceSet ->+ tasks.register(+ sourceSet.getTaskName("ecjLint", null),+ task -> configureEcjTask(sourceSet, javaExt, javadocPrefsPath, task)))+ .toList();++ var ecjLintTask =+ tasks.register(+ "ecjLint",+ task -> {+ task.setGroup("Verification");+ task.setDescription("Lint Java sources using ECJ.");+ task.dependsOn(lintTasks);+ });++ // Attach ecjLint to check.+ tasks+ .named("check")+ .configure(+ task -> {+ task.dependsOn(ecjLintTask);+ });+ }++ private void configureEcjTask(+ SourceSet sourceSet, JavaPluginExtension javaExt, Path javadocPrefsPath, Task task) {+ // This dependency is on a configuration; technically it causes+ // all dependencies to be resolved before this task executes+ // (this includes scheduling tasks that compile the+ // sources from other projects for example).+ task.dependsOn(sourceSet.getCompileClasspath());++ task.mustRunAfter(task.getProject().getTasks().withType(ApplyGoogleJavaFormatTask.class));++ // We create a task for all source sets but don't run+ // tasks for which Java source directories are empty.+ // We also ignore mr-jar source sets.+ if (sourceSet.getJava().getAsFileTree().isEmpty() || sourceSet.getName().matches("main\\d+")) {+ task.setEnabled(false);+ }++ List<String> args = new ArrayList<>();+ args.add("-d");+ args.add("none");++ // Use -source/-target as it is significantly faster than --release++ args.addAll(List.of("-source", javaExt.getSourceCompatibility().toString()));+ args.addAll(List.of("-target", javaExt.getTargetCompatibility().toString()));++ args.addAll(List.of("-encoding", "UTF-8"));+ args.add("-proc:none");+ args.add("-nowarn");+ args.add("-enableJavadoc");+ args.addAll(List.of("-properties", javadocPrefsPath.toAbsolutePath().toString()));++ // We depend on modular paths.+ ModularPathsExtensionApi modularPaths =+ (ModularPathsExtensionApi) sourceSet.getExtensions().getByName("modularPathsForEcj");+ task.dependsOn(modularPaths);++ // Collect modular dependencies and their transitive dependencies to module path.+ CommandLineArgumentProvider compilationArguments = modularPaths.getCompilationArguments();+ // this isn't exactly right but the returned CommandLineArgumentProvider isn't serializable...+ task.getInputs().files(sourceSet.getCompileClasspath());++ // Collect classpath locations+ FileCollection classpathArguments = modularPaths.getCompilationClasspath().filter(File::exists);+ task.getInputs().files(classpathArguments);++ // Input sources.+ var inputSources = sourceSet.getJava().getAsFileTree();+ task.getInputs().files(inputSources);++ // Add all arguments so far as inputs.+ task.getInputs().property("args", args);++ var argsEchoFile =+ task.getProject()+ .getLayout()+ .getBuildDirectory()+ .file("ecj/" + task.getName() + "-args.txt");++ var ecjOutput =+ task.getProject()+ .getLayout()+ .getBuildDirectory()+ .file("ecj/" + task.getName() + "-output.txt");++ task.getOutputs().files(argsEchoFile, ecjOutput);++ task.doFirst(+ _ -> {+ List<String> allArgs = new ArrayList<>(args);++ compilationArguments.asArguments().forEach(allArgs::add);+ if (!classpathArguments.isEmpty()) {+ allArgs.add("-classpath");+ allArgs.add(+ classpathArguments.getFiles().stream()+ .map(File::toString)+ .collect(Collectors.joining(File.pathSeparator)));+ }++ var sources =+ inputSources.getFiles().stream()+ .map(File::toString)+ .filter(+ file -> {+ // Exclude the benchmark class with dependencies on nekohtml,+ // which causes module-classpath conflicts and breaks ecj.+ return !file.endsWith("DemoHTMLParser.java");+ })+ .sorted()+ .toList();++ allArgs.addAll(sources);++ File ecjOutputFile = ecjOutput.get().getAsFile();+ try (var out = new PrintWriter(ecjOutputFile, StandardCharsets.UTF_8)) {+ Files.writeString(argsEchoFile.get().getAsFile().toPath(), String.join("\n", allArgs));++ var success =+ new Main(out, out, false /* systemExit */, null /* options */, null /* progress */)+ .compile(allArgs.toArray(String[]::new));++ out.flush();+ if (!success) {+ throw new GradleException(+ "ecj linter failed:\n" + Files.readString(ecjOutputFile.toPath()));+ }+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ });+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.javanew file mode 100644index 00000000000..c338b4640f8--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.java@@ -0,0 +1,30 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.java;++import org.gradle.api.file.FileCollection;+import org.gradle.process.CommandLineArgumentProvider;++/**+ * This interface should be replaced with ModularPathsExtension, once lucene.java.modules plugin is+ * converted to Java.+ */+public interface ModularPathsExtensionApi {+ CommandLineArgumentProvider getCompilationArguments();++ FileCollection getCompilationClasspath();+}diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.tomlindex 59adfec0468..c2c49726af9 100644--- a/gradle/libs.versions.toml+++ b/gradle/libs.versions.toml@@ -9,7 +9,7 @@ commons-codec = "1.18.0"# benchmark/ bzip2 compressioncommons-compress = "1.27.1"# See ecj-lint.gradle (used for more linting).-ecj = "4.36M1"+ecj = "3.42.0"# java lintingerrorprone = "2.38.0"# markdown documentation generation