Skip to content

Apache Lucene open-source search software

Run ecj in-process (#14915)

Dawid Weiss015b3bbf9c3a
diff --git a/build-tools/build-infra-shadow/build.gradle b/build-tools/build-infra-shadow/build.gradle
index 7d8043837f9..d7768878140 100644
--- a/build-tools/build-infra-shadow/build.gradle
+++ b/build-tools/build-infra-shadow/build.gradle
@@ -46,6 +46,7 @@ dependencies {
implementation deps.randomizedtesting.runner
implementation deps.gjf
implementation deps.jgit
+ implementation deps.ecj
implementation plugin(deps.plugins.carrotsearch.buildopts)
implementation plugin(deps.plugins.carrotsearch.dependencychecks)
diff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradle
index 84ed5c64b37..a0d0dc79e0c 100644
--- a/build-tools/build-infra/build.gradle
+++ b/build-tools/build-infra/build.gradle
@@ -68,6 +68,7 @@ dependencies {
implementation deps.flexmark.ext.tables
implementation deps.gjf
implementation deps.jgit
+ implementation deps.ecj
implementation plugin(deps.plugins.carrotsearch.buildopts)
implementation plugin(deps.plugins.carrotsearch.dependencychecks)
diff --git a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle
index 7fa69f63aec..6f0171078df 100644
--- a/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle
+++ b/build-tools/build-infra/src/main/groovy/lucene.java-projects.conventions.gradle
@@ -22,6 +22,7 @@ import org.apache.lucene.gradle.plugins.java.JarManifestConfigurationPlugin
import org.apache.lucene.gradle.plugins.java.TestsAndRandomizationPlugin
import org.apache.lucene.gradle.plugins.java.TestsBeastingPlugin
import org.apache.lucene.gradle.plugins.java.AlternativeJdkSupportPlugin
+import org.apache.lucene.gradle.plugins.java.EcjLintPlugin
allprojects { project ->
if (['src/java', 'src/test'].any {path ->
@@ -38,7 +39,7 @@ allprojects { project ->
plugins.apply("lucene.java.per-project-test-summary")
plugins.apply("lucene.validation.forbidden-apis")
- plugins.apply("lucene.validation.ecj-lint")
+ plugins.apply(EcjLintPlugin)
plugins.apply(GoogleJavaFormatPlugin)
}
}
diff --git a/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle b/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle
index daa29d3211d..839c9cb4c82 100644
--- a/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle
+++ b/build-tools/build-infra/src/main/groovy/lucene.java.modules.gradle
@@ -19,6 +19,7 @@
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
+import org.apache.lucene.gradle.plugins.java.ModularPathsExtensionApi
// Configures miscellaneous aspects required to support the java module system layer.
@@ -274,7 +275,7 @@ allprojects {
// moduleImplementation - 'requires'
// moduleCompileOnly - 'requires static'
//
-class ModularPathsExtension implements Cloneable, Iterable<Object> {
+class ModularPathsExtension implements Cloneable, Iterable<Object>, ModularPathsExtensionApi {
/**
* Determines how paths are split between module path and classpath.
*/
diff --git a/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradle b/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradle
deleted file mode 100644
index 9f6bec29a64..00000000000
--- a/build-tools/build-infra/src/main/groovy/lucene.validation.ecj-lint.gradle
+++ /dev/null
@@ -1,195 +0,0 @@
-import org.apache.lucene.gradle.plugins.spotless.ApplyGoogleJavaFormatTask
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This adds 'ecjLint' task to Java projects.
-
-// Check if we need to use Lucene's storage space for interim "drop" versions of ecj
-// that are ahead of the official Maven publications.
-//
-// Drop versions are at:
-// https://www.eclipse.org/downloads/download.php?r=1&file=/eclipse/downloads/drops4/S-4.36M1-202504031800/ecj-${ecjVersion}.jar
-//
-// dweiss has access to Lucene's AWS storage space (requested via Infra).
-//
-boolean useDropMirror = deps.versions.ecj.get() in ["4.36M1"]
-if (useDropMirror) {
- repositories {
- exclusiveContent {
- forRepository {
- ivy {
- url = 'https://s3.amazonaws.com/lucene-testdata/temp-repo/ecj/'
- patternLayout {
- artifact "[artifact]-[revision](.[ext])"
- m2compatible = true
- }
- metadataSources {
- it.artifact()
- }
- }
- }
- filter {
- includeGroup "org.eclipse.jdt"
- }
- }
- }
-}
-
-configurations {
- ecjDeps
-}
-
-dependencies {
- ecjDeps deps.ecj
-}
-
-Configuration ecjConfiguration = configurations.ecjDeps
-String javadocPrefsPath = rootProject.layout.projectDirectory.file("gradle/validation/ecj-lint/ecj.javadocs.prefs").asFile.absolutePath
-
-// Create a [sourceSetName]EcjLint task for each source set
-// with a non-empty java.srcDirs. These tasks are then
-// attached to project's "ecjLint" task.
-def lintTasks = sourceSets.collect { SourceSet sourceSet ->
- def srcDirs = sourceSet.java.sourceDirectories
- .filter { dir -> dir.exists() }
-
- tasks.register(sourceSet.getTaskName("ecjLint", null), JavaExec, { JavaExec task ->
- // This dependency is on a configuration; technically it causes
- // all dependencies to be resolved before this task executes
- // (this includes scheduling tasks that compile the
- // sources from other projects for example).
- dependsOn sourceSet.compileClasspath
- dependsOn ecjConfiguration
-
- mustRunAfter tasks.withType(ApplyGoogleJavaFormatTask)
-
- // The inputs are all source files from the sourceSet.
- inputs.files sourceSet.allSource.asFileTree
-
- // We create a task for all source sets but ignore those
- // that don't have any Java source directories.
- enabled = !srcDirs.isEmpty()
-
- // Ignore MR-JAR sourceSets.
- if (sourceSet.name ==~ /main\d+/) {
- enabled = false
- }
-
- classpath = ecjConfiguration
- mainClass = "org.eclipse.jdt.internal.compiler.batch.Main"
-
- // DOCS: https://help.eclipse.org/latest/index.jsp?topic=%2Forg.eclipse.jdt.doc.user%2Ftasks%2Ftask-using_batch_compiler.htm
-
- // Don't emit any .class files.
- // Hack around "-d none" still emitting package-info.class
- // by running in a temporary directory.
- def tmpDst = getTemporaryDir()
- workingDir tmpDst
-
- args += ["-d", "none"]
-
- assert project.java.sourceCompatibility == project.java.targetCompatibility
-
- // Compilation environment.
- // we use -source/-target as it is significantly faster than --release
- args += [
- "-source",
- project.java.sourceCompatibility
- ]
- args += [
- "-target",
- project.java.targetCompatibility
- ]
- args += ["-encoding", "UTF-8"]
- args += ["-proc:none"]
- args += ["-nowarn"]
- args += ["-enableJavadoc"]
- args += [
- "-properties",
- javadocPrefsPath
- ]
-
- // We depend on modular paths.
- def modularPaths = sourceSet.modularPathsForEcj
- dependsOn modularPaths
-
- // Add modular dependencies and their transitive dependencies to module path.
- task.argumentProviders.add(modularPaths.compilationArguments)
-
- // Add classpath, if needed.
- task.argumentProviders.add((CommandLineArgumentProvider) {
- // Add classpath locations in a lazy provider (can't resolve the
- // configuration at evaluation time). Filter out non-existing entries
- // (output folders for non-existing input source dirs like resources).
- FileCollection cpath = modularPaths.compilationClasspath.filter { p -> p.exists() }
- if (!cpath.isEmpty()) {
- return [
- "-classpath",
- cpath.join(File.pathSeparator)
- ]
- } else {
- return []
- }
- })
-
- // Place input files in an external file to dodge command line argument
- // limits. We could pass a directory but ecj seems to be buggy: when it
- // encounters a module-info.java file it no longer compiles other source files.
- def inputsFile = file("${tmpDst}/ecj-inputs.txt")
- task.argumentProviders.add((CommandLineArgumentProvider) {
- return ["@" + inputsFile.absolutePath]
- })
-
- doFirst {
- modularPaths.logCompilationPaths(logger)
-
- tmpDst.mkdirs()
-
- // escape filename accoring to ECJ's rules:
- // https://github.com/eclipse/aspectj.eclipse.jdt.core/blob/a05312e746b9bc2b48b4b039f6e7b5e061b5b393/org.eclipse.jdt.core/batch/org/eclipse/jdt/internal/compiler/batch/Main.java#L1533-L1537
- // Basically surround all whitespace by quotes:
- def escapeFileName = { String s -> s.replaceAll(/ +/, /"$0"/) }
- inputsFile.setText(
- srcDirs.collectMany { dir ->
- project.fileTree(
- dir: dir,
- include: "**/*.java",
- // Exclude the benchmark class with dependencies on nekohtml, which causes module-classpath conflicts and breaks ecj.
- exclude: "**/DemoHTMLParser.java"
- ).files
- }
- // Try to sort all input files; a side-effect of this should be that module-info.java
- // is placed first on the list, which works around ECJ bug:
- // https://bugs.eclipse.org/bugs/show_bug.cgi?id=569833
- .sort()
- .collect {file -> escapeFileName(file.absolutePath.toString())}.join("\n"), "UTF-8")
- }
- })
-}
-
-def ecjLint = tasks.register("ecjLint", {
- description = "Lint Java sources using ECJ."
- group = "Verification"
-
- dependsOn lintTasks
-})
-
-// Attach ecjLint to check.
-tasks.named("check").configure {
- dependsOn ecjLint
-}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.java
new file mode 100644
index 00000000000..8c378f979e0
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/EcjLintPlugin.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.java;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;
+import org.apache.lucene.gradle.plugins.spotless.ApplyGoogleJavaFormatTask;
+import org.eclipse.jdt.internal.compiler.batch.Main;
+import org.gradle.api.GradleException;
+import org.gradle.api.Project;
+import org.gradle.api.Task;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.plugins.JavaPluginExtension;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.process.CommandLineArgumentProvider;
+
+/**
+ * This adds additional linting of Java sources using ECJ (eclipse's compiler).
+ *
+ * @see
+ * "https://help.eclipse.org/latest/index.jsp?topic=%2Forg.eclipse.jdt.doc.user%2Ftasks%2Ftask-using_batch_compiler.htm"
+ */
+public class EcjLintPlugin extends LuceneGradlePlugin {
+
+ /*
+ * Using an explicit dependency means we can't be as flexible in using interim "drops" of ecj,
+ * as previously implemented in lucene.validation.ecj-lint.gradle. Defer the solution
+ * until it really becomes a problem.
+ */
+
+ @Override
+ public void apply(Project project) {
+ requiresAppliedPlugin(project, JavaPlugin.class);
+
+ Path javadocPrefsPath = gradlePluginResource(project, "validation/ecj-lint/ecj.javadocs.prefs");
+
+ // Create a [sourceSetName]EcjLint task for each source set
+ // with a non-empty java.srcDirs. These tasks are then
+ // attached to project's "ecjLint" task.
+ var javaExt = project.getExtensions().getByType(JavaPluginExtension.class);
+ assert javaExt.getSourceCompatibility().equals(javaExt.getTargetCompatibility());
+
+ var sourceSets = javaExt.getSourceSets();
+ var tasks = project.getTasks();
+ List<? extends TaskProvider<?>> lintTasks =
+ sourceSets.stream()
+ .map(
+ sourceSet ->
+ tasks.register(
+ sourceSet.getTaskName("ecjLint", null),
+ task -> configureEcjTask(sourceSet, javaExt, javadocPrefsPath, task)))
+ .toList();
+
+ var ecjLintTask =
+ tasks.register(
+ "ecjLint",
+ task -> {
+ task.setGroup("Verification");
+ task.setDescription("Lint Java sources using ECJ.");
+ task.dependsOn(lintTasks);
+ });
+
+ // Attach ecjLint to check.
+ tasks
+ .named("check")
+ .configure(
+ task -> {
+ task.dependsOn(ecjLintTask);
+ });
+ }
+
+ private void configureEcjTask(
+ SourceSet sourceSet, JavaPluginExtension javaExt, Path javadocPrefsPath, Task task) {
+ // This dependency is on a configuration; technically it causes
+ // all dependencies to be resolved before this task executes
+ // (this includes scheduling tasks that compile the
+ // sources from other projects for example).
+ task.dependsOn(sourceSet.getCompileClasspath());
+
+ task.mustRunAfter(task.getProject().getTasks().withType(ApplyGoogleJavaFormatTask.class));
+
+ // We create a task for all source sets but don't run
+ // tasks for which Java source directories are empty.
+ // We also ignore mr-jar source sets.
+ if (sourceSet.getJava().getAsFileTree().isEmpty() || sourceSet.getName().matches("main\\d+")) {
+ task.setEnabled(false);
+ }
+
+ List<String> args = new ArrayList<>();
+ args.add("-d");
+ args.add("none");
+
+ // Use -source/-target as it is significantly faster than --release
+
+ args.addAll(List.of("-source", javaExt.getSourceCompatibility().toString()));
+ args.addAll(List.of("-target", javaExt.getTargetCompatibility().toString()));
+
+ args.addAll(List.of("-encoding", "UTF-8"));
+ args.add("-proc:none");
+ args.add("-nowarn");
+ args.add("-enableJavadoc");
+ args.addAll(List.of("-properties", javadocPrefsPath.toAbsolutePath().toString()));
+
+ // We depend on modular paths.
+ ModularPathsExtensionApi modularPaths =
+ (ModularPathsExtensionApi) sourceSet.getExtensions().getByName("modularPathsForEcj");
+ task.dependsOn(modularPaths);
+
+ // Collect modular dependencies and their transitive dependencies to module path.
+ CommandLineArgumentProvider compilationArguments = modularPaths.getCompilationArguments();
+ // this isn't exactly right but the returned CommandLineArgumentProvider isn't serializable...
+ task.getInputs().files(sourceSet.getCompileClasspath());
+
+ // Collect classpath locations
+ FileCollection classpathArguments = modularPaths.getCompilationClasspath().filter(File::exists);
+ task.getInputs().files(classpathArguments);
+
+ // Input sources.
+ var inputSources = sourceSet.getJava().getAsFileTree();
+ task.getInputs().files(inputSources);
+
+ // Add all arguments so far as inputs.
+ task.getInputs().property("args", args);
+
+ var argsEchoFile =
+ task.getProject()
+ .getLayout()
+ .getBuildDirectory()
+ .file("ecj/" + task.getName() + "-args.txt");
+
+ var ecjOutput =
+ task.getProject()
+ .getLayout()
+ .getBuildDirectory()
+ .file("ecj/" + task.getName() + "-output.txt");
+
+ task.getOutputs().files(argsEchoFile, ecjOutput);
+
+ task.doFirst(
+ _ -> {
+ List<String> allArgs = new ArrayList<>(args);
+
+ compilationArguments.asArguments().forEach(allArgs::add);
+ if (!classpathArguments.isEmpty()) {
+ allArgs.add("-classpath");
+ allArgs.add(
+ classpathArguments.getFiles().stream()
+ .map(File::toString)
+ .collect(Collectors.joining(File.pathSeparator)));
+ }
+
+ var sources =
+ inputSources.getFiles().stream()
+ .map(File::toString)
+ .filter(
+ file -> {
+ // Exclude the benchmark class with dependencies on nekohtml,
+ // which causes module-classpath conflicts and breaks ecj.
+ return !file.endsWith("DemoHTMLParser.java");
+ })
+ .sorted()
+ .toList();
+
+ allArgs.addAll(sources);
+
+ File ecjOutputFile = ecjOutput.get().getAsFile();
+ try (var out = new PrintWriter(ecjOutputFile, StandardCharsets.UTF_8)) {
+ Files.writeString(argsEchoFile.get().getAsFile().toPath(), String.join("\n", allArgs));
+
+ var success =
+ new Main(out, out, false /* systemExit */, null /* options */, null /* progress */)
+ .compile(allArgs.toArray(String[]::new));
+
+ out.flush();
+ if (!success) {
+ throw new GradleException(
+ "ecj linter failed:\n" + Files.readString(ecjOutputFile.toPath()));
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ });
+ }
+}
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.java
new file mode 100644
index 00000000000..c338b4640f8
--- /dev/null
+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ModularPathsExtensionApi.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle.plugins.java;
+
+import org.gradle.api.file.FileCollection;
+import org.gradle.process.CommandLineArgumentProvider;
+
+/**
+ * This interface should be replaced with ModularPathsExtension, once lucene.java.modules plugin is
+ * converted to Java.
+ */
+public interface ModularPathsExtensionApi {
+ CommandLineArgumentProvider getCompilationArguments();
+
+ FileCollection getCompilationClasspath();
+}
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 59adfec0468..c2c49726af9 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -9,7 +9,7 @@ commons-codec = "1.18.0"
# benchmark/ bzip2 compression
commons-compress = "1.27.1"
# See ecj-lint.gradle (used for more linting).
-ecj = "4.36M1"
+ecj = "3.42.0"
# java linting
errorprone = "2.38.0"
# markdown documentation generation