Apache Lucene open-source search software
Final Gradle plugins to Java (#15377)
Dawid Weiss912e90663980
diff --git a/build-tools/build-infra/build.gradle b/build-tools/build-infra/build.gradleindex 011c050c0cc..258cc71c8e1 100644--- a/build-tools/build-infra/build.gradle+++ b/build-tools/build-infra/build.gradle@@ -17,8 +17,6 @@plugins {id "java-gradle-plugin"- id "groovy-gradle-plugin"-alias(deps.plugins.forbiddenapis) apply false}@@ -40,14 +38,9 @@ gradlePlugin {automatedPublishing = falseplugins {- CheckEnvironmentPlugin {- id = 'lucene.root-project.check-environment'- implementationClass = 'org.apache.lucene.gradle.plugins.misc.CheckEnvironmentPlugin'- }-RootProjectSetupPlugin {id = 'lucene.root-project.setup'- implementationClass = 'org.apache.lucene.gradle.plugins.misc.RootProjectSetupPlugin'+ implementationClass = 'org.apache.lucene.gradle.plugins.misc.LuceneRootConfigurationPlugin'}}}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradle b/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradledeleted file mode 100644index cfcffadbb00..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.antlr.gradle+++ /dev/null@@ -1,106 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--//-// Regenerate antlr4 grammars.-//--configure(project(":lucene:expressions")) {- configurations {- antlr- }-- dependencies {- antlr deps.antlr.core- }-- tasks.register("regenerateAntlr", {- description = "Regenerate Javascript.g4"-- dependsOn configurations.antlr-- def generatedPatterns = [- "**/JavascriptBaseVisitor.java",- "**/JavascriptLexer.java",- "**/JavascriptParser.java",- "**/JavascriptVisitor.java",- ]-- def sourceDir = "src/java/org/apache/lucene/expressions/js"- def inputFiles = file("$sourceDir/Javascript.g4")- def tempOutput = layout.buildDirectory.dir("antlr").get().asFile- def outputFiles = fileTree(dir: sourceDir, includes: generatedPatterns)-- inputs.property "antlr-version", deps.antlr.core.get().version- inputs.files inputFiles- outputs.files outputFiles-- doFirst {- buildGlobals.getFileOps().delete(tempOutput)- buildGlobals.getExecOps().javaexec {- mainClass = "org.antlr.v4.Tool"- classpath = configurations.antlr-- ignoreExitValue = false- args = [- "-no-listener",- "-visitor",- "-package",- "org.apache.lucene.expressions.js",- "-o",- tempOutput,- inputFiles- ]- }-- def generatedFiles = fileTree(dir: tempOutput, includes: generatedPatterns)- generatedFiles.each {file ->- buildGlobals.modifyFile(file, {text ->- text = text.replaceAll("public ((interface|class) Javascript\\w+)", "\$1")- text = text.replaceAll("// Generated from .*",- """-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--// ANTLR GENERATED CODE: DO NOT EDIT.-""")- return text- })- }-- buildGlobals.getFileOps().copy({CopySpec spec ->- spec.from(tempOutput)- spec.into(sourceDir)- spec.include(generatedPatterns)- })- }- })-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradle b/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradledeleted file mode 100644index 71d2a603a1d..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.forUtil.gradle+++ /dev/null@@ -1,78 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import org.apache.lucene.gradle.plugins.misc.QuietExec--//-// Regenerates "ForUtil.java" code used by codecs.-//--void addGenerateFromPythonTask(Project project, File genDir, String taskName, String pyScript, String javaFile) {- project.getTasks().register(- "regenerate" + taskName, QuietExec, {- description = "Regenerate " + javaFile + " using " + pyScript + " in " + genDir-- def genScript = project.file("${genDir}/${pyScript}")- def genOutput = project.file("${genDir}/${javaFile}")- inputs.file genScript- outputs.file genOutput-- workingDir = genDir- executable = buildGlobals.externalTool("python3")- args = ['-B', genScript.toString()]- })-}--configure(project(":lucene:core")) {- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/codecs/lucene104"), "ForUtil", "gen_ForUtil.py", "ForUtil.java")-}--configure(project(":lucene:backward-codecs")) {- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene84"), "ForUtil84", "gen_ForUtil.py", "ForUtil.java")-- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene90"), "ForUtil90", "gen_ForUtil.py", "ForUtil.java")-- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene99"), "ForUtil99", "gen_ForUtil.py", "ForUtil.java")-- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene912"), "ForUtil912", "gen_ForUtil.py", "ForUtil.java")- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene912"),- "ForDeltaUtil912",- "gen_ForDeltaUtil.py",- "ForDeltaUtil.java")-- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene101"), "ForUtil101", "gen_ForUtil.py", "ForUtil.java")- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene101"),- "ForDeltaUtil101",- "gen_ForDeltaUtil.py",- "ForDeltaUtil.java")-- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene103"), "ForUtil103", "gen_ForUtil.py", "ForUtil.java")- addGenerateFromPythonTask(project,- file("src/java/org/apache/lucene/backward_codecs/lucene103"),- "ForDeltaUtil103",- "gen_ForDeltaUtil.py",- "ForDeltaUtil.java")-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.icu.gradle b/build-tools/build-infra/src/main/groovy/regenerate.icu.gradledeleted file mode 100644index 63c5e3e5ea3..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.icu.gradle+++ /dev/null@@ -1,310 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import org.apache.tools.ant.taskdefs.condition.Os-import java.nio.file.Files--def resources = rootProject.file("gradle/regenerate/icu")--/*- * Regenerates ICU-related data files.- *- * This build file contains regeneration code utilizing both icu4j and icu4c.- *- * The icu4c version must match exactly the icu4j version in version.props:- * The one on your system is probably different. This script will attempt to- * download and compile a matching icu4c version automatically.- */---def setupIcuDependencies = { proj ->- // Configure different icu4j dependencies.- proj.configurations {- icu_current- groovy- }-- proj.dependencies {- icu_current deps.icu4j- groovy deps.groovy- }-}--def icu4jVersion = deps.icu4j.get().version--configure(project(":lucene:analysis:icu")) {- setupIcuDependencies(project)- def utr30DataDir = file("src/data/utr30")-- def icuBuildDir = file("${buildDir}/icu")- def icuBinDir- def gennorm- def icupkg-- if (Os.isFamily(Os.FAMILY_WINDOWS)) {- icuBinDir = file("${icuBuildDir}/bin64")- gennorm = file("${icuBinDir}/gennorm2.exe")- icupkg = file("${icuBinDir}/icupkg.exe")- } else {- icuBinDir = file("${icuBuildDir}/icu/source/bin")- gennorm = file("${icuBinDir}/gennorm2")- icupkg = file("${icuBinDir}/icupkg")- }-- def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"-- tasks.register("regenerateUtr30DataFiles", {- description = "Regenerate utf30 norm files using ICU"-- def icuConfig = configurations.icu_current-- dependsOn icuConfig- dependsOn icuCompileTask-- regenerate {- ifSkippedAlsoSkip = [icuCompileTask]- }-- // May be undefined yet, so use a provider.- dependsOn { sourceSets.tools.runtimeClasspath }-- // gennorm generates file order-dependent output, so make it constant here.- def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt")- .asList().toSorted(Comparator.comparing { File f -> f.name })- def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")-- inputs.files inputFiles- inputs.property "icuConfig", icu4jVersion- outputs.file outputFile-- doFirst {- // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm- buildGlobals.execOps.javaexec {- mainClass = "org.apache.lucene.analysis.icu.GenerateUTR30DataFiles"- classpath = sourceSets.tools.runtimeClasspath-- ignoreExitValue = false- workingDir utr30DataDir- args = [- "release-${icu4jVersion.replace(".", "-")}"- ]- }-- buildGlobals.quietExec(it, {- executable gennorm- args = [- "-v",- "-s",- utr30DataDir,- "-o",- "${buildDir}/utr30.tmp",- *(inputFiles.collect { it.name })- ]- })-- buildGlobals.quietExec(it, {- executable icupkg- args = [- "-tb",- "${buildDir}/utr30.tmp",- outputFile- ]- })- }- })-- tasks.register("regenerateRbbi", {- description = "Regenerate rbbi rules using ICU"-- def icuConfig = configurations.icu_current-- dependsOn icuConfig-- // May be undefined yet, so use a provider.- dependsOn { sourceSets.tools.runtimeClasspath }-- def sourceDir = file("src/data/uax29")- def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")-- inputs.files fileTree(dir: sourceDir, include: "*.rbbi")- inputs.property "icuConfig", icu4jVersion- outputs.files fileTree(dir: targetDir, include: "*.brk")-- doFirst {- buildGlobals.execOps.javaexec {- mainClass = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"- classpath = sourceSets.tools.runtimeClasspath-- ignoreExitValue = false- enableAssertions true- args = [sourceDir, targetDir]- }- }- })-- tasks.register("compileIcuWindows", {- doFirst {- def v = icu4jVersion- def icuBinZip = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}.zip")- if (!icuBinZip.exists()) {- icuBuildDir.mkdirs()-- // Download binaries matching icu4j version in version.props- def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-Win64-MSVC2019.zip")-- logger.lifecycle("Trying to download binary ICU version: ${v} from:\n ${src}")- Files.write(icuBinZip.toPath(), src.toURL().openStream().bytes)- logger.lifecycle("Downloaded ${icuBinZip.size()} bytes.")- }-- // Unzip.- buildGlobals.fileOps.copy({- into icuBuildDir- from zipTree(icuBinZip)- })- }- })-- tasks.register("compileIcuLinux", {- doFirst {- if (Os.isFamily(Os.FAMILY_WINDOWS)) {- throw new GradleException("ICU compilation not supported on Windows.")- }-- def v = icu4jVersion- def icuSrcTgz = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}-src.tgz")-- // Download sources for version matching icu4j version in version.props- if (!icuSrcTgz.exists()) {- icuBuildDir.mkdirs()- def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-src.tgz")- logger.lifecycle("Trying to download and compile ICU version: ${v} from:\n ${src}")- Files.write(icuSrcTgz.toPath(), src.toURL().openStream().bytes)- logger.lifecycle("Downloaded ${icuSrcTgz.size()} bytes.")- }-- def icuSrcDir = file("${icuBuildDir}/icu/source")- buildGlobals.fileOps.delete(icuSrcDir)-- // Extract the tgz- buildGlobals.quietExec(it, {- executable "tar"- workingDir icuBuildDir- args = [- "-zxf",- icuSrcTgz- ]- })-- // Compile: (cd icu/source && ./configure --prefix=$(pwd) --enable-rpath && make -j4)- buildGlobals.quietExec(it, {- executable "sh"-- workingDir icuSrcDir- environment("CFLAGS", "-O0")- environment("CXXFLAGS", "-O0")- args = [- "configure",- "--prefix=${icuSrcDir}",- "--enable-rpath"- ]- })-- buildGlobals.quietExec(it, {- executable "make"- workingDir icuSrcDir- args = [- "-j${buildOptions['tests.jvms'].get()}"- ]- })-- // Test that the binaries work: derb -V- logger.lifecycle("Compiled ICU, checking...")- buildGlobals.quietExec(it, {- executable "./derb"- workingDir icuBinDir- args = ["-V"]- })- }- })-}---// Regenerates UnicodeProps.java-configure(project(":lucene:analysis:common")) {- setupIcuDependencies(project)-- tasks.register("regenerateUnicodeProps", {- def icuConfig = configurations.icu_current- def groovyConfig = configurations.groovy-- def outputFile = file("src/java/org/apache/lucene/analysis/util/UnicodeProps.java")-- description = "Regenerate ${outputFile} using ICU"-- dependsOn icuConfig-- inputs.property "icuConfig", icu4jVersion- outputs.file outputFile-- doFirst {- buildGlobals.execOps.javaexec({ JavaExecSpec spec ->- spec.mainClass = "groovy.lang.GroovyShell"- spec.classpath(icuConfig, groovyConfig)-- spec.args([- "--encoding",- "UTF-8",- file("${resources}/GenerateUnicodeProps.groovy").toString(),- outputFile.toString()- ])- })- }- })-}--// Regenerates CaseFolding.java-configure(project(":lucene:core")) {- setupIcuDependencies(project)-- tasks.register("generateUnicodeProps", {- def icuConfig = configurations.icu_current- def groovyConfig = configurations.groovy- def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")-- description = "Regenerate ${outputFile} using ICU"-- dependsOn icuConfig-- inputs.property "icuConfig", icu4jVersion- outputs.file outputFile-- doFirst {- buildGlobals.execOps.javaexec({- mainClass = "groovy.lang.GroovyShell"- classpath icuConfig, groovyConfig-- args = [- "--encoding",- "UTF-8",- file("${resources}/GenerateCaseFolding.groovy"),- outputFile- ]- })- }- })-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradle b/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradledeleted file mode 100644index 73b6bf88f34..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.javacc.gradle+++ /dev/null@@ -1,329 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import java.nio.charset.Charset-import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension---// This adds javacc generation support.--configure(rootProject) {- tasks.register("javacc", {- description = "Regenerate sources for corresponding javacc grammar files."- group = "generation"-- dependsOn allprojects.collect { prj -> prj.tasks.withType(JavaCCTask) }- })-}--def commonCleanups = { FileTree generatedFiles ->- // This is a minor typo in a comment that nonetheless people have hand-corrected in the past.- generatedFiles.matching({ include "CharStream.java" }).each {file ->- buildGlobals.modifyFile(file, { text ->- return text.replace(- "implemetation",- "implementation");- })- }-- generatedFiles.each {file ->- buildGlobals.modifyFile(file, { text ->- // Normalize EOLs and tabs (EOLs are a side-effect of modifyFile).- text = text.replace("\t", " ");- text = text.replaceAll("JavaCC - OriginalChecksum=[^*]+", "(filtered)")- text = text.replace("StringBuffer", "StringBuilder")- return text- })- }-- generatedFiles.matching({ include "*TokenManager.java" }).each { file ->- buildGlobals.modifyFile(file, { text ->- // Remove redundant imports.- text = text.replaceAll(- /(?m)^import .+/,- "")- // Add CharStream imports.- text = text.replaceAll(- /package (.+)/,- '''- package $1- import org.apache.lucene.queryparser.charstream.CharStream;- '''.trim())- // Eliminates redundant cast message.- text = text.replace(- "int hiByte = (int)(curChar >> 8);",- "int hiByte = curChar >> 8;")- // Access to forbidden APIs.- text = text.replace(- "public java.io.PrintStream debugStream = System.out;",- "// (debugStream omitted).")- text = text.replace(- "public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }",- "// (setDebugStream omitted).")- // Remove dangling doc comments- text = text.replace(- "/** Debug output. */",- "// Debug output.")- text = text.replace(- "/** Set debug output. */",- "// Set debug output.")- return text- })- }-}--configure(project(":lucene:queryparser")) {- configurations.create("javacc")-- dependencies {- add("javacc", deps.javacc)- }-- tasks.register("regenerateClassicQueryParser", JavaCCTask, {- description = "Regenerate classic query parser from lucene/queryparser/classic/QueryParser.jj"-- javaccFile = file('src/java/org/apache/lucene/queryparser/classic/QueryParser.jj')-- afterGenerate << commonCleanups- afterGenerate << { FileTree generatedFiles ->- generatedFiles.matching { include "QueryParser.java" }.each { file ->- buildGlobals.modifyFile(file, { text ->- text = text.replace(- "public QueryParser(CharStream ",- "protected QueryParser(CharStream ")- text = text.replace(- "public QueryParser(QueryParserTokenManager ",- "protected QueryParser(QueryParserTokenManager ")- text = text.replace(- "new java.util.ArrayList<int[]>",- "new java.util.ArrayList<>")- text = text.replace(- "final private LookaheadSuccess jj_ls =",- "static final private LookaheadSuccess jj_ls =")- text = text.replace(- "public class QueryParser ",- '@SuppressWarnings({"unused","null"}) public class QueryParser ')- text = text.replace(- "final public Query TopLevelQuery(",- "@Override final public Query TopLevelQuery(")- text = text.replace(- "public void ReInit(CharStream ",- "@Override public void ReInit(CharStream ")- return text- })- }- }- })-- tasks.register("regenerateSurroundQueryParser", JavaCCTask, {- description = "Regenerate surround query parser from lucene/queryparser/surround/parser/QueryParser.jj"-- javaccFile = file('src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj')-- afterGenerate << commonCleanups- afterGenerate << { FileTree generatedFiles ->- generatedFiles.matching { include "QueryParser.java" }.each { file ->- buildGlobals.modifyFile(file, { text ->- text = text.replace(- "import org.apache.lucene.analysis.TokenStream;",- "")- text = text.replace(- "new java.util.ArrayList<int[]>",- "new java.util.ArrayList<>")- text = text.replace(- "public class QueryParser ",- '@SuppressWarnings({"unused","null"}) public class QueryParser ')- return text- })- }- }- })-- tasks.register("regenerateFlexibleStandardSyntaxParser", JavaCCTask, {- description = "Regenerate flexible query parser from queryparser/flexible/standard/parser/StandardSyntaxParser.jj"-- javaccFile = file('src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj')-- afterGenerate << commonCleanups- afterGenerate << { FileTree generatedFiles ->- generatedFiles.matching { include "ParseException.java" }.each { file ->- buildGlobals.modifyFile(file, { text ->- // Modify constructor.- text = text.replace(- "class ParseException extends Exception",- "class ParseException extends QueryNodeParseException")-- // Modify imports.- text = text.replace(- "package org.apache.lucene.queryparser.flexible.standard.parser;", '''\- package org.apache.lucene.queryparser.flexible.standard.parser;-- import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;- import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;- import org.apache.lucene.queryparser.flexible.messages.Message;- import org.apache.lucene.queryparser.flexible.messages.MessageImpl;- ''')-- // Modify constructors and code bits- text = text.replaceAll(- /(?s)[ ]*public ParseException\(Token currentTokenVal[^}]+[}]/, '''\- public ParseException(Token currentTokenVal,- int[][] expectedTokenSequencesVal, String[] tokenImageVal)- {- super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(- currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));- this.currentToken = currentTokenVal;- this.expectedTokenSequences = expectedTokenSequencesVal;- this.tokenImage = tokenImageVal;- }- ''')-- text = text.replaceAll(- /(?s)[ ]*public ParseException\(String message\)[^}]+[}]/, '''\- public ParseException(Message message)- {- super(message);- }- ''')-- text = text.replaceAll(- /(?s)[ ]*public ParseException\(\)[^}]+[}]/, '''\- public ParseException()- {- super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));- }- ''')- return text- })- }-- generatedFiles.matching { include "StandardSyntaxParser.java" }.each { file ->- buildGlobals.modifyFile(file, { text ->- // Remove redundant cast- text = text.replace(- "new java.util.ArrayList<int[]>",- "new java.util.ArrayList<>")- text = text.replace(- "new ArrayList<QueryNode>()",- "new ArrayList<>()")- text = text.replace(- "Collections.<QueryNode> singletonList",- "Collections.singletonList")- text = text.replace(- "public class StandardSyntaxParser ",- '@SuppressWarnings({"unused","null"}) public class StandardSyntaxParser ')- return text- })- }- }- })-}--// We always regenerate, no need to declare outputs.-class JavaCCTask extends DefaultTask {- @InputFile- File javaccFile-- /**- * Apply closures to all generated files before they're copied back- * to mainline code.- */- // A subtle bug here is that this makes it not an input... should be a list of replacements instead?- @Internal- List<Closure<FileTree>> afterGenerate = new ArrayList<>()-- @OutputFiles- List<File> getGeneratedSources() {- // Return the list of generated files.- def baseDir = javaccFile.parentFile- def baseName = javaccFile.name.replace(".jj", "")-- return [- project.file("${baseDir}/${baseName}.java"),- project.file("${baseDir}/${baseName}Constants.java"),- project.file("${baseDir}/${baseName}TokenManager.java"),- project.file("${baseDir}/ParseException.java"),- project.file("${baseDir}/Token.java"),- project.file("${baseDir}/TokenMgrError.java")- ]- }-- JavaCCTask() {- dependsOn(project.configurations.javacc)- }-- @TaskAction- def generate() {- if (!javaccFile || !javaccFile.exists()) {- throw new GradleException("Input file does not exist: ${javaccFile}")- }-- def buildGlobals = project.getExtensions().getByType(LuceneBuildGlobalsExtension)-- // Run javacc generation into temporary folder so that we know all the generated files- // and can post-process them easily.- def tempDir = this.getTemporaryDir()- tempDir.mkdirs()- buildGlobals.fileOps.delete(project.fileTree(tempDir, { include: "**/*.java" }))-- def targetDir = javaccFile.parentFile- logger.lifecycle("Recompiling JavaCC: ${project.rootDir.relativePath(javaccFile)}")-- def output = new ByteArrayOutputStream()- def result = buildGlobals.execOps.javaexec({- classpath {- project.configurations.javacc- }-- ignoreExitValue = true- standardOutput = output- errorOutput = output-- mainClass = "org.javacc.parser.Main"- args += [- "-OUTPUT_DIRECTORY=${tempDir}",- javaccFile- ]- })-- // Unless we request verbose logging, don't emit javacc output.- if (result.exitValue != 0) {- throw new GradleException("JavaCC failed to compile ${javaccFile}, here is the compilation output:\n${output}")- }-- // Make sure we don't have warnings.- if (output.toString(Charset.defaultCharset()).contains("Warning:")) {- throw new GradleException("JavaCC emitted warnings for ${javaccFile}, here is the compilation output:\n${output}")- }-- // Apply any custom modifications.- def generatedFiles = project.fileTree(tempDir)-- afterGenerate.each {closure ->- closure.call(generatedFiles)- }-- // Copy back to mainline sources.- buildGlobals.fileOps.copy({ CopySpec spec ->- spec.from tempDir- spec.into targetDir-- // We don't need CharStream interface as we redirect to our own.- spec.exclude "CharStream.java"- })- }-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradle b/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradledeleted file mode 100644index 854f9f37c7e..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.jflex.gradle+++ /dev/null@@ -1,275 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import org.apache.lucene.gradle.plugins.misc.QuietExec-import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension--// Add a top-level pseudo-task to which we will attach individual regenerate tasks.--def resources = rootProject.file("gradle/regenerate/jflex")-def skeletonDefault = file("${resources}/skeleton.default.txt")-def skeletonNoBufferExpansion = file("${resources}/skeleton.disable.buffer.expansion.txt")--configure(project(":lucene:core")) {- configurations {- jflex- }-- dependencies {- jflex deps.jflex- }-- tasks.register("regenerateStandardTokenizer", JFlexTask, {- description = "Regenerate StandardTokenizerImpl.java"-- jflexFile = file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex')- skeleton = skeletonNoBufferExpansion-- doLast {- ant.replace(- file: file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java'),- encoding: "UTF-8",- token: "private static final int ZZ_BUFFERSIZE =",- value: "private int ZZ_BUFFERSIZE ="- )- }- })-}--configure(project(":lucene:analysis:common")) {- configurations {- jflex- }-- dependencies {- jflex deps.jflex- }-- def regenerateTlds = tasks.register("regenerateTlds", {- def tldZones = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"- def jflexMacro = file("src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex")- def tldList = file("src/test/org/apache/lucene/analysis/email/TLDs.txt")-- description = "Regenerate top-level domain jflex macros and tests"-- dependsOn { sourceSets.tools.runtimeClasspath }-- inputs.property "tldZones", tldZones- outputs.files jflexMacro, tldList-- doFirst {- File tmpJflexMacro = File.createTempFile(jflexMacro.getName(), ".tmp", getTemporaryDir())- File tmpTldList = File.createTempFile(tldList.getName(), ".tmp", getTemporaryDir())-- project.extensions.buildGlobals.execOps.javaexec {- mainClass = "org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"- classpath = sourceSets.tools.runtimeClasspath-- ignoreExitValue = false- args = [- tldZones,- tmpJflexMacro,- tmpTldList- ]- }-- // LUCENE-9926: tldZones is regenerated daily. Compare the generated content (excluding comments) so that- // we only update actual output files if non-comments have changed.- def contentLines = { File file ->- if (file.exists()) {- List<String> lines = file.readLines("UTF-8")- lines.removeIf { line -> line.isBlank() || line.startsWith("//") }- return lines- } else {- return []- }- }-- if (contentLines(tmpTldList).equals(contentLines(tldList))) {- logger.lifecycle("Generated TLD content identical as before, not updating.")- } else {- tldList.setBytes tmpTldList.bytes- jflexMacro.setBytes tmpJflexMacro.bytes- logger.lifecycle("You've regenerated the TLD include file, remember to regenerate UAX29URLEmailTokenizerImpl too.")- }- }- })-- tasks.register("regenerateWikipediaTokenizer", JFlexTask, {- description = "Regenerate WikipediaTokenizerImpl.java"-- jflexFile = file('src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex')- skeleton = skeletonDefault- })-- tasks.register("regenerateClassicTokenizer", JFlexTask, {- description = "Regenerate ClassicTokenizerImpl.java"-- jflexFile = file('src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex')- skeleton = skeletonDefault- })-- tasks.register("regenerateUAX29URLEmailTokenizer", JFlexTask, {- description = "Regenerate UAX29URLEmailTokenizerImpl.java"-- // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex- // so we make sure it is up to date. Also, make sure the ordering of internal- // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).- dependsOn regenerateTlds-- jflexFile = file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex')- skeleton = skeletonNoBufferExpansion- heapSize = "12g"-- // Add included files as inputs.- inputs.file file('src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex')-- doFirst {- logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).")- }-- doLast {- ant.replace(- file: file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java'),- encoding: "UTF-8",- token: "private static final int ZZ_BUFFERSIZE =",- value: "private int ZZ_BUFFERSIZE ="- )- }- })-- def generateHTMLCharacterEntities = tasks.register("regenerateHTMLCharacterEntities", QuietExec, {- description = "Regenerate HTMLCharacterEntities.jflex"-- def target = file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')- def script = file("${resources}/htmlentity.py")-- outputs.files target- inputs.file script-- executable = buildGlobals.externalTool("python3")- workingDir = target.parentFile- args += [- "-B",- // don't write any bytecode cache- script,- target- ]-- doLast {- project.ant.fixcrlf(- file: target,- encoding: "UTF-8",- eol: "lf"- )- }- })-- tasks.register("regenerateHTMLStripCharFilter", JFlexTask, {- description = "Regenerate HTMLStripCharFilter.java"-- // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.- dependsOn generateHTMLCharacterEntities-- // Add included files as inputs.- inputs.file file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')-- jflexFile = file('src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex')- skeleton = skeletonDefault- })-}--class JFlexTask extends DefaultTask {- @InputFile- File jflexFile-- @InputFile- File skeleton-- @Internal- String heapSize-- @OutputFile- File getGeneratedFile() {- return project.file(jflexFile.absolutePath.replace(".jflex", ".java"))- }-- JFlexTask() {- dependsOn(project.configurations.jflex)- }-- @TaskAction- def generate() {- if (!jflexFile || !jflexFile.exists()) {- throw new GradleException("JFlex file does not exist: ${jflexFile}")- }-- def target = project.file(jflexFile.absolutePath.replace(".jflex", ".java"))-- logger.lifecycle("Recompiling JFlex: ${project.rootDir.relativePath(jflexFile)}")-- project.extensions.buildGlobals.execOps.javaexec {- classpath {- project.configurations.jflex- }-- mainClass = "jflex.Main"- args += [- "-nobak",- "--quiet",- "--encoding",- "UTF-8",- ]-- if (heapSize) {- maxHeapSize = heapSize- }-- if (skeleton) {- args += [- "--skel",- skeleton.absolutePath- ]- }-- args += [- "-d",- target.parentFile.absolutePath,- jflexFile- ]- }-- // fix invalid SuppressWarnings from jflex, so that it works with javac.- // we also need to suppress 'unused' because of dead code, so it passes ecjLint- // you can't "stack" SuppressWarnings annotations, jflex adds its own, this is the only way- // https://github.com/jflex-de/jflex/issues/762- project.ant.replace(- file: target,- encoding: "UTF-8",- token: 'SuppressWarnings("fallthrough")',- value: 'SuppressWarnings({"fallthrough","unused"})'- )-- // Seems like a bug in gjf. https://github.com/google/google-java-format/issues/1260- project.ant.replace(- file: target,- encoding: "UTF-8",- token: '/* Break so we don\'t hit fall-through warning: */',- value: ''- )- }-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradle b/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradledeleted file mode 100644index 7c55c357ef7..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.kuromoji.gradle+++ /dev/null@@ -1,143 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-import de.undercouch.gradle.tasks.download.Download--// This downloads and compiles Kuromoji dictionaries.--def recompileDictionary(project, dictionaryName, Closure closure) {- buildGlobals.execOps.javaexec {- mainClass = "org.apache.lucene.analysis.ja.dict.DictionaryBuilder"- classpath = project.sourceSets.main.runtimeClasspath-- jvmArgs '-Xmx1G'-- with closure- }- project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")-}--configure(project(":lucene:analysis:kuromoji")) {- plugins.withType(JavaPlugin).configureEach {- project.ext {- targetDir = project.file("src/resources")- }-- def mecabDictionaryName = "mecab-ipadic-2.7.0-20070801"- // original source: "https://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/${mecabDictionaryName}.tar.gz"- def mecabDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${mecabDictionaryName}.tar.gz"-- def compileMecab = tasks.register("compileMecab", Download, {- dependsOn sourceSets.main.runtimeClasspath-- def dictionaryFile = file("${buildDir}/generate/${mecabDictionaryName}.tar.gz")- def unpackedDir = file("${buildDir}/generate/${mecabDictionaryName}")-- src mecabDictionarySource- dest dictionaryFile- onlyIfModified true-- doLast {- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")- })-- // Unpack the downloaded archive.- delete unpackedDir- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {- ant.cutdirsmapper(dirs: "1")- }-- // Apply patch via local git.- buildGlobals.quietExec(it, {- workingDir = unpackedDir- executable "git"- args += [- "apply",- project.file("src/tools/patches/Noun.proper.csv.patch").absolutePath- ]- })-- // Compile the dictionary- recompileDictionary(project, mecabDictionaryName, {- args += [- "ipadic",- unpackedDir,- targetDir,- "euc-jp",- false- ]- })- }- })-- tasks.register("regenerateDictionariesFromMecab", {- description = "Regenerate dictionaries from Mecab data"-- dependsOn compileMecab-- inputs.property("dictionarySource", mecabDictionarySource)- outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ja/dict", include: "*.dat"))-- regenerate {- // No need to run any follow-ups, we generate binary files (not sources).- followedUpBy = []- ifSkippedAlsoSkip = ["compileMecab"]- }- })-- // TODO: this regenerates data files over the same files as mecab, above. This seems odd and inconsistent?-- // original source (down): https://rwthaachen.dl.osdn.jp/naist-jdic/53500/mecab-naist-jdic-0.6.3b-20111013.tar.gz- def naistDictionaryName = "mecab-naist-jdic-0.6.3b-20111013"- def naistDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${naistDictionaryName}.tar.gz"- tasks.register("compileNaist", Download, {- description = "Recompile dictionaries from Naist data."-- dependsOn sourceSets.main.runtimeClasspath-- def dictionaryFile = file("${buildDir}/generate/${naistDictionaryName}.tar.gz")- def unpackedDir = file("${buildDir}/generate/${naistDictionaryName}")-- src naistDictionarySource- dest dictionaryFile- onlyIfModified true-- doLast {- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")- })-- // Unpack the downloaded archive.- delete unpackedDir- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {- ant.cutdirsmapper(dirs: "1")- }-- // Compile the dictionary- recompileDictionary(project, naistDictionaryName, {- args += [- "ipadic",- unpackedDir,- targetDir,- "euc-jp",- false- ]- })- }- })- }-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.moman.gradle b/build-tools/build-infra/src/main/groovy/regenerate.moman.gradledeleted file mode 100644index 936e34c0c8f..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.moman.gradle+++ /dev/null@@ -1,114 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-import de.undercouch.gradle.tasks.download.Download--def resources = rootProject.file("gradle/regenerate/moman")--configure(project(":lucene:core")) {-- tasks.register("regeneratePacked64SingleBlock", {- description("Regenerate Packed64SingleBlock and BulkOperation*.java")-- def targetDir = file("src/java/org/apache/lucene/util/packed")-- def templateFiles = [- file("${resources}/gen_BulkOperation.py"),- file("${resources}/gen_Packed64SingleBlock.py")- ]-- inputs.files templateFiles- outputs.files fileTree(dir: targetDir, includes: [- "Packed64SingleBlock.java",- "BulkOperation*.java"- ])-- doLast {- templateFiles.each {prog ->- logger.lifecycle("Executing: ${prog} in ${targetDir}")- buildGlobals.quietExec(it, {- workingDir = targetDir- executable = buildGlobals.externalTool("python3")- args = ['-B', "${prog}"]- })- }- // Correct line endings for Windows.- project.ant.fixcrlf(- srcDir: targetDir,- includes: 'Packed64SingleBlock.java, BulkOperation*.java',- encoding: 'UTF-8',- eol: 'lf'- )- }- })-- def momanSource = "https://github.com/jpbarrette/moman/archive/497c90e34e412b6494db6dabf0d95db8034bd325.zip"- def momanDir = file("${buildDir}/moman")-- def installMoman = tasks.register("installMoman", Download, {- def momanZip = file("${momanDir}/moman.zip")-- src momanSource- dest momanZip- onlyIfModified true-- doLast {- ant.unzip(src: momanZip, dest: momanDir, overwrite: "true") {- ant.cutdirsmapper(dirs: "1")- }- }- })-- tasks.register("regenerateParametricDescription", {- dependsOn installMoman-- description("Regenerate *ParametricDescription")- regenerate {- ifSkippedAlsoSkip = ["installMoman"]- }-- def targetDir = file("src/java/org/apache/lucene/util/automaton")- def templateFile = file("${resources}/createLevAutomata.py")-- inputs.property("source", momanSource)- inputs.file templateFile- outputs.files fileTree(dir: targetDir, includes: ["*ParametricDescription.java"])-- doLast {- ['1', '2'].each {num ->- ['True', 'False'].each {transpose ->- buildGlobals.quietExec(it, {- workingDir = targetDir- executable = buildGlobals.externalTool("python3")- args = [- '-B',- templateFile.toString(),- num,- transpose,- "${momanDir}/finenight/python"- ]- })- }- }- project.ant.fixcrlf(- srcDir: targetDir,- includes: '*ParametricDescription.java',- encoding: 'UTF-8',- eol: 'lf'- )- }- })-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.nori.gradle b/build-tools/build-infra/src/main/groovy/regenerate.nori.gradledeleted file mode 100644index 155ce3884ad..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.nori.gradle+++ /dev/null@@ -1,90 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-import de.undercouch.gradle.tasks.download.Download--// This downloads and compiles Nori dictionaries.--def recompileDictionary(project, dictionaryName, Closure closure) {- buildGlobals.execOps.javaexec {- mainClass = "org.apache.lucene.analysis.ko.dict.DictionaryBuilder"- classpath = project.sourceSets.main.runtimeClasspath-- jvmArgs '-Xmx1G'-- with closure- }- project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")-}--configure(project(":lucene:analysis:nori")) {- plugins.withType(JavaPlugin).configureEach {- project.ext {- targetDir = project.file("src/resources")- }-- def dictionaryName = "mecab-ko-dic-2.1.1-20180720"- def dictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${dictionaryName}.tar.gz"- def compileMecabKo = tasks.register("compileMecabKo", Download, {-- dependsOn sourceSets.main.runtimeClasspath-- def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")- def unpackedDir = file("${buildDir}/generate/${dictionaryName}")-- src dictionarySource- dest dictionaryFile- onlyIfModified true-- doLast {- buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->- fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")- })-- // Unpack the downloaded archive.- delete unpackedDir- ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {- ant.cutdirsmapper(dirs: "1")- }-- // Compile the dictionary- recompileDictionary(project, dictionaryName, {- args += [- unpackedDir,- targetDir,- "utf-8",- false- ]- })- }- })-- tasks.register("regenerateDictionariesFromMecabKo", {- description = "Recompile dictionaries from Mecab-Ko data."-- dependsOn compileMecabKo-- inputs.property("dictionarySource", dictionarySource)- outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ko/dict", include: "*.dat"))-- regenerate {- // No need to run any follow-ups, we generate binary files (not sources).- followedUpBy = []- ifSkippedAlsoSkip = ["compileMecabKo"]- }- })- }-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradle b/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradledeleted file mode 100644index 185a7f2bfa1..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.snowball.gradle+++ /dev/null@@ -1,115 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-import org.apache.lucene.gradle.plugins.misc.QuietExec-import de.undercouch.gradle.tasks.download.Download-import org.apache.tools.ant.taskdefs.condition.Os--def resources = rootProject.file("gradle/regenerate/snowball")--configure(project(":lucene:analysis:common")) {- // git commit hash of source code https://github.com/snowballstem/snowball/- def snowballStemmerCommit = "34f3612e5e8c48975243bc2e87561abdac5aa9bb"- // git commit hash of stopwords https://github.com/snowballstem/snowball-website- def snowballWebsiteCommit = "424fd1f75044160bb4a92f1daf08cce618459374"-- def snowballWorkDir = file("${buildDir}/snowball")-- def snowballStemmerDir = file("${snowballWorkDir}/stemmers-${snowballStemmerCommit}")- def snowballWebsiteDir = file("${snowballWorkDir}/website-${snowballWebsiteCommit}")-- def snowballScript = file("${resources}/snowball.sh")-- def unpackFromZip = { zipFile, targetDir ->- buildGlobals.fileOps.sync({SyncSpec spec ->- spec.from(zipTree(zipFile), {- eachFile { fcd ->- fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))- }- })- spec.into targetDir- })- }-- // downloads snowball stemmers (or use a cached copy)- def downloadSnowballStemmers = tasks.register("downloadSnowballStemmers", Download, {- src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip"- dest file("${snowballStemmerDir}.zip")- overwrite false- tempAndMove true-- doLast {- unpackFromZip(dest, snowballStemmerDir)- }- })-- // downloads snowball website (or use a cached copy)- def downloadSnowballWebsite = tasks.register("downloadSnowballWebsite", Download, {- src "https://github.com/snowballstem/snowball-website/archive/${snowballWebsiteCommit}.zip"- def snowballWebsiteZip = file("${snowballWebsiteDir}.zip")- dest snowballWebsiteZip- overwrite false- tempAndMove true-- doLast {- unpackFromZip(snowballWebsiteZip, snowballWebsiteDir)- }- })-- // runs a shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.- tasks.register("regenerateSnowball", QuietExec, {- description = "Regenerate snowball stemmers."-- dependsOn downloadSnowballStemmers- dependsOn downloadSnowballWebsite-- regenerate {- ifSkippedAlsoSkip = [- downloadSnowballStemmers.name,- downloadSnowballWebsite.name- ]- }-- inputs.files fileTree(- dir: "src/java/org/tartarus/snowball",- include: [- "Among.java",- "SnowballStemmer.java",- "SnowballProgram.java",- "ext/*Stemmer.java"- ])-- inputs.files fileTree(- dir: "src/resources/org/apache/lucene/analysis/snowball",- include: "*_stop.txt")-- // Don't bother on Windows, won't work.- if (Os.isFamily(Os.FAMILY_WINDOWS)) {- doFirst {- // Just emit a big fat error message. Fail the build so that checksums are not regenerated.- throw new GradleException("Snowball generation does not work on Windows (bash must be available).")- }- }-- executable "bash"- args = [- snowballScript,- snowballStemmerDir,- snowballWebsiteDir,- projectDir- ]- })-}diff --git a/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradle b/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradledeleted file mode 100644index 43d8d01b738..00000000000--- a/build-tools/build-infra/src/main/groovy/regenerate.unicode-test-classes.gradle+++ /dev/null@@ -1,66 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import org.apache.lucene.gradle.plugins.misc.QuietExec--// Regenerates test classes from unicode data.--configure(project(":lucene:test-framework")) {- tasks.register("regenerateEmojiTokenizationTest", QuietExec, {- def unicodeVersion = "12.1"-- def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")- def genScript = file("${genDir}/generateEmojiTokenizationTest.pl")- def genOutput = file("${genDir}/EmojiTokenizationTestUnicode_${unicodeVersion.replace('.', '_')}.java")-- description = "Regenerate ${genOutput}"-- inputs.file genScript- inputs.property "unicodeVersion", unicodeVersion- outputs.file genOutput-- workingDir = genDir- executable = buildGlobals.externalTool("perl")- args = [- genScript,- "-v",- unicodeVersion- ]- })-- tasks.register("regenerateJavaUnicodeWordBreakTest", QuietExec, {- def unicodeVersion = "12.1"-- def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")- def genScript = file("${genDir}/generateJavaUnicodeWordBreakTest.pl")- def genOutput = file("${genDir}/WordBreakTestUnicode_${unicodeVersion.replace('.', '_')}.java")-- description = "Regenerate ${genOutput}"-- inputs.file genScript- inputs.property "unicodeVersion", unicodeVersion- outputs.file genOutput-- workingDir = genDir- executable = buildGlobals.externalTool("perl")- args = [- genScript,- "-v",- unicodeVersion- ]- })-}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.javaindex 266d1a18549..2b02a84ce5c 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/hacks/WipeGradleTempPlugin.java@@ -16,6 +16,7 @@*/package org.apache.lucene.gradle.plugins.hacks;+import java.io.File;import java.io.IOException;import java.io.UncheckedIOException;import java.nio.file.Files;@@ -25,13 +26,16 @@ import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.List;import java.util.stream.Stream;-import javax.inject.Inject;import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;import org.gradle.api.Project;-import org.gradle.api.file.FileTree;-import org.gradle.api.internal.file.FileOperations;+import org.gradle.api.file.Directory;+import org.gradle.api.file.DirectoryProperty;import org.gradle.api.invocation.Gradle;+import org.gradle.api.logging.Logger;+import org.gradle.api.logging.Logging;import org.gradle.api.plugins.JavaPlugin;+import org.gradle.api.services.BuildService;+import org.gradle.api.services.BuildServiceParameters;import org.gradle.api.tasks.Delete;import org.gradle.api.tasks.TaskCollection;import org.gradle.api.tasks.TaskContainer;@@ -43,14 +47,33 @@ import org.gradle.api.tasks.testing.Test;* @see "LUCENE-9471"*/public abstract class WipeGradleTempPlugin extends LuceneGradlePlugin {- @Inject- public abstract FileOperations getFileOps();-@Overridepublic void apply(Project rootProject) {applicableToRootProjectOnly(rootProject);- cleanTempFilesAfterBuildFinished(rootProject);+ Gradle gradle = rootProject.getGradle();+ gradle+ .getSharedServices()+ .registerIfAbsent(+ "gradleTempCleanupService",+ TempCleanupService.class,+ service -> {+ service.parameters(+ params -> {+ params.getProjectRootDir().set(getProjectRootPath(rootProject).toFile());+ File gradleUserHome = gradle.getGradleUserHomeDir();+ params.getUserHomeDir().set(gradleUserHome);+ params+ .getGradleDaemonDir()+ .set(+ gradleUserHome+ .toPath()+ .resolve("daemon")+ .resolve(gradle.getGradleVersion())+ .toFile());+ });+ })+ .get(); // make sure it'll run.rootProject.getAllprojects()@@ -93,72 +116,84 @@ public abstract class WipeGradleTempPlugin extends LuceneGradlePlugin {});}- private void cleanTempFilesAfterBuildFinished(Project rootProject) {- rootProject- .getGradle()- .buildFinished(- _ -> {- cleanUpRedirectedTmpDir(rootProject);- cleanUpGradlesTempDir(rootProject);- });- }+ public abstract static class TempCleanupService+ implements BuildService<TempCleanupServiceParams>, AutoCloseable {+ private static final Logger LOGGER = Logging.getLogger(TempCleanupService.class);- private static void cleanUpGradlesTempDir(Project rootProject) {- try {- // clean up any files older than 3 hours from the user's gradle temp. the time- // limit is added so that we don't interfere with any concurrent builds... just in- // case.- Instant deadline = Instant.now().minus(3, ChronoUnit.HOURS);- List<Stream<Path>> toDelete = new ArrayList<>();+ @Override+ public void close() {+ cleanUpRedirectedTmpDir(getParameters().getProjectRootDir().get());+ cleanUpGradlesTempDir(+ getParameters().getUserHomeDir().get(), getParameters().getGradleDaemonDir().get());+ }- Gradle gradle = rootProject.getGradle();- Path gradleUserHome = gradle.getGradleUserHomeDir().toPath();+ private void cleanUpGradlesTempDir(Directory gradleUserHomeDir, Directory gradleDaemonDir) {+ try {+ // clean up any files older than 3 hours from the user's gradle temp. the time+ // limit is added so that we don't interfere with any concurrent builds... just in+ // case.+ Instant deadline = Instant.now().minus(3, ChronoUnit.HOURS);+ List<Stream<Path>> toDelete = new ArrayList<>();- var gradleTmp = gradleUserHome.resolve(".tmp");- if (Files.exists(gradleTmp)) {- toDelete.add(Files.list(gradleTmp));- }+ Path gradleUserHome = gradleUserHomeDir.getAsFile().toPath();- var daemonDir = gradleUserHome.resolve("daemon").resolve(gradle.getGradleVersion());- if (Files.exists(daemonDir)) {- toDelete.add(Files.list(daemonDir).filter(path -> path.toString().endsWith(".out.log")));- }+ var gradleTmp = gradleUserHome.resolve(".tmp");+ LOGGER.info("Cleaning up gradle's user temp dir: {}", gradleTmp);+ if (Files.exists(gradleTmp)) {+ toDelete.add(Files.list(gradleTmp));+ }- for (Stream<Path> stream : toDelete) {- try {- for (var path : stream.toList()) {- if (!Files.isRegularFile(path)) {- continue;- }- if (!Files.getLastModifiedTime(path).toInstant().isBefore(deadline)) {- continue;+ var daemonDir = gradleDaemonDir.getAsFile().toPath();+ LOGGER.info("Cleaning up gradle's daemon logs dir: {}", daemonDir);+ if (Files.exists(daemonDir)) {+ toDelete.add(Files.list(daemonDir).filter(path -> path.toString().endsWith(".out.log")));+ }++ for (Stream<Path> stream : toDelete) {+ try {+ for (var path : stream.toList()) {+ if (!Files.isRegularFile(path)) {+ continue;+ }+ if (!Files.getLastModifiedTime(path).toInstant().isBefore(deadline)) {+ continue;+ }+ Files.deleteIfExists(path);}- Files.deleteIfExists(path);+ } finally {+ stream.close();}- } finally {- stream.close();}+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ }++ private void cleanUpRedirectedTmpDir(Directory projectRootDir) {+ // Clean up the java.io.tmpdir we've redirected gradle to use (LUCENE-9471).+ // these are still used and populated with junk.+ try {+ try (var files = Files.list(projectRootDir.dir(".gradle/tmp").getAsFile().toPath())) {+ var workerPaths =+ files+ .filter(+ path -> path.getFileName().toString().startsWith("gradle-worker-classpath"))+ .toList();+ for (var path : workerPaths) {+ Files.delete(path);+ }+ }+ } catch (IOException e) {+ throw new RuntimeException(e);}- } catch (IOException e) {- throw new UncheckedIOException(e);}}- private void cleanUpRedirectedTmpDir(Project rootProject) {- // Clean up the java.io.tmpdir we've redirected gradle to use (LUCENE-9471).- // these are still used and populated with junk.- FileTree tempFiles =- rootProject- .fileTree(".gradle/tmp")- .matching(- patternFilterable -> {- patternFilterable.include("gradle-worker-classpath*");- });- getFileOps()- .delete(- spec -> {- spec.delete(tempFiles);- spec.setFollowSymlinks(false);- });+ public interface TempCleanupServiceParams extends BuildServiceParameters {+ DirectoryProperty getProjectRootDir();++ DirectoryProperty getUserHomeDir();++ DirectoryProperty getGradleDaemonDir();}}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.javaindex 58fc2f03d12..af5589bd5c0 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/ErrorPronePlugin.java@@ -103,9 +103,9 @@ public class ErrorPronePlugin extends LuceneGradlePlugin {skipReason = "skipped explicitly on a CI build it seems";} else {skipReason =- "skipped on builds not running inside CI environments, pass -P"- + OPT_VALIDATION_ERRORPRONE- + "=true to enable";+ "skipped on non-CI environments, pass "+ + ("-P" + OPT_VALIDATION_ERRORPRONE + "=true")+ + " to enable";}}}@@ -116,6 +116,15 @@ public class ErrorPronePlugin extends LuceneGradlePlugin {.register(TASK_ERROR_PRONE_SKIPPED,task -> {+ boolean hasCheckTask =+ project.getGradle().getStartParameter().getTaskNames().contains("check");++ task.onlyIf(+ _ -> {+ // Only complain if we're running a 'check'.+ return hasCheckTask;+ });+task.doFirst(t -> {t.getLogger().warn("Errorprone linting turned off ({})", skipReason);diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.javaindex 2c1c6ee6255..42c650836a3 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CheckJarChecksumsAndLicensesPlugin.java@@ -36,7 +36,6 @@ import org.apache.lucene.gradle.plugins.LuceneGradlePlugin;import org.gradle.api.GradleException;import org.gradle.api.Project;import org.gradle.api.file.Directory;-import org.gradle.api.file.FileTree;import org.gradle.api.plugins.JavaPlugin;import org.gradle.api.tasks.TaskContainer;@@ -141,20 +140,14 @@ public class CheckJarChecksumsAndLicensesPlugin extends LuceneGradlePlugin {}// Dangling files (not referenced)- Set<File> allExisting =- task.getProject()- .fileTree(- licensesDir,- (cfg) -> {- cfg.exclude(- Arrays.asList(- "elegant-icon-font-*", // Used by Luke.- "glove-LICENSE-PDDL.txt" // glove knn dictionary.- ));- })- .getFiles();+ var allExisting = task.getFileOperations().fileTree(licensesDir);+ allExisting.exclude(+ Arrays.asList(+ "elegant-icon-font-*", // Used by Luke.+ "glove-LICENSE-PDDL.txt" // glove knn dictionary.+ ));- List<File> dangling = new ArrayList<>(allExisting);+ List<File> dangling = new ArrayList<>(allExisting.getFiles());dangling.removeAll(referencedFiles);dangling.sort(Comparator.comparing(File::getPath));@@ -214,9 +207,9 @@ public class CheckJarChecksumsAndLicensesPlugin extends LuceneGradlePlugin {candidates.add(new File(licensesDir, baseName + "-LICENSE-[type].txt"));var finalBaseName = baseName;- FileTree tree =- task.getProject()- .fileTree(licensesDir, cfg -> cfg.include(finalBaseName + "-LICENSE-*.txt"));+ var tree = task.getFileOperations().fileTree(licensesDir);+ tree.include(finalBaseName + "-LICENSE-*.txt");+found.addAll(tree.getFiles());String prefix = baseName.replaceAll("[\\-][^-]+$", "");diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.javaindex 60674013a1e..461545b0185 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/licenses/CollectJarInfos.java@@ -18,8 +18,10 @@ package org.apache.lucene.gradle.plugins.licenses;import java.util.List;import java.util.stream.Collectors;+import javax.inject.Inject;import org.gradle.api.DefaultTask;import org.gradle.api.file.ConfigurableFileCollection;+import org.gradle.api.internal.file.FileOperations;import org.gradle.api.tasks.InputFiles;import org.gradle.api.tasks.Internal;@@ -31,6 +33,9 @@ public abstract class CollectJarInfos extends DefaultTask {@InputFilespublic abstract ConfigurableFileCollection getJarInfos();+ @Inject+ protected abstract FileOperations getFileOperations();+@Internalprotected List<JarInfo> getUniqueJarInfos() {return getJarInfos().getFiles().stream()diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.javasimilarity index 98%rename from build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.javarename to build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.javaindex 508948f2d07..62b9dc99f59 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/RootProjectSetupPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/misc/LuceneRootConfigurationPlugin.java@@ -48,11 +48,14 @@ import org.gradle.api.tasks.TaskContainer;import org.gradle.api.tasks.bundling.AbstractArchiveTask;/** Initialization of the root project, globals, etc. */-public class RootProjectSetupPlugin extends LuceneGradlePlugin {+public class LuceneRootConfigurationPlugin extends LuceneGradlePlugin {@Overridepublic void apply(Project rootProject) {applicableToRootProjectOnly(rootProject);+ PluginContainer plugins = rootProject.getPlugins();+ plugins.apply(CheckEnvironmentPlugin.class);+// Register these plugins early, for everything.rootProject.getAllprojects()@@ -64,7 +67,6 @@ public class RootProjectSetupPlugin extends LuceneGradlePlugin {});// Register other root-level plugins.- PluginContainer plugins = rootProject.getPlugins();plugins.apply(RegisterBuildGlobalsPlugin.class);plugins.apply(GitInfoPlugin.class);diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.javanew file mode 100644index 00000000000..3137c7d0b07--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JFlexTask.java@@ -0,0 +1,125 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.regenerate;++import java.io.File;+import java.io.IOException;+import java.io.UncheckedIOException;+import java.nio.file.Files;+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;+import org.gradle.api.DefaultTask;+import org.gradle.api.GradleException;+import org.gradle.api.file.ConfigurableFileCollection;+import org.gradle.api.file.RegularFileProperty;+import org.gradle.api.provider.Property;+import org.gradle.api.tasks.InputFile;+import org.gradle.api.tasks.Internal;+import org.gradle.api.tasks.OutputFile;+import org.gradle.api.tasks.TaskAction;++public abstract class JFlexTask extends DefaultTask {+ @InputFile+ public abstract RegularFileProperty getJflexFile();++ @InputFile+ public abstract RegularFileProperty getSkeleton();++ @Internal+ public abstract Property<String> getHeapSize();++ @OutputFile+ public abstract RegularFileProperty getGeneratedFile();++ // This is marked internal instead of @Classpath because classpath+ // files are then added as inputs and end up in the regenerate's+ // extension checksums file.+ @Internal+ public abstract ConfigurableFileCollection getClasspath();++ public JFlexTask() {+ getGeneratedFile()+ .convention(+ getJflexFile()+ .map(+ f ->+ getProject()+ .getLayout()+ .getProjectDirectory()+ .file(f.getAsFile().getAbsolutePath().replace(".jflex", ".java"))));+ }++ @TaskAction+ public void generate() {+ var project = getProject();+ var logger = getLogger();++ var jflexFile = getJflexFile().get().getAsFile();+ if (!jflexFile.exists()) {+ throw new GradleException("JFlex file does not exist: " + jflexFile);+ }++ final File target = getGeneratedFile().get().getAsFile();++ logger.lifecycle("Recompiling JFlex: {}", project.relativePath(jflexFile));++ LuceneBuildGlobalsExtension buildGlobals =+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class);++ buildGlobals+ .getExecOps()+ .javaexec(+ spec -> {+ spec.setIgnoreExitValue(false);++ spec.setClasspath(getClasspath());+ spec.getMainClass().set("jflex.Main");++ spec.args("-nobak", "--quiet", "--encoding", "UTF-8");++ if (getHeapSize().isPresent()) {+ spec.setMaxHeapSize(getHeapSize().get());+ }++ if (getSkeleton().isPresent()) {+ spec.args("--skel", getSkeleton().get().getAsFile().getAbsolutePath());+ }++ spec.args(+ "-d", target.getParentFile().getAbsolutePath(), jflexFile.getAbsolutePath());+ });++ try {+ var text = Files.readString(target.toPath());++ // fix invalid SuppressWarnings from jflex, so that it works with javac.+ // we also need to suppress 'unused' because of dead code, so it passes ecjLint+ // you can't "stack" SuppressWarnings annotations, jflex adds its own, this is the only way+ // https://github.com/jflex-de/jflex/issues/762+ text =+ text.replace(+ "SuppressWarnings(\"fallthrough\")",+ "SuppressWarnings({\"fallthrough\",\"unused\"})");++ // Seems like a bug in gjf. https://github.com/google/google-java-format/issues/1260+ text = text.replace("/* Break so we don't hit fall-through warning: */", "");++ Files.writeString(target.toPath(), text);+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.javanew file mode 100644index 00000000000..cce5c8276a2--- /dev/null+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/JavaCCTask.java@@ -0,0 +1,160 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.gradle.plugins.regenerate;++import groovy.lang.Closure;+import java.io.ByteArrayOutputStream;+import java.io.File;+import java.nio.charset.Charset;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.List;+import java.util.function.Consumer;+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension;+import org.gradle.api.DefaultTask;+import org.gradle.api.GradleException;+import org.gradle.api.Project;+import org.gradle.api.file.ConfigurableFileCollection;+import org.gradle.api.file.CopySpec;+import org.gradle.api.file.FileTree;+import org.gradle.api.file.RegularFileProperty;+import org.gradle.api.tasks.InputFile;+import org.gradle.api.tasks.Internal;+import org.gradle.api.tasks.OutputFiles;+import org.gradle.api.tasks.TaskAction;+import org.gradle.process.ExecResult;+import org.gradle.process.JavaExecSpec;++public abstract class JavaCCTask extends DefaultTask {+ /** Apply closures to all generated files before they're copied back to mainline code. */+ // A subtle bug here is that this makes it not an input... should be a list of replacements+ // instead?+ private final List<Consumer<FileTree>> afterGenerate = new ArrayList<>();++ @InputFile+ public abstract RegularFileProperty getJavaccFile();++ // This is marked internal instead of @Classpath because classpath+ // files are then added as inputs and end up in the regenerate's+ // extension checksums file.+ @Internal+ public abstract ConfigurableFileCollection getClasspath();++ public void afterGenerate(Consumer<FileTree> action) {+ this.afterGenerate.add(action);+ }++ public void afterGenerate(Closure<?> action) {+ afterGenerate(action::call);+ }++ @OutputFiles+ public List<File> getGeneratedSources() {+ var javaccFile = getJavaccFile().get().getAsFile();+ File baseDir = javaccFile.getParentFile();+ String baseName = javaccFile.getName().replace(".jj", "");++ Project p = getProject();+ return Arrays.asList(+ p.file(new File(baseDir, baseName + ".java")),+ p.file(new File(baseDir, baseName + "Constants.java")),+ p.file(new File(baseDir, baseName + "TokenManager.java")),+ p.file(new File(baseDir, "ParseException.java")),+ p.file(new File(baseDir, "Token.java")),+ p.file(new File(baseDir, "TokenMgrError.java")));+ }++ @TaskAction+ public void generate() {+ var javaccFile = getJavaccFile().get().getAsFile();+ if (!javaccFile.exists()) {+ throw new GradleException("Input file does not exist: " + javaccFile);+ }++ var logger = getLogger();+ var project = getProject();++ // Run JavaCC into a temporary dir so we can post-process & know exactly what's generated+ File tempDir = getTemporaryDir();+ if (!tempDir.exists() && !tempDir.mkdirs()) {+ throw new GradleException("Unable to create temporary directory: " + tempDir);+ }++ // Run javacc generation into temporary folder so that we know all the generated files+ // and can post-process them easily.+ LuceneBuildGlobalsExtension buildGlobals =+ project.getExtensions().getByType(LuceneBuildGlobalsExtension.class);+ buildGlobals+ .getFileOps()+ .delete(spec -> spec.delete(project.fileTree(tempDir, tree -> tree.include("**/*.java"))));++ File targetDir = javaccFile.getParentFile();+ logger.lifecycle(+ "Recompiling JavaCC: {}", project.getRootDir().toPath().relativize(javaccFile.toPath()));++ ByteArrayOutputStream output = new ByteArrayOutputStream();+ ExecResult result =+ buildGlobals+ .getExecOps()+ .javaexec(+ (JavaExecSpec spec) -> {+ spec.setClasspath(getClasspath());++ spec.setIgnoreExitValue(true);+ spec.setStandardOutput(output);+ spec.setErrorOutput(output);++ spec.getMainClass().set("org.javacc.parser.Main");+ spec.args(+ "-OUTPUT_DIRECTORY=" + tempDir.getAbsolutePath(),+ javaccFile.getAbsolutePath());+ });++ if (result.getExitValue() != 0) {+ throw new GradleException(+ "JavaCC failed to compile "+ + javaccFile+ + ", here is the compilation output:\n"+ + output);+ }++ String outString = output.toString(Charset.defaultCharset());+ if (outString.contains("Warning:")) {+ throw new GradleException(+ "JavaCC emitted warnings for "+ + javaccFile+ + ", here is the compilation output:\n"+ + output);+ }++ // Apply custom modifications+ FileTree generatedFiles = project.fileTree(tempDir);+ for (Consumer<FileTree> action : afterGenerate) {+ action.accept(generatedFiles);+ }++ // Copy back to mainline sources (excluding CharStream.java)+ buildGlobals+ .getFileOps()+ .copy(+ (CopySpec spec) -> {+ spec.from(tempDir);+ spec.into(targetDir);+ spec.exclude("CharStream.java");+ });+ }+}diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.javaindex 407cfa5b8d3..41d557819ca 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/regenerate/RegenerateTasksSupportPlugin.java@@ -44,8 +44,8 @@ import org.gradle.api.file.FileCollection;import org.gradle.api.logging.LogLevel;import org.gradle.api.provider.Provider;import org.gradle.api.tasks.Delete;+import org.gradle.api.tasks.TaskCollection;import org.gradle.api.tasks.TaskContainer;-import org.gradle.api.tasks.TaskProvider;/*** Adds support for tasks that regenerate (versioned) sources within Lucene.@@ -84,10 +84,38 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {@Overridepublic void apply(Project rootProject) {applicableToRootProjectOnly(rootProject);- rootProject.getAllprojects().forEach(RegenerateTasksSupportPlugin::applyRegeneratePlugin);++ var regenTasks =+ rootProject.getAllprojects().stream()+ .map(RegenerateTasksSupportPlugin::applyRegeneratePlugin)+ .toList();++ // wait until after the evaluation is completed to give tasks some time to configure+ // extension properties, then process these tasks that have the regenerate task extension+ // configured.+ rootProject+ .getGradle()+ .projectsEvaluated(+ _ -> {+ for (var taskCollection : regenTasks) {+ for (Task task : taskCollection.stream().toList()) {+ configureRegenerateTask(task);+ }+ }+ });++ // Nearly all regeneration tasks touch input sources, which are inputs to checkLicenses+ // so schedule this globally, once.+ rootProject+ .getTasks()+ .matching(task -> task.getName().equals(CheckLicensesPlugin.CHECK_LICENSES_TASK))+ .configureEach(+ task -> {+ task.mustRunAfter(regenTasks);+ });}- private static void applyRegeneratePlugin(Project project) {+ private static TaskCollection<Task> applyRegeneratePlugin(Project project) {TaskContainer tasks = project.getTasks();var regenerateTasks =@@ -112,18 +140,6 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {.addAll(Set.of("args", "executable", "ignoreExitValue"));});- // wait until after the evaluation is completed to give tasks some time to configure- // extension properties, then process these tasks. I don't- // think this can be done reasonably well based on lazy properties alone.- project.afterEvaluate(- _ -> {- for (String name : project.getTasks().getNames()) {- if (REGEN_TASK_PATTERN.matcher(name).matches()) {- configureRegenerateTask(project, regenerateTasks.named(name));- }- }- });-// register a single task to invoke all regeneration tasks.tasks.register(REGEN_TASK_NAME,@@ -141,23 +157,14 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {task.delete(project.fileTree("src/generated/checksums", cfg -> cfg.include("*.json")));});- // Nearly all regeneration tasks touch input sources, which are inputs to checkLicenses- // so schedule this globally, once.- project- .getRootProject()- .getTasks()- .matching(task -> task.getName().equals(CheckLicensesPlugin.CHECK_LICENSES_TASK))- .configureEach(- task -> {- task.mustRunAfter(regenerateTasks);- });+ return regenerateTasks;}- private static void configureRegenerateTask(- Project project, TaskProvider<? extends Task> taskProvider) {+ private static void configureRegenerateTask(Task delegate) {+ var project = delegate.getProject();var tasks = project.getTasks();- String taskName = taskProvider.getName();- String taskPrefix = taskProvider.getName();+ String taskName = delegate.getName();+ String taskPrefix = delegate.getName();Path checksumsFile = project.file("src/generated/checksums/" + taskPrefix + ".json").toPath();@@ -166,8 +173,6 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {tasks.register(taskPrefix + "ChecksumSave",t -> {- var delegate = taskProvider.get();-// Check delegate's sanity.if (delegate.getGroup() != null && !delegate.getGroup().isBlank()) {throw new GradleException(@@ -208,13 +213,8 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {});});- // TODO: so this entire block could/should reside inside checksumSaveTask.register() and be lazy- // but gradle throws an 'invalid context' exception on any task.configureEach within this block.- // I don't know how to do implement this without eager instantiation of tasks, sorry.+ // Configure task dependencies and wiring of after/before.{- Task t = checksumSaveTask.get();- Task delegate = taskProvider.get();-RegenerateTaskExtension regenerateExt =delegate.getExtensions().getByType(RegenerateTaskExtension.class);@@ -227,7 +227,7 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {.matching(task -> taskNames.contains(task.getName())).configureEach(other -> {- other.mustRunAfter(t);+ other.mustRunAfter(checksumSaveTask);});}@@ -235,7 +235,7 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {// checksums are saved but after the source task ran).{var taskNames = regenerateExt.getFollowedUpBy().get();- t.dependsOn(taskNames);+ checksumSaveTask.configure(t -> t.dependsOn(taskNames));project.getTasks().matching(task -> taskNames.contains(task.getName()))@@ -273,18 +273,13 @@ public class RegenerateTasksSupportPlugin extends LuceneGradlePlugin {}}- taskProvider.configure(- task -> {- task.finalizedBy(checksumSaveTask);- });+ delegate.finalizedBy(checksumSaveTask);// Configure checksum check task (verification).var checksumCheckTask =tasks.register(taskName + "ChecksumCheck",t -> {- var delegate = taskProvider.get();-// if we're regenerating, run any checks after the save.t.mustRunAfter(checksumSaveTask);diff --git a/build.gradle b/build.gradleindex bb541ec3399..d95183a0642 100644--- a/build.gradle+++ b/build.gradle@@ -17,21 +17,7 @@plugins {id "base"-- id "lucene.root-project.check-environment"id "lucene.root-project.setup"-- // TODO: move to java.- id 'regenerate.forUtil'- id 'regenerate.antlr'- id 'regenerate.icu'- id 'regenerate.javacc'- id 'regenerate.jflex'- id 'regenerate.kuromoji'- id 'regenerate.nori'- id 'regenerate.unicode-test-classes'- id 'regenerate.moman'- id 'regenerate.snowball'}description = 'Grandparent project for Apache Lucene Core'diff --git a/gradle/ide/eclipse/run-solr-cloud.launch b/gradle/ide/eclipse/run-solr-cloud.launchdeleted file mode 100644index 4f8399a5bc6..00000000000--- a/gradle/ide/eclipse/run-solr-cloud.launch+++ /dev/null@@ -1,20 +0,0 @@-<?xml version="1.0" encoding="UTF-8" standalone="no"?>-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">- <listEntry value="/@ECLIPSEPROJECTNAME@"/>- </listAttribute>- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">- <listEntry value="4"/>- </listAttribute>- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>- <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry containerPath="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8" javaProject="@ECLIPSEPROJECTNAME@" path="1" type="4"/> "/>- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry id="org.eclipse.jdt.launching.classpathentry.defaultClasspath"> <memento exportedEntriesOnly="false" project="@ECLIPSEPROJECTNAME@"/> </runtimeClasspathEntry> "/>- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry internalArchive="/@ECLIPSEPROJECTNAME@/solr/server/start.jar" path="3" type="2"/> "/>- </listAttribute>- <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.jetty.start.Main"/>- <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="--module=http"/>- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms512m -Djetty.port=8983 -DSTOP.PORT=7983 -DSTOP.KEY=solrrock -Dlog4j.configurationFile="file:${workspace_loc:@ECLIPSEPROJECTNAME@}/solr/server/resources/log4j2.xml" -Djetty.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Djetty.base="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Dsolr.log="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build" -Dsolr.log.dir="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build" -Dsolr.solr.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server/solr" -DzkRun"/>-</launchConfiguration>diff --git a/gradle/ide/eclipse/run-solr.launch b/gradle/ide/eclipse/run-solr.launchdeleted file mode 100644index 4d6b033a405..00000000000--- a/gradle/ide/eclipse/run-solr.launch+++ /dev/null@@ -1,20 +0,0 @@-<?xml version="1.0" encoding="UTF-8" standalone="no"?>-<launchConfiguration type="org.eclipse.jdt.launching.localJavaApplication">- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">- <listEntry value="/@ECLIPSEPROJECTNAME@"/>- </listAttribute>- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">- <listEntry value="4"/>- </listAttribute>- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>- <listAttribute key="org.eclipse.jdt.launching.CLASSPATH">- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry containerPath="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8" javaProject="@ECLIPSEPROJECTNAME@" path="1" type="4"/> "/>- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry id="org.eclipse.jdt.launching.classpathentry.defaultClasspath"> <memento exportedEntriesOnly="false" project="@ECLIPSEPROJECTNAME@"/> </runtimeClasspathEntry> "/>- <listEntry value="<?xml version="1.0" encoding="UTF-8" standalone="no"?> <runtimeClasspathEntry internalArchive="/@ECLIPSEPROJECTNAME@/solr/server/start.jar" path="3" type="2"/> "/>- </listAttribute>- <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="false"/>- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value="org.eclipse.jetty.start.Main"/>- <stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="--module=http"/>- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Xms512m -Djetty.port=8983 -DSTOP.PORT=7983 -DSTOP.KEY=solrrock -Dlog4j.configurationFile="file:${workspace_loc:@ECLIPSEPROJECTNAME@}/solr/server/resources/log4j2.xml" -Djetty.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Djetty.base="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server" -Dsolr.log="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build" -Dsolr.log.dir="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build" -Dsolr.solr.home="${workspace_loc:@ECLIPSEPROJECTNAME@}/eclipse-build/solr-server/solr""/>-</launchConfiguration>diff --git a/gradle/ide/eclipse/run-test-cases.launch b/gradle/ide/eclipse/run-test-cases.launchdeleted file mode 100644index 713628cebab..00000000000--- a/gradle/ide/eclipse/run-test-cases.launch+++ /dev/null@@ -1,17 +0,0 @@-<?xml version="1.0" encoding="UTF-8" standalone="no"?>-<launchConfiguration type="org.eclipse.jdt.junit.launchconfig">- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">- <listEntry value="/@ECLIPSEPROJECTNAME@"/>- </listAttribute>- <listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">- <listEntry value="4"/>- </listAttribute>- <stringAttribute key="org.eclipse.jdt.junit.CONTAINER" value="=@ECLIPSEPROJECTNAME@"/>- <booleanAttribute key="org.eclipse.jdt.junit.KEEPRUNNING_ATTR" value="false"/>- <stringAttribute key="org.eclipse.jdt.junit.TESTNAME" value=""/>- <stringAttribute key="org.eclipse.jdt.junit.TEST_KIND" value="org.eclipse.jdt.junit.loader.junit4"/>- <booleanAttribute key="org.eclipse.jdt.launching.ATTR_USE_START_ON_FIRST_THREAD" value="true"/>- <stringAttribute key="org.eclipse.jdt.launching.MAIN_TYPE" value=""/>- <stringAttribute key="org.eclipse.jdt.launching.PROJECT_ATTR" value="@ECLIPSEPROJECTNAME@"/>- <stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-ea"/>-</launchConfiguration>diff --git a/gradle/regenerate/moman/gen_BulkOperation.py b/gradle/regenerate/packed/gen_BulkOperation.pysimilarity index 100%rename from gradle/regenerate/moman/gen_BulkOperation.pyrename to gradle/regenerate/packed/gen_BulkOperation.pydiff --git a/gradle/regenerate/moman/gen_Packed64SingleBlock.py b/gradle/regenerate/packed/gen_Packed64SingleBlock.pysimilarity index 100%rename from gradle/regenerate/moman/gen_Packed64SingleBlock.pyrename to gradle/regenerate/packed/gen_Packed64SingleBlock.pydiff --git a/lucene/analysis/common/build.gradle b/lucene/analysis/common/build.gradleindex 59c95921e11..480f7b45e97 100644--- a/lucene/analysis/common/build.gradle+++ b/lucene/analysis/common/build.gradle@@ -15,7 +15,9 @@* limitations under the License.*/-+import org.apache.lucene.gradle.plugins.misc.QuietExec+import de.undercouch.gradle.tasks.download.Download+import org.apache.tools.ant.taskdefs.condition.Osimport com.carrotsearch.gradle.buildinfra.buildoptions.BuildOptionimport java.nio.file.Files@@ -27,6 +29,105 @@ dependencies {moduleTestImplementation project(':lucene:test-framework')}+//+// Regenerate snowball resources+//++def resources = rootProject.file("gradle/regenerate/snowball")++// git commit hash of source code https://github.com/snowballstem/snowball/+def snowballStemmerCommit = "34f3612e5e8c48975243bc2e87561abdac5aa9bb"+// git commit hash of stopwords https://github.com/snowballstem/snowball-website+def snowballWebsiteCommit = "424fd1f75044160bb4a92f1daf08cce618459374"++def snowballWorkDir = file("${buildDir}/snowball")++def snowballStemmerDir = file("${snowballWorkDir}/stemmers-${snowballStemmerCommit}")+def snowballWebsiteDir = file("${snowballWorkDir}/website-${snowballWebsiteCommit}")++def snowballScript = file("${resources}/snowball.sh")++def unpackFromZip = { zipFile, targetDir ->+ buildGlobals.fileOps.sync({SyncSpec spec ->+ spec.from(zipTree(zipFile), {+ eachFile { fcd ->+ fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))+ }+ })+ spec.into targetDir+ })+}++// downloads snowball stemmers (or use a cached copy)+def downloadSnowballStemmers = tasks.register("downloadSnowballStemmers", Download, {+ src "https://github.com/snowballstem/snowball/archive/${snowballStemmerCommit}.zip"+ dest file("${snowballStemmerDir}.zip")+ overwrite false+ tempAndMove true++ doLast {+ unpackFromZip(dest, snowballStemmerDir)+ }+})++// downloads snowball website (or use a cached copy)+def downloadSnowballWebsite = tasks.register("downloadSnowballWebsite", Download, {+ src "https://github.com/snowballstem/snowball-website/archive/${snowballWebsiteCommit}.zip"+ def snowballWebsiteZip = file("${snowballWebsiteDir}.zip")+ dest snowballWebsiteZip+ overwrite false+ tempAndMove true++ doLast {+ unpackFromZip(snowballWebsiteZip, snowballWebsiteDir)+ }+})++// runs a shell script to regenerate stemmers, base stemming subclasses, test data, and stopwords.+tasks.register("regenerateSnowball", QuietExec, {+ description = "Regenerate snowball stemmers."++ dependsOn downloadSnowballStemmers+ dependsOn downloadSnowballWebsite++ regenerate {+ ifSkippedAlsoSkip = [+ downloadSnowballStemmers.name,+ downloadSnowballWebsite.name+ ]+ }++ inputs.files fileTree(+ dir: "src/java/org/tartarus/snowball",+ include: [+ "Among.java",+ "SnowballStemmer.java",+ "SnowballProgram.java",+ "ext/*Stemmer.java"+ ])++ inputs.files fileTree(+ dir: "src/resources/org/apache/lucene/analysis/snowball",+ include: "*_stop.txt")++ // Don't bother on Windows, won't work.+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {+ doFirst {+ // Just emit a big fat error message. Fail the build so that checksums are not regenerated.+ throw new GradleException("Snowball generation does not work on Windows (bash must be available).")+ }+ }++ executable "bash"+ args = [+ snowballScript,+ snowballStemmerDir,+ snowballWebsiteDir,+ projectDir+ ]+})++// Enable Hunspell tests against LibreOffice/ Woorm dictionaries. We pull// these dictionaries dynamically from git of each respective project. To keep// things consistent across pull requests/ re-runs, we use a fixed git commitdiff --git a/lucene/analysis/icu/build.gradle b/lucene/analysis/icu/build.gradleindex 3bcde2832f4..d307ec65520 100644--- a/lucene/analysis/icu/build.gradle+++ b/lucene/analysis/icu/build.gradle@@ -15,7 +15,8 @@* limitations under the License.*/-+import org.apache.tools.ant.taskdefs.condition.Os+import java.nio.file.Filesdescription = 'Analysis integration with ICU (International Components for Unicode)'@@ -27,3 +28,286 @@ dependencies {moduleTestImplementation project(':lucene:test-framework')}++/*+ * Regenerates ICU-related data files in this and other projects.+ *+ * The icu4c version must match exactly the icu4j version in libs.versions.toml.+ * The one on your system is probably different. This script will attempt to+ * download and compile a matching icu4c version automatically.+ */++def resources = rootProject.file("gradle/regenerate/icu")++def setupIcuDependencies = { Project proj ->+ // Configure different icu4j dependencies.+ proj.configurations {+ icu_current+ groovy+ }++ proj.dependencies {+ icu_current deps.icu4j+ groovy deps.groovy+ }+}++def icu4jVersion = deps.icu4j.get().version++setupIcuDependencies(project)+def utr30DataDir = file("src/data/utr30")++def icuBuildDir = file("${buildDir}/icu")+def icuBinDir+def gennorm+def icupkg++if (Os.isFamily(Os.FAMILY_WINDOWS)) {+ icuBinDir = file("${icuBuildDir}/bin64")+ gennorm = file("${icuBinDir}/gennorm2.exe")+ icupkg = file("${icuBinDir}/icupkg.exe")+} else {+ icuBinDir = file("${icuBuildDir}/icu/source/bin")+ gennorm = file("${icuBinDir}/gennorm2")+ icupkg = file("${icuBinDir}/icupkg")+}++def icuCompileTask = Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"++tasks.register("regenerateUtr30DataFiles", {+ description = "Regenerate utf30 norm files using ICU"++ def icuConfig = configurations.icu_current++ dependsOn icuConfig+ dependsOn icuCompileTask++ regenerate {+ ifSkippedAlsoSkip = [icuCompileTask]+ }++ // May be undefined yet, so use a provider.+ dependsOn { sourceSets.tools.runtimeClasspath }++ // gennorm generates file order-dependent output, so make it constant here.+ def inputFiles = fileTree(dir: utr30DataDir, include: "*.txt")+ .asList().toSorted(Comparator.comparing { File f -> f.name })+ def outputFile = file("src/resources/org/apache/lucene/analysis/icu/utr30.nrm")++ inputs.files inputFiles+ inputs.property "icuConfig", icu4jVersion+ outputs.file outputFile++ doFirst {+ // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm+ buildGlobals.execOps.javaexec {+ mainClass = "org.apache.lucene.analysis.icu.GenerateUTR30DataFiles"+ classpath = sourceSets.tools.runtimeClasspath++ ignoreExitValue = false+ workingDir utr30DataDir+ args = [+ "release-${icu4jVersion.replace(".", "-")}"+ ]+ }++ buildGlobals.quietExec(it, {+ executable gennorm+ args = [+ "-v",+ "-s",+ utr30DataDir,+ "-o",+ "${buildDir}/utr30.tmp",+ *(inputFiles.collect { it.name })+ ]+ })++ buildGlobals.quietExec(it, {+ executable icupkg+ args = [+ "-tb",+ "${buildDir}/utr30.tmp",+ outputFile+ ]+ })+ }+})++tasks.register("regenerateRbbi", {+ description = "Regenerate rbbi rules using ICU"++ def icuConfig = configurations.icu_current++ dependsOn icuConfig++ // May be undefined yet, so use a provider.+ dependsOn { sourceSets.tools.runtimeClasspath }++ def sourceDir = file("src/data/uax29")+ def targetDir = file("src/resources/org/apache/lucene/analysis/icu/segmentation")++ inputs.files fileTree(dir: sourceDir, include: "*.rbbi")+ inputs.property "icuConfig", icu4jVersion+ outputs.files fileTree(dir: targetDir, include: "*.brk")++ doFirst {+ buildGlobals.execOps.javaexec {+ mainClass = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"+ classpath = sourceSets.tools.runtimeClasspath++ ignoreExitValue = false+ enableAssertions true+ args = [sourceDir, targetDir]+ }+ }+})++tasks.register("compileIcuWindows", {+ doFirst {+ def v = icu4jVersion+ def icuBinZip = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}.zip")+ if (!icuBinZip.exists()) {+ icuBuildDir.mkdirs()++ // Download binaries matching icu4j version in version.props+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-Win64-MSVC2019.zip")++ logger.lifecycle("Trying to download binary ICU version: ${v} from:\n ${src}")+ Files.write(icuBinZip.toPath(), src.toURL().openStream().bytes)+ logger.lifecycle("Downloaded ${icuBinZip.size()} bytes.")+ }++ // Unzip.+ buildGlobals.fileOps.copy({+ into icuBuildDir+ from zipTree(icuBinZip)+ })+ }+})++tasks.register("compileIcuLinux", {+ doFirst {+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {+ throw new GradleException("ICU compilation not supported on Windows.")+ }++ def v = icu4jVersion+ def icuSrcTgz = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}-src.tgz")++ // Download sources for version matching icu4j version in version.props+ if (!icuSrcTgz.exists()) {+ icuBuildDir.mkdirs()+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-src.tgz")+ logger.lifecycle("Trying to download and compile ICU version: ${v} from:\n ${src}")+ Files.write(icuSrcTgz.toPath(), src.toURL().openStream().bytes)+ logger.lifecycle("Downloaded ${icuSrcTgz.size()} bytes.")+ }++ def icuSrcDir = file("${icuBuildDir}/icu/source")+ buildGlobals.fileOps.delete(icuSrcDir)++ // Extract the tgz+ buildGlobals.quietExec(it, {+ executable "tar"+ workingDir icuBuildDir+ args = [+ "-zxf",+ icuSrcTgz+ ]+ })++ // Compile: (cd icu/source && ./configure --prefix=$(pwd) --enable-rpath && make -j4)+ buildGlobals.quietExec(it, {+ executable "sh"++ workingDir icuSrcDir+ environment("CFLAGS", "-O0")+ environment("CXXFLAGS", "-O0")+ args = [+ "configure",+ "--prefix=${icuSrcDir}",+ "--enable-rpath"+ ]+ })++ buildGlobals.quietExec(it, {+ executable "make"+ workingDir icuSrcDir+ args = [+ "-j${buildOptions['tests.jvms'].get()}"+ ]+ })++ // Test that the binaries work: derb -V+ logger.lifecycle("Compiled ICU, checking...")+ buildGlobals.quietExec(it, {+ executable "./derb"+ workingDir icuBinDir+ args = ["-V"]+ })+ }+})++configure(project(":lucene:analysis:common")) {+ setupIcuDependencies(project)++ tasks.register("regenerateUnicodeProps", {+ def icuConfig = configurations.icu_current+ def groovyConfig = configurations.groovy++ def outputFile = file("src/java/org/apache/lucene/analysis/util/UnicodeProps.java")++ description = "Regenerate ${outputFile} using ICU"++ dependsOn icuConfig++ inputs.property "icuConfig", icu4jVersion+ outputs.file outputFile++ doFirst {+ buildGlobals.execOps.javaexec({ JavaExecSpec spec ->+ spec.mainClass = "groovy.lang.GroovyShell"+ spec.classpath(icuConfig, groovyConfig)++ spec.args([+ "--encoding",+ "UTF-8",+ file("${resources}/GenerateUnicodeProps.groovy").toString(),+ outputFile.toString()+ ])+ })+ }+ })+}++configure(project(":lucene:core")) {+ setupIcuDependencies(project)++ tasks.register("generateUnicodeProps", {+ def icuConfig = configurations.icu_current+ def groovyConfig = configurations.groovy+ def outputFile = file("src/java/org/apache/lucene/util/automaton/CaseFolding.java")++ description = "Regenerate ${outputFile} using ICU"++ dependsOn icuConfig++ inputs.property "icuConfig", icu4jVersion+ outputs.file outputFile++ doFirst {+ buildGlobals.execOps.javaexec({+ mainClass = "groovy.lang.GroovyShell"+ classpath icuConfig, groovyConfig++ args = [+ "--encoding",+ "UTF-8",+ file("${resources}/GenerateCaseFolding.groovy"),+ outputFile+ ]+ })+ }+ })+}diff --git a/lucene/analysis/kuromoji/build.gradle b/lucene/analysis/kuromoji/build.gradleindex 9119a5c50fa..fdbd8a6a6db 100644--- a/lucene/analysis/kuromoji/build.gradle+++ b/lucene/analysis/kuromoji/build.gradle@@ -1,3 +1,6 @@+import de.undercouch.gradle.tasks.download.Download+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension+/** Licensed to the Apache Software Foundation (ASF) under one or more* contributor license agreements. See the NOTICE file distributed with@@ -25,3 +28,126 @@ dependencies {moduleTestImplementation project(':lucene:test-framework')}++//+// This configures regeneration of Kuromoji dictionaries.+//++static void recompileDictionary(Project project, String dictionaryName, Closure closure) {+ project.getExtensions().getByType(LuceneBuildGlobalsExtension).execOps.javaexec {+ mainClass = "org.apache.lucene.analysis.ja.dict.DictionaryBuilder"+ classpath = project.sourceSets.main.runtimeClasspath++ jvmArgs '-Xmx1G'++ with closure+ }+ project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")+}++def targetDir = project.file("src/resources")++def mecabDictionaryName = "mecab-ipadic-2.7.0-20070801"+// original source: "https://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/${mecabDictionaryName}.tar.gz"+def mecabDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${mecabDictionaryName}.tar.gz"++def compileMecab = tasks.register("compileMecab", Download, {+ dependsOn sourceSets.main.runtimeClasspath++ def dictionaryFile = file("${buildDir}/generate/${mecabDictionaryName}.tar.gz")+ def unpackedDir = file("${buildDir}/generate/${mecabDictionaryName}")++ src mecabDictionarySource+ dest dictionaryFile+ onlyIfModified true++ doLast {+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")+ })++ // Unpack the downloaded archive.+ delete unpackedDir+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {+ ant.cutdirsmapper(dirs: "1")+ }++ // Apply patch via local git.+ buildGlobals.quietExec(it, {+ workingDir = unpackedDir+ executable "git"+ args += [+ "apply",+ project.file("src/tools/patches/Noun.proper.csv.patch").absolutePath+ ]+ })++ // Compile the dictionary+ recompileDictionary(project, mecabDictionaryName, {+ args += [+ "ipadic",+ unpackedDir,+ targetDir,+ "euc-jp",+ false+ ]+ })+ }+})++tasks.register("regenerateDictionariesFromMecab", {+ description = "Regenerate dictionaries from Mecab data"++ dependsOn compileMecab++ inputs.property("dictionarySource", mecabDictionarySource)+ outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ja/dict", include: "*.dat"))++ regenerate {+ // No need to run any follow-ups, we generate binary files (not sources).+ followedUpBy = []+ ifSkippedAlsoSkip = [compileMecab.name]+ }+})++// TODO: this regenerates data files over the same files as mecab, above. This seems odd and inconsistent?+// https://github.com/apache/lucene/issues/15365++// original source (down): https://rwthaachen.dl.osdn.jp/naist-jdic/53500/mecab-naist-jdic-0.6.3b-20111013.tar.gz+def naistDictionaryName = "mecab-naist-jdic-0.6.3b-20111013"+def naistDictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${naistDictionaryName}.tar.gz"+tasks.register("compileNaist", Download, {+ description = "Recompile dictionaries from Naist data."++ dependsOn sourceSets.main.runtimeClasspath++ def dictionaryFile = file("${buildDir}/generate/${naistDictionaryName}.tar.gz")+ def unpackedDir = file("${buildDir}/generate/${naistDictionaryName}")++ src naistDictionarySource+ dest dictionaryFile+ onlyIfModified true++ doLast {+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")+ })++ // Unpack the downloaded archive.+ delete unpackedDir+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {+ ant.cutdirsmapper(dirs: "1")+ }++ // Compile the dictionary+ recompileDictionary(project, naistDictionaryName, {+ args += [+ "ipadic",+ unpackedDir,+ targetDir,+ "euc-jp",+ false+ ]+ })+ }+})diff --git a/lucene/analysis/nori/build.gradle b/lucene/analysis/nori/build.gradleindex b9e82d0c90a..56ded495e6e 100644--- a/lucene/analysis/nori/build.gradle+++ b/lucene/analysis/nori/build.gradle@@ -15,7 +15,8 @@* limitations under the License.*/-+import de.undercouch.gradle.tasks.download.Download+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtensiondescription = 'Korean Morphological Analyzer'@@ -25,3 +26,76 @@ dependencies {moduleTestImplementation project(':lucene:test-framework')}++//+// This configures regeneration of Nori dictionaries.+//++static void recompileDictionary(Project project, String dictionaryName, Closure closure) {+ project.getExtensions().getByType(LuceneBuildGlobalsExtension).execOps.javaexec {+ mainClass = "org.apache.lucene.analysis.ko.dict.DictionaryBuilder"+ classpath = project.sourceSets.main.runtimeClasspath++ jvmArgs '-Xmx1G'++ with closure+ }+ project.logger.lifecycle("Automaton regenerated from dictionary: ${dictionaryName}")+}++plugins.withType(JavaPlugin).configureEach {+ project.ext {+ targetDir = project.file("src/resources")+ }++ def dictionaryName = "mecab-ko-dic-2.1.1-20180720"+ def dictionarySource = "https://s3.amazonaws.com/lucene-testdata/mecab/${dictionaryName}.tar.gz"+ def compileMecabKo = tasks.register("compileMecabKo", Download, {++ dependsOn sourceSets.main.runtimeClasspath++ def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")+ def unpackedDir = file("${buildDir}/generate/${dictionaryName}")++ src dictionarySource+ dest dictionaryFile+ onlyIfModified true++ doLast {+ buildGlobals.fileOps.delete(sourceSets.main.resources.srcDirs.collect {File location ->+ fileTree(dir: location, include: "org/apache/lucene/analysis/ja/dict/*.dat")+ })++ // Unpack the downloaded archive.+ delete unpackedDir+ ant.untar(src: dictionaryFile, dest: unpackedDir, compression: "gzip") {+ ant.cutdirsmapper(dirs: "1")+ }++ // Compile the dictionary+ recompileDictionary(project, dictionaryName, {+ args += [+ unpackedDir,+ targetDir,+ "utf-8",+ false+ ]+ })+ }+ })++ tasks.register("regenerateDictionariesFromMecabKo", {+ description = "Recompile dictionaries from Mecab-Ko data."++ dependsOn compileMecabKo++ inputs.property("dictionarySource", dictionarySource)+ outputs.files(fileTree(dir: "src/resources/org/apache/lucene/analysis/ko/dict", include: "*.dat"))++ regenerate {+ // No need to run any follow-ups, we generate binary files (not sources).+ followedUpBy = []+ ifSkippedAlsoSkip = ["compileMecabKo"]+ }+ })+}diff --git a/lucene/benchmark-jmh/build.gradle b/lucene/benchmark-jmh/build.gradleindex 2a8232c37ce..6f874e410b9 100644--- a/lucene/benchmark-jmh/build.gradle+++ b/lucene/benchmark-jmh/build.gradle@@ -30,7 +30,7 @@ dependencies {// Exclude JMH-generated files and certain classes that require incubating classes// from forbiddenapis validation.-tasks.matching { it.name == "forbiddenApisMain" }.configureEach {+tasks.named("forbiddenApisMain").configure {patternSet.exclude(["**/jmh_generated/**"])}@@ -62,7 +62,7 @@ tasks.compileJava.configure {options.javaModuleMainClass.set("org.apache.lucene.benchmark.jmh.Main")}-jar.configure { Jar jarTask ->+tasks.named("jar").configure {dependsOn configurations.runtimeClasspathmanifest {diff --git a/lucene/core/build.gradle b/lucene/core/build.gradleindex 2f7f461f71e..ae4f31261ae 100644--- a/lucene/core/build.gradle+++ b/lucene/core/build.gradle@@ -14,8 +14,13 @@* See the License for the specific language governing permissions and* limitations under the License.*/++import de.undercouch.gradle.tasks.download.Download+import org.apache.lucene.gradle.plugins.globals.LuceneBuildGlobalsExtension+import org.apache.lucene.gradle.plugins.misc.QuietExecimport org.apache.lucene.gradle.plugins.mrjar.LuceneJavaCoreMrjarPluginimport org.apache.lucene.gradle.plugins.mrjar.MrJarsExtension+import org.apache.lucene.gradle.plugins.regenerate.JFlexTaskplugins.apply(LuceneJavaCoreMrjarPlugin)@@ -28,3 +33,342 @@ dependencies {// Setup api stub generation and multi-release jar support for the following JDKs.project.getExtensions().getByType(MrJarsExtension).setupFor(Integer.parseInt(deps.versions.minJava.get()))++//+// Regenerates "ForUtil.java" code used by codecs in this project and in backward-codecs+//++static void createForUtilRegenerateTask(+ Project project, String taskName, String javaFile, String pyScript, File genDir) {+ project.getTasks().register(+ "regenerate" + taskName, QuietExec, {+ description = "Regenerate " + javaFile + " using " + pyScript + " in " + genDir++ def genScript = project.file("${genDir}/${pyScript}")+ def genOutput = project.file("${genDir}/${javaFile}")+ inputs.file genScript+ outputs.file genOutput++ workingDir = genDir+ executable = project.getExtensions().getByType(LuceneBuildGlobalsExtension).externalTool("python3")+ args = ['-B', genScript.toString()]+ })+}++createForUtilRegenerateTask(project, "ForUtil", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/codecs/lucene104"))++configure(project(":lucene:backward-codecs")) {+ createForUtilRegenerateTask(project, "ForUtil84", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene84"))++ createForUtilRegenerateTask(project, "ForUtil90", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene90"))++ createForUtilRegenerateTask(project, "ForUtil99", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene99"))++ createForUtilRegenerateTask(project, "ForUtil912", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene912"))+ createForUtilRegenerateTask(project, "ForDeltaUtil912", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene912"))++ createForUtilRegenerateTask(project, "ForUtil101", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene101"))+ createForUtilRegenerateTask(project, "ForDeltaUtil101", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene101"))++ createForUtilRegenerateTask(project, "ForUtil103", "ForUtil.java", "gen_ForUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene103"))+ createForUtilRegenerateTask(project, "ForDeltaUtil103", "ForDeltaUtil.java", "gen_ForDeltaUtil.py",+ file("src/java/org/apache/lucene/backward_codecs/lucene103"))+}++//+// Regenerate jflex generated sources in this and other projects+//++def resources = rootProject.file("gradle/regenerate/jflex")+def skeletonDefault = file("${resources}/skeleton.default.txt")+def skeletonNoBufferExpansion = file("${resources}/skeleton.disable.buffer.expansion.txt")++configure(project(":lucene:core")) {+ configurations {+ jflex+ }++ dependencies {+ jflex deps.jflex+ }++ tasks.register("regenerateStandardTokenizer", JFlexTask, {+ description = "Regenerate StandardTokenizerImpl.java"++ classpath = configurations.jflex++ jflexFile = file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex')+ skeleton = skeletonNoBufferExpansion++ doLast {+ ant.replace(+ file: file('src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java'),+ encoding: "UTF-8",+ token: "private static final int ZZ_BUFFERSIZE =",+ value: "private int ZZ_BUFFERSIZE ="+ )+ }+ })+}++configure(project(":lucene:analysis:common")) {+ configurations {+ jflex+ }++ dependencies {+ jflex deps.jflex+ }++ def regenerateTlds = tasks.register("regenerateTlds", {+ def tldZones = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"+ def jflexMacro = file("src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex")+ def tldList = file("src/test/org/apache/lucene/analysis/email/TLDs.txt")++ description = "Regenerate top-level domain jflex macros and tests"++ dependsOn { sourceSets.tools.runtimeClasspath }++ inputs.property "tldZones", tldZones+ outputs.files jflexMacro, tldList++ doFirst {+ File tmpJflexMacro = File.createTempFile(jflexMacro.getName(), ".tmp", getTemporaryDir())+ File tmpTldList = File.createTempFile(tldList.getName(), ".tmp", getTemporaryDir())++ project.extensions.buildGlobals.execOps.javaexec {+ mainClass = "org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"+ classpath = sourceSets.tools.runtimeClasspath++ ignoreExitValue = false+ args = [+ tldZones,+ tmpJflexMacro,+ tmpTldList+ ]+ }++ // LUCENE-9926: tldZones is regenerated daily. Compare the generated content (excluding comments) so that+ // we only update actual output files if non-comments have changed.+ def contentLines = { File file ->+ if (file.exists()) {+ List<String> lines = file.readLines("UTF-8")+ lines.removeIf { line -> line.isBlank() || line.startsWith("//") }+ return lines+ } else {+ return []+ }+ }++ if (contentLines(tmpTldList).equals(contentLines(tldList))) {+ logger.lifecycle("Generated TLD content identical as before, not updating.")+ } else {+ tldList.setBytes tmpTldList.bytes+ jflexMacro.setBytes tmpJflexMacro.bytes+ logger.lifecycle("You've regenerated the TLD include file, remember to regenerate UAX29URLEmailTokenizerImpl too.")+ }+ }+ })++ tasks.register("regenerateWikipediaTokenizer", JFlexTask, {+ description = "Regenerate WikipediaTokenizerImpl.java"++ classpath = configurations.jflex++ jflexFile = file('src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex')+ skeleton = skeletonDefault+ })++ tasks.register("regenerateClassicTokenizer", JFlexTask, {+ description = "Regenerate ClassicTokenizerImpl.java"++ classpath = configurations.jflex++ jflexFile = file('src/java/org/apache/lucene/analysis/classic/ClassicTokenizerImpl.jflex')+ skeleton = skeletonDefault+ })++ tasks.register("regenerateUAX29URLEmailTokenizer", JFlexTask, {+ description = "Regenerate UAX29URLEmailTokenizerImpl.java"++ classpath = configurations.jflex++ // UAX29URLEmailTokenizerImpl.jflex includes: ASCIITLD.jflex+ // so we make sure it is up to date. Also, make sure the ordering of internal+ // tasks is enforced (just scheduling top-level wrappers doesn't mean their subtask graphs cannot be reordered).+ dependsOn regenerateTlds++ jflexFile = file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.jflex')+ skeleton = skeletonNoBufferExpansion+ heapSize = "12g"++ // Add included files as inputs.+ inputs.file file('src/java/org/apache/lucene/analysis/email/ASCIITLD.jflex')++ doFirst {+ logger.lifecycle("Regenerating UAX29URLEmailTokenizerImpl. This may take a long time (and requires ${heapSize} of memory!).")+ }++ doLast {+ ant.replace(+ file: file('src/java/org/apache/lucene/analysis/email/UAX29URLEmailTokenizerImpl.java'),+ encoding: "UTF-8",+ token: "private static final int ZZ_BUFFERSIZE =",+ value: "private int ZZ_BUFFERSIZE ="+ )+ }+ })++ def generateHTMLCharacterEntities = tasks.register("regenerateHTMLCharacterEntities", QuietExec, {+ description = "Regenerate HTMLCharacterEntities.jflex"++ def target = file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')+ def script = file("${resources}/htmlentity.py")++ outputs.files target+ inputs.file script++ executable = buildGlobals.externalTool("python3")+ workingDir = target.parentFile+ args += [+ "-B",+ // don't write any bytecode cache+ script,+ target+ ]++ doLast {+ project.ant.fixcrlf(+ file: target,+ encoding: "UTF-8",+ eol: "lf"+ )+ }+ })++ tasks.register("regenerateHTMLStripCharFilter", JFlexTask, {+ description = "Regenerate HTMLStripCharFilter.java"++ classpath = configurations.jflex++ // HTMLStripCharFilter.jflex includes HTMLCharacterEntities.jflex so we make sure it's up to date.+ dependsOn generateHTMLCharacterEntities++ // Add included files as inputs.+ inputs.file file('src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex')++ jflexFile = file('src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex')+ skeleton = skeletonDefault+ })+}++//+// Regenerate precomputed Packed* and bulkOperation* classes.+//++tasks.register("regeneratePacked64SingleBlock", {+ description = "Regenerate Packed64SingleBlock and BulkOperation*.java"++ def targetDir = file("src/java/org/apache/lucene/util/packed")++ def packedResources = rootProject.file("gradle/regenerate/packed")+ def templateFiles = [+ file("${packedResources}/gen_BulkOperation.py"),+ file("${packedResources}/gen_Packed64SingleBlock.py")+ ]++ inputs.files templateFiles+ outputs.files fileTree(dir: targetDir, includes: [+ "Packed64SingleBlock.java",+ "BulkOperation*.java"+ ])++ doLast {+ templateFiles.each {prog ->+ logger.lifecycle("Executing: ${prog} in ${targetDir}")+ buildGlobals.quietExec(it, {+ workingDir = targetDir+ executable = buildGlobals.externalTool("python3")+ args = ['-B', "${prog}"]+ })+ }+ // Correct line endings for Windows.+ project.ant.fixcrlf(+ srcDir: targetDir,+ includes: 'Packed64SingleBlock.java, BulkOperation*.java',+ encoding: 'UTF-8',+ eol: 'lf'+ )+ }+})++//+// Regenerate moman-based resources+//++def momanSource = "https://github.com/jpbarrette/moman/archive/497c90e34e412b6494db6dabf0d95db8034bd325.zip"+def momanDir = file("${buildDir}/moman")++def installMoman = tasks.register("installMoman", Download, {+ def momanZip = file("${momanDir}/moman.zip")++ src momanSource+ dest momanZip+ onlyIfModified true++ doLast {+ ant.unzip(src: momanZip, dest: momanDir, overwrite: "true") {+ ant.cutdirsmapper(dirs: "1")+ }+ }+})++tasks.register("regenerateParametricDescription", {+ dependsOn installMoman++ description = "Regenerate *ParametricDescription"+ regenerate {+ ifSkippedAlsoSkip = ["installMoman"]+ }++ def momanResources = rootProject.file("gradle/regenerate/moman")+ def targetDir = file("src/java/org/apache/lucene/util/automaton")+ def templateFile = file("${momanResources}/createLevAutomata.py")++ inputs.property("source", momanSource)+ inputs.file templateFile+ outputs.files fileTree(dir: targetDir, includes: ["*ParametricDescription.java"])++ doLast {+ ['1', '2'].each {num ->+ ['True', 'False'].each {transpose ->+ buildGlobals.quietExec(it, {+ workingDir = targetDir+ executable = buildGlobals.externalTool("python3")+ args = [+ '-B',+ templateFile.toString(),+ num,+ transpose,+ "${momanDir}/finenight/python"+ ]+ })+ }+ }+ project.ant.fixcrlf(+ srcDir: targetDir,+ includes: '*ParametricDescription.java',+ encoding: 'UTF-8',+ eol: 'lf'+ )+ }+})diff --git a/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json b/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.jsonindex 9942d9f77aa..0127428f6ed 100644--- a/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json+++ b/lucene/core/src/generated/checksums/regeneratePacked64SingleBlock.json@@ -1,6 +1,6 @@{- "gradle/regenerate/moman/gen_BulkOperation.py": "da9eccf0c8dd33325df5f1e76a1de74226035734",- "gradle/regenerate/moman/gen_Packed64SingleBlock.py": "0fd2a498c3edbd195a68c2d1d686be0aed51a104",+ "gradle/regenerate/packed/gen_BulkOperation.py": "da9eccf0c8dd33325df5f1e76a1de74226035734",+ "gradle/regenerate/packed/gen_Packed64SingleBlock.py": "0fd2a498c3edbd195a68c2d1d686be0aed51a104","lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java": "35a89e4f3dd37084cc059a784bcbeb384256773b","lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java": "2b0d9226bae8a07ce4970bcaa9d4d0cd4fe2c79a","lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java": "c483aa35c275bacc1f3a010c5b879441be502108",diff --git a/lucene/distribution.tests/build.gradle b/lucene/distribution.tests/build.gradleindex de4b2953361..76b73f7cb7a 100644--- a/lucene/distribution.tests/build.gradle+++ b/lucene/distribution.tests/build.gradle@@ -37,7 +37,7 @@ dependencies {moduleTestImplementation deps.assertj}-test {+tasks.named("test").configure {dependsOn configurations.binaryDistribution// We need to pass the system property using a lazy provider, not supported at the moment:diff --git a/lucene/expressions/build.gradle b/lucene/expressions/build.gradleindex 3fcfb7e4f84..b53cfdf7ff2 100644--- a/lucene/expressions/build.gradle+++ b/lucene/expressions/build.gradle@@ -24,3 +24,91 @@ dependencies {moduleImplementation deps.antlr.runtimemoduleTestImplementation project(':lucene:test-framework')}++//+// Regenerate antlr4 grammars.+//++configurations {+ antlr+}++dependencies {+ antlr deps.antlr.core+}++tasks.register("regenerateAntlr", {+ description = "Regenerate Javascript.g4"++ dependsOn configurations.antlr++ def generatedPatterns = [+ "**/JavascriptBaseVisitor.java",+ "**/JavascriptLexer.java",+ "**/JavascriptParser.java",+ "**/JavascriptVisitor.java",+ ]++ def sourceDir = "src/java/org/apache/lucene/expressions/js"+ def inputFiles = file("$sourceDir/Javascript.g4")+ def tempOutput = layout.buildDirectory.dir("antlr").get().asFile+ def outputFiles = fileTree(dir: sourceDir, includes: generatedPatterns)++ inputs.property "antlr-version", deps.antlr.core.get().version+ inputs.files inputFiles+ outputs.files outputFiles++ doFirst {+ buildGlobals.getFileOps().delete(tempOutput)+ buildGlobals.getExecOps().javaexec {+ mainClass = "org.antlr.v4.Tool"+ classpath = configurations.antlr++ ignoreExitValue = false+ args = [+ "-no-listener",+ "-visitor",+ "-package",+ "org.apache.lucene.expressions.js",+ "-o",+ tempOutput,+ inputFiles+ ]+ }++ def generatedFiles = fileTree(dir: tempOutput, includes: generatedPatterns)+ generatedFiles.each {file ->+ buildGlobals.modifyFile(file, {text ->+ text = text.replaceAll("public ((interface|class) Javascript\\w+)", "\$1")+ text = text.replaceAll("// Generated from .*",+ """+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// ANTLR GENERATED CODE: DO NOT EDIT.+""")+ return text+ })+ }++ buildGlobals.getFileOps().copy({CopySpec spec ->+ spec.from(tempOutput)+ spec.into(sourceDir)+ spec.include(generatedPatterns)+ })+ }+})diff --git a/lucene/queryparser/build.gradle b/lucene/queryparser/build.gradleindex 6e4fdb9443f..d3fed13119a 100644--- a/lucene/queryparser/build.gradle+++ b/lucene/queryparser/build.gradle@@ -15,7 +15,7 @@* limitations under the License.*/-+import org.apache.lucene.gradle.plugins.regenerate.JavaCCTaskdescription = 'Query parsers and parsing framework'@@ -26,3 +26,216 @@ dependencies {moduleTestImplementation project(':lucene:test-framework')}++//+// This adds regeneration of javacc-generated sources+//++def commonCleanups = { FileTree generatedFiles ->+ // This is a minor typo in a comment that nonetheless people have hand-corrected in the past.+ generatedFiles.matching({include "CharStream.java"}).each {file ->+ buildGlobals.modifyFile(file, {text ->+ return text.replace(+ "implemetation",+ "implementation");+ })+ }++ generatedFiles.each {file ->+ buildGlobals.modifyFile(file, {text ->+ // Normalize EOLs and tabs (EOLs are a side-effect of modifyFile).+ text = text.replace("\t", " ");+ text = text.replaceAll("JavaCC - OriginalChecksum=[^*]+", "(filtered)")+ text = text.replace("StringBuffer", "StringBuilder")+ return text+ })+ }++ generatedFiles.matching({include "*TokenManager.java"}).each {file ->+ buildGlobals.modifyFile(file, {text ->+ // Remove redundant imports.+ text = text.replaceAll(+ /(?m)^import .+/,+ "")+ // Add CharStream imports.+ text = text.replaceAll(+ /package (.+)/,+ '''+ package $1+ import org.apache.lucene.queryparser.charstream.CharStream;+ '''.trim())+ // Eliminates redundant cast message.+ text = text.replace(+ "int hiByte = (int)(curChar >> 8);",+ "int hiByte = curChar >> 8;")+ // Access to forbidden APIs.+ text = text.replace(+ "public java.io.PrintStream debugStream = System.out;",+ "// (debugStream omitted).")+ text = text.replace(+ "public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }",+ "// (setDebugStream omitted).")+ // Remove dangling doc comments+ text = text.replace(+ "/** Debug output. */",+ "// Debug output.")+ text = text.replace(+ "/** Set debug output. */",+ "// Set debug output.")+ return text+ })+ }+}++configurations.create("javacc")++dependencies {+ add("javacc", deps.javacc)+}++tasks.withType(JavaCCTask).configureEach {+ classpath = configurations.javacc+}++tasks.register("regenerateClassicQueryParser", JavaCCTask, {+ javaccFile = file('src/java/org/apache/lucene/queryparser/classic/QueryParser.jj')+ description = "Regenerate classic query parser from ${javaccFile}"++ afterGenerate(commonCleanups)+ afterGenerate({ FileTree generatedFiles ->+ generatedFiles.matching {include "QueryParser.java"}.each {file ->+ buildGlobals.modifyFile(file, {text ->+ text = text.replace(+ "public QueryParser(CharStream ",+ "protected QueryParser(CharStream ")+ text = text.replace(+ "public QueryParser(QueryParserTokenManager ",+ "protected QueryParser(QueryParserTokenManager ")+ text = text.replace(+ "new java.util.ArrayList<int[]>",+ "new java.util.ArrayList<>")+ text = text.replace(+ "final private LookaheadSuccess jj_ls =",+ "static final private LookaheadSuccess jj_ls =")+ text = text.replace(+ "public class QueryParser ",+ '@SuppressWarnings({"unused","null"}) public class QueryParser ')+ text = text.replace(+ "final public Query TopLevelQuery(",+ "@Override final public Query TopLevelQuery(")+ text = text.replace(+ "public void ReInit(CharStream ",+ "@Override public void ReInit(CharStream ")+ return text+ })+ }+ })+})+++tasks.register("regenerateSurroundQueryParser", JavaCCTask, {+ javaccFile = file('src/java/org/apache/lucene/queryparser/surround/parser/QueryParser.jj')+ description = "Regenerate surround query parser from ${javaccFile}"++ classpath = configurations.javacc++ afterGenerate(commonCleanups)++ afterGenerate({FileTree generatedFiles ->+ generatedFiles.matching {include "QueryParser.java"}.each {file ->+ buildGlobals.modifyFile(file, {text ->+ text = text.replace(+ "import org.apache.lucene.analysis.TokenStream;",+ "")+ text = text.replace(+ "new java.util.ArrayList<int[]>",+ "new java.util.ArrayList<>")+ text = text.replace(+ "public class QueryParser ",+ '@SuppressWarnings({"unused","null"}) public class QueryParser ')+ return text+ })+ }+ })+})++tasks.register("regenerateFlexibleStandardSyntaxParser", JavaCCTask, {+ javaccFile = file('src/java/org/apache/lucene/queryparser/flexible/standard/parser/StandardSyntaxParser.jj')+ description = "Regenerate flexible query parser from ${javaccFile}"++ classpath = configurations.javacc++ afterGenerate(commonCleanups)++ afterGenerate({FileTree generatedFiles ->+ generatedFiles.matching {include "ParseException.java"}.each {file ->+ buildGlobals.modifyFile(file, {text ->+ // Modify constructor.+ text = text.replace(+ "class ParseException extends Exception",+ "class ParseException extends QueryNodeParseException")++ // Modify imports.+ text = text.replace(+ "package org.apache.lucene.queryparser.flexible.standard.parser;", '''\+ package org.apache.lucene.queryparser.flexible.standard.parser;++ import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException;+ import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages;+ import org.apache.lucene.queryparser.flexible.messages.Message;+ import org.apache.lucene.queryparser.flexible.messages.MessageImpl;+ ''')++ // Modify constructors and code bits+ text = text.replaceAll(+ /(?s)[ ]*public ParseException\(Token currentTokenVal[^}]+[}]/, '''\+ public ParseException(Token currentTokenVal,+ int[][] expectedTokenSequencesVal, String[] tokenImageVal)+ {+ super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, initialise(+ currentTokenVal, expectedTokenSequencesVal, tokenImageVal)));+ this.currentToken = currentTokenVal;+ this.expectedTokenSequences = expectedTokenSequencesVal;+ this.tokenImage = tokenImageVal;+ }+ ''')++ text = text.replaceAll(+ /(?s)[ ]*public ParseException\(String message\)[^}]+[}]/, '''\+ public ParseException(Message message)+ {+ super(message);+ }+ ''')++ text = text.replaceAll(+ /(?s)[ ]*public ParseException\(\)[^}]+[}]/, '''\+ public ParseException()+ {+ super(new MessageImpl(QueryParserMessages.INVALID_SYNTAX, "Error"));+ }+ ''')+ return text+ })+ }++ generatedFiles.matching {include "StandardSyntaxParser.java"}.each {file ->+ buildGlobals.modifyFile(file, {text ->+ // Remove redundant cast+ text = text.replace(+ "new java.util.ArrayList<int[]>",+ "new java.util.ArrayList<>")+ text = text.replace(+ "new ArrayList<QueryNode>()",+ "new ArrayList<>()")+ text = text.replace(+ "Collections.<QueryNode> singletonList",+ "Collections.singletonList")+ text = text.replace(+ "public class StandardSyntaxParser ",+ '@SuppressWarnings({"unused","null"}) public class StandardSyntaxParser ')+ return text+ })+ }+ })+})diff --git a/lucene/test-framework/build.gradle b/lucene/test-framework/build.gradleindex 54128c52427..36687e87b83 100644--- a/lucene/test-framework/build.gradle+++ b/lucene/test-framework/build.gradle@@ -15,7 +15,7 @@* limitations under the License.*/-+import org.apache.lucene.gradle.plugins.misc.QuietExecdescription = 'Framework for testing Lucene-based applications'@@ -32,3 +32,51 @@ dependencies {moduleImplementation project(':lucene:codecs')}++//+// Regenerate unicode resources+//++tasks.register("regenerateEmojiTokenizationTest", QuietExec, {+ def unicodeVersion = "12.1"++ def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")+ def genScript = file("${genDir}/generateEmojiTokenizationTest.pl")+ def genOutput = file("${genDir}/EmojiTokenizationTestUnicode_${unicodeVersion.replace('.', '_')}.java")++ description = "Regenerate ${genOutput}"++ inputs.file genScript+ inputs.property "unicodeVersion", unicodeVersion+ outputs.file genOutput++ workingDir = genDir+ executable = buildGlobals.externalTool("perl")+ args = [+ genScript,+ "-v",+ unicodeVersion+ ]+})++tasks.register("regenerateJavaUnicodeWordBreakTest", QuietExec, {+ def unicodeVersion = "12.1"++ def genDir = file("src/java/org/apache/lucene/tests/analysis/standard")+ def genScript = file("${genDir}/generateJavaUnicodeWordBreakTest.pl")+ def genOutput = file("${genDir}/WordBreakTestUnicode_${unicodeVersion.replace('.', '_')}.java")++ description = "Regenerate ${genOutput}"++ inputs.file genScript+ inputs.property "unicodeVersion", unicodeVersion+ outputs.file genOutput++ workingDir = genDir+ executable = buildGlobals.externalTool("perl")+ args = [+ genScript,+ "-v",+ unicodeVersion+ ]+})