Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9767: infrastructure for icu regeneration in place. (#2362)

Dawid Weiss8f56ae0a4b8a
diff --git a/build.gradle b/build.gradle
index 37e52cd009d..25086fd5248 100644
--- a/build.gradle
+++ b/build.gradle
@@ -154,6 +154,7 @@ apply from: file('gradle/generation/util.gradle')
apply from: file('gradle/generation/snowball.gradle')
apply from: file('gradle/generation/kuromoji.gradle')
apply from: file('gradle/generation/nori.gradle')
+apply from: file('gradle/generation/icu.gradle')
// Shared configuration of subprojects containing native code.
apply from: file('gradle/native/disable-native.gradle')
diff --git a/gradle/generation/icu.gradle b/gradle/generation/icu.gradle
new file mode 100644
index 00000000000..a68b0e189e5
--- /dev/null
+++ b/gradle/generation/icu.gradle
@@ -0,0 +1,210 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.nio.file.Files
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Regenerates ICU data files.
+ *
+ * The icu4c version must match exactly the icu4j version in version.props:
+ * The one on your system is probably different. This script will attempt to
+ * download and compile a matching icu4c version automatically.
+ */
+configure(project(":lucene:analysis:icu")) {
+ def utr30DataDir = file("src/data/utr30")
+
+ def icuBuildDir = file("${buildDir}/icu")
+ def icuBinDir
+ def gennorm
+ def icupkg
+
+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ icuBinDir = file("${icuBuildDir}/bin64")
+ gennorm = file("${icuBinDir}/gennorm2.exe")
+ icupkg = file("${icuBinDir}/icupkg.exe")
+ } else {
+ icuBinDir = file("${icuBuildDir}/icu/source/bin")
+ gennorm = file("${icuBinDir}/gennorm2")
+ icupkg = file("${icuBinDir}/icupkg")
+ }
+
+ task genUtr30DataFiles() {
+ dependsOn Os.isFamily(Os.FAMILY_WINDOWS) ? "compileIcuWindows" : "compileIcuLinux"
+
+ // May be undefined yet, so use a provider.
+ dependsOn { sourceSets.tools.runtimeClasspath }
+
+ doFirst {
+ // all these steps must be done sequentially: it's a pipeline resulting in utr30.nrm
+ project.javaexec {
+ main = "org.apache.lucene.analysis.icu.GenerateUTR30DataFiles"
+ classpath = sourceSets.tools.runtimeClasspath
+
+ ignoreExitValue false
+ workingDir utr30DataDir
+ }
+
+ project.exec {
+ executable gennorm
+ ignoreExitValue = false
+ args = [
+ "-v",
+ "-s",
+ utr30DataDir,
+ "-o",
+ "${buildDir}/utr30.tmp",
+ "nfc.txt", "nfkc.txt", "nfkc_cf.txt", "BasicFoldings.txt",
+ "DiacriticFolding.txt", "DingbatFolding.txt", "HanRadicalFolding.txt",
+ "NativeDigitFolding.txt"
+ ]
+ }
+ project.exec {
+ executable icupkg
+ ignoreExitValue = false
+ args = [
+ "-tb",
+ "${buildDir}/utr30.tmp",
+ "src/resources/org/apache/lucene/analysis/icu/utr30.nrm"
+ ]
+ }
+ }
+ }
+
+ task genRbbi() {
+ // May be undefined yet, so use a provider.
+ dependsOn { sourceSets.tools.runtimeClasspath }
+
+ doFirst {
+ project.javaexec {
+ main = "org.apache.lucene.analysis.icu.RBBIRuleCompiler"
+ classpath = sourceSets.tools.runtimeClasspath
+
+ ignoreExitValue false
+ enableAssertions true
+ args = [
+ "src/data/uax29",
+ "src/resources/org/apache/lucene/analysis/icu/segmentation"
+ ]
+ }
+ }
+ }
+
+ task regenerate() {
+ dependsOn genUtr30DataFiles
+ dependsOn genRbbi
+ }
+
+ task compileIcuWindows() {
+ doFirst {
+ def v = getVersion('com.ibm.icu', 'icu4j');
+
+ def icuBinZip = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}.zip")
+ if (!icuBinZip.exists()) {
+ icuBuildDir.mkdirs()
+
+ // No official Windows binaries for ICU 62.2... I've checked the script
+ // against ICU 68.2 (below) and it works but results in a different generated file.
+ // def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-68-2/icu4c-68_2-Win64-MSVC2019.zip")
+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-Win64-MSVC2019.zip")
+
+ logger.lifecycle("Trying to download binary ICU version: ${v} from:\n ${src}")
+ Files.write(icuBinZip.toPath(), src.toURL().openStream().bytes)
+ logger.lifecycle("Downloaded ${icuBinZip.size()} bytes.")
+ }
+
+ // Unzip.
+ project.copy {
+ into icuBuildDir
+ from zipTree(icuBinZip)
+ }
+
+ }
+ }
+
+ task compileIcuLinux() {
+ doFirst {
+ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+ throw new GradleException("ICU compilation not supported on Windows.")
+ }
+
+ def v = getVersion('com.ibm.icu', 'icu4j');
+ def icuSrcTgz = file("${icuBuildDir}/icu4c-${v.replace(".", "_")}-src.tgz")
+
+ // download version matching icu4j version in version.props
+ // curl -fLO https://github.com/unicode-org/icu/releases/download/release-62-2/icu4c-62_2-src.tgz
+ if (!icuSrcTgz.exists()) {
+ icuBuildDir.mkdirs()
+ def src = URI.create("https://github.com/unicode-org/icu/releases/download/release-${v.replace(".", "-")}/icu4c-${v.replace(".", "_")}-src.tgz")
+ logger.lifecycle("Trying to download and compile ICU version: ${v} from:\n ${src}")
+ Files.write(icuSrcTgz.toPath(), src.toURL().openStream().bytes)
+ logger.lifecycle("Downloaded ${icuSrcTgz.size()} bytes.")
+ }
+
+ def icuSrcDir = file("${icuBuildDir}/icu/source")
+ project.delete icuSrcDir
+
+ // extract tgz
+ // tar -zxvf icu4c-62_2-src.tgz
+ project.exec {
+ executable "tar"
+ ignoreExitValue false
+
+ workingDir icuBuildDir
+ args = [
+ "-zxvf",
+ icuSrcTgz
+ ]
+ }
+
+ // compile
+ // (cd icu/source && ./configure --prefix=$(pwd) --enable-rpath && make -j4)
+ project.exec {
+ executable "sh"
+ ignoreExitValue false
+
+ workingDir icuSrcDir
+ args = [
+ "configure",
+ "--prefix=${icuSrcDir}",
+ "--enable-rpath"
+ ]
+ }
+
+ project.exec {
+ executable "make"
+ ignoreExitValue false
+ workingDir icuSrcDir
+ args = [
+ "-j4"
+ ]
+ }
+
+ // test that the binaries work
+ // derb -V
+ logger.lifecycle("Compiled ICU, checking...")
+ project.exec {
+ executable "./derb"
+ ignoreExitValue false
+ workingDir icuBinDir
+ args = [
+ "-V"
+ ]
+ }
+ }
+ }
+}
diff --git a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
index bb7d8a3f7e6..ee5ea85d33b 100644
--- a/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
+++ b/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateUTR30DataFiles.java
@@ -61,7 +61,7 @@ public class GenerateUTR30DataFiles {
private static final String NFC_TXT = "nfc.txt";
private static final String NFKC_TXT = "nfkc.txt";
private static final String NFKC_CF_TXT = "nfkc_cf.txt";
- private static byte[] DOWNLOAD_BUFFER = new byte[8192];
+
private static final Pattern ROUND_TRIP_MAPPING_LINE_PATTERN =
Pattern.compile("^\\s*([^=]+?)\\s*=\\s*(.*)$");
private static final Pattern VERBATIM_RULE_LINE_PATTERN =
@@ -209,16 +209,9 @@ public class GenerateUTR30DataFiles {
private static void download(URL url, String outputFile) throws IOException {
final URLConnection connection = openConnection(url);
- final InputStream inputStream = connection.getInputStream();
- final OutputStream outputStream = Files.newOutputStream(Path.of(outputFile));
- int numBytes;
- try {
- while (-1 != (numBytes = inputStream.read(DOWNLOAD_BUFFER))) {
- outputStream.write(DOWNLOAD_BUFFER, 0, numBytes);
- }
- } finally {
- inputStream.close();
- outputStream.close();
+ try (InputStream inputStream = connection.getInputStream();
+ OutputStream outputStream = Files.newOutputStream(Path.of(outputFile))) {
+ inputStream.transferTo(outputStream);
}
}