Skip to content

Apache Lucene open-source search software

Correct getEnWikiRandomLines and getEnWiki URLs to use s3 bucket. Use zst instead of bz2 (smaller size, faster decompression). #13647

Dawid Weiss1a023bb6f5c1
diff --git a/gradle/datasets/external-datasets.gradle b/gradle/datasets/external-datasets.gradle
index bcaa5ad4e10..b9601ebf6f7 100644
--- a/gradle/datasets/external-datasets.gradle
+++ b/gradle/datasets/external-datasets.gradle
@@ -47,14 +47,11 @@ configure(project(":lucene:benchmark")) {
task getEnWiki(type: Download) {
ext {
name = "enwiki-20070527-pages-articles.xml"
- src = "https://home.apache.org/~dsmiley/data/${name}.bz2"
+ src = "https://s3.amazonaws.com/lucene-testdata/wikipedia/${name}.zst"
intermediate = file("${dataDir}/${name}.bz2")
dst = file("${dataDir}/${name}")
}
- // TODO: dataset gone. https://github.com/apache/lucene/issues/13647
- onlyIf { false }
-
outputs.file ext.dst
src ext.src
@@ -64,21 +61,18 @@ configure(project(":lucene:benchmark")) {
doLast {
logger.lifecycle("Decompressing ${ext.name}...")
- ant.bunzip2(src: ext.intermediate, dest: ext.dst)
+ unzstd(ext.intermediate.toPath(), ext.dst.toPath())
}
}
task getEnWikiRandomLines(type: Download) {
ext {
name = "enwiki.random.lines.txt"
- src = "https://home.apache.org/~mikemccand/${name}.zst"
+ src = "https://s3.amazonaws.com/lucene-testdata/wikipedia/${name}.zst"
intermediate = file("${dataDir}/${name}.zst")
dst = file("${dataDir}/${name}")
}
- // TODO: dataset gone. https://github.com/apache/lucene/issues/13647
- onlyIf { false }
-
outputs.file ext.dst
src ext.src