Apache Solr open-source search software
SOLR-18031: Respect --filetypes for files and globs in bin/solr post (#5032)
sarkaramrit1993d97bc70cefb8
diff --git a/changelog/unreleased/solr-18031-post-filetypes.yml b/changelog/unreleased/solr-18031-post-filetypes.ymlnew file mode 100644index 00000000000..65e19647a09--- /dev/null+++ b/changelog/unreleased/solr-18031-post-filetypes.yml@@ -0,0 +1,7 @@+title: bin/solr post now honors --filetypes for files and globs, and reports the right file count.+type: fixed+authors:+ - name: Amrit Sarkar+links:+ - name: SOLR-18031+ url: https://issues.apache.org/jira/browse/SOLR-18031diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.javaindex 8c455cefa3a..afd1c50467a 100644--- a/solr/core/src/java/org/apache/solr/cli/PostTool.java+++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java@@ -482,6 +482,10 @@ public class PostTool extends ToolBase {return Files.exists(srcFile);}+ private boolean matchesFileTypes(Path file) {+ return !auto || fileFilter == null || fileFilter.test(file);+ }+private static Collection<Path> listFiles(Path directory, Predicate<Path> fileFilter)throws IOException {Predicate<Path> filter = fileFilter != null ? fileFilter : p -> true;@@ -523,7 +527,7 @@ public class PostTool extends ToolBase {reset();int filesPosted = 0;for (int j = startIndexInArgs; j < args.length; j++) {- filesPosted = getFilesPosted(out, type, args[j]);+ filesPosted += getFilesPosted(out, type, args[j]);}return filesPosted;}@@ -536,7 +540,11 @@ public class PostTool extends ToolBase {if (isValidPath && Files.isDirectory(srcFile) && Files.isReadable(srcFile)) {filesPosted += postDirectory(srcFile, out, type);} else if (isValidPath && Files.isRegularFile(srcFile) && Files.isReadable(srcFile)) {- filesPosted += postFiles(List.of(srcFile), out, type);+ if (matchesFileTypes(srcFile)) {+ filesPosted += postFiles(List.of(srcFile), out, type);+ } else {+ warn("Skipping " + srcFile + ", its file type is not in " + fileTypes);+ }} else {filesPosted += handleGlob(src, out, type);}@@ -612,9 +620,12 @@ public class PostTool extends ToolBase {}String fileGlob = globPath.getFileName().toString();GlobFilter ff = new GlobFilter(fileGlob, false);- Collection<Path> fileList = listFiles(parent, ff);- if (fileList.isEmpty()) {+ Collection<Path> matches = listFiles(parent, ff);+ List<Path> fileList = matches.stream().filter(this::matchesFileTypes).toList();+ if (matches.isEmpty()) {warn("No files or directories matching " + globPath);+ } else if (fileList.isEmpty()) {+ warn("No files matching " + globPath + " have a file type in " + fileTypes);} else {filesPosted = postFiles(fileList, out, type);}diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.javaindex a238bd62c15..5390fbeafcf 100644--- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java+++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java@@ -33,6 +33,7 @@ import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import org.apache.lucene.util.Constants;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrResponse;@@ -293,6 +294,45 @@ public class PostToolTest extends SolrCloudTestCase {assertEquals(2, num);}+ @Test+ public void testFileTypesAppliedToFileArgs() throws IOException {+ Path dir = createTempDir();+ Path a = Files.writeString(dir.resolve("a.xml"), "<add/>");+ Path b = Files.writeString(dir.resolve("B.XML"), "<add/>");+ Path c = Files.writeString(dir.resolve("c.csv"), "id\n1\n");+ Path d = Files.writeString(dir.resolve("noext"), "text");++ PostTool postTool = dryRunPostTool("xml");+ String[] files = {a.toString(), b.toString(), c.toString(), d.toString()};+ assertEquals(2, postTool.postFiles(files, 0, null, null));++ postTool.auto = false;+ assertEquals(4, postTool.postFiles(files, 0, null, null));+ }++ @Test+ public void testFileTypesAppliedToGlobs() throws IOException {+ assumeFalse("'*' is not a valid path character on Windows", Constants.WINDOWS);+ Path dir = createTempDir();+ Files.writeString(dir.resolve("a.xml"), "<add/>");+ Files.writeString(dir.resolve("B.XML"), "<add/>");+ Files.writeString(dir.resolve("c.csv"), "id\n1\n");++ PostTool postTool = dryRunPostTool("xml");+ assertEquals(2, postTool.postFiles(new String[] {dir + "/*"}, 0, null, null));+ assertEquals(0, postTool.postFiles(new String[] {dir + "/*.csv"}, 0, null, null));+ }++ private PostTool dryRunPostTool(String fileTypes) {+ PostTool postTool = new PostTool(new CLITestHelper.TestingRuntime(false));+ postTool.recursive = 0;+ postTool.dryRun = true;+ postTool.solrUpdateUrl = URI.create("http://localhost:8983/solr/fake/update");+ postTool.fileTypes = fileTypes;+ postTool.fileFilter = postTool.getFileFilterFromFileTypes(fileTypes);+ return postTool;+ }+@Testpublic void testDetectingIfRecursionPossibleInFilesMode() throws IOException {ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/post-command.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/post-command.adocindex 9c008b51ab3..b7a8cfb8738 100644--- a/solr/solr-ref-guide/modules/indexing-guide/pages/post-command.adoc+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/post-command.adoc@@ -121,6 +121,9 @@ Index all CSV and JSON files into `gettingstarted` from current directory:$ bin/solr post -c gettingstarted --filetypes json,csv .----+`--filetypes` also applies to files named on the command line, including a shell-expanded `*`.+A file with any other extension is skipped with a warning, unless `--type` is set.+Index a tab-separated file into `gettingstarted`:[,console]