Apache Solr open-source search software
SOLR-16027: Enable spotless on extraction module
Kevin Risdenf0cd4cdeeab6
diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradleindex 031e25182c8..c1c068e3e4b 100644--- a/gradle/validation/spotless.gradle+++ b/gradle/validation/spotless.gradle@@ -44,7 +44,6 @@ configure(project(":solr").subprojects) { prj ->// Exclude certain files (generated ones, mostly).switch (project.path) {- case ":solr:modules:extraction":case ":solr:modules:gcs-repository":case ":solr:modules:hadoop-auth":case ":solr:modules:hdfs":diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.javaindex 5be32e0c260..615986b5b81 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingDocumentLoader.java@@ -21,7 +21,6 @@ import java.io.InputStream;import java.io.StringWriter;import java.lang.invoke.MethodHandles;import java.util.Locale;-import org.apache.commons.io.IOUtils;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.SolrParams;@@ -60,28 +59,17 @@ import org.slf4j.LoggerFactory;import org.xml.sax.ContentHandler;import org.xml.sax.SAXException;--/**- * The class responsible for loading extracted content into Solr.- *- **/+/** The class responsible for loading extracted content into Solr. */public class ExtractingDocumentLoader extends ContentStreamLoader {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());- /**- * Extract Only supported format- */+ /** Extract Only supported format */public static final String TEXT_FORMAT = "text";- /**- * Extract Only supported format. Default- */+ /** Extract Only supported format. Default */public static final String XML_FORMAT = "xml";- /**- * XHTML XPath parser.- */- private static final XPathParser PARSER =- new XPathParser("xhtml", XHTMLContentHandler.XHTML);+ /** XHTML XPath parser. */+ private static final XPathParser PARSER = new XPathParser("xhtml", XHTMLContentHandler.XHTML);final SolrCore core;final SolrParams params;@@ -95,9 +83,12 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {protected ParseContextConfig parseContextConfig;protected SolrContentHandlerFactory factory;- public ExtractingDocumentLoader(SolrQueryRequest req, UpdateRequestProcessor processor,- TikaConfig config, ParseContextConfig parseContextConfig,- SolrContentHandlerFactory factory) {+ public ExtractingDocumentLoader(+ SolrQueryRequest req,+ UpdateRequestProcessor processor,+ TikaConfig config,+ ParseContextConfig parseContextConfig,+ SolrContentHandlerFactory factory) {this.params = req.getParams();this.core = req.getCore();this.config = config;@@ -108,20 +99,15 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true);templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1);- //this is lightweight+ // this is lightweightautoDetectParser = new AutoDetectParser(config);this.factory = factory;-+ignoreTikaException = params.getBool(ExtractingParams.IGNORE_TIKA_EXCEPTION, false);}-- /**- * this must be MT safe... may be called concurrently from multiple threads.- *- */- void doAdd(SolrContentHandler handler, AddUpdateCommand template)- throws IOException {+ /** this must be MT safe... may be called concurrently from multiple threads. */+ void doAdd(SolrContentHandler handler, AddUpdateCommand template) throws IOException {template.solrDoc = handler.newDocument();processor.processAdd(template);}@@ -132,12 +118,16 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {}@Override- public void load(SolrQueryRequest req, SolrQueryResponse rsp,- ContentStream stream, UpdateRequestProcessor processor) throws Exception {+ public void load(+ SolrQueryRequest req,+ SolrQueryResponse rsp,+ ContentStream stream,+ UpdateRequestProcessor processor)+ throws Exception {Parser parser = null;String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null);if (streamType != null) {- //Cache? Parsers are lightweight to construct and thread-safe, so I'm told+ // Cache? Parsers are lightweight to construct and thread-safe, so I'm toldMediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT));parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt);} else {@@ -153,7 +143,7 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName);}// Provide stream's content type as hint for auto detection- if(stream.getContentType() != null) {+ if (stream.getContentType() != null) {metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType());}@@ -166,13 +156,14 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {metadata.add(ExtractingMetadataConstants.STREAM_CONTENT_TYPE, stream.getContentType());// HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadataString charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());- if(charset != null){+ if (charset != null) {metadata.add(HttpHeaders.CONTENT_ENCODING, charset);}String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION);boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false);- SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, req.getSchema());+ SolrContentHandler handler =+ factory.createSolrContentHandler(metadata, params, req.getSchema());ContentHandler parsingHandler = handler;StringWriter writer = null;@@ -188,56 +179,59 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {serializer = new XMLSerializer(writer, new OutputFormat("XML", "UTF-8", true));}if (xpathExpr != null) {- Matcher matcher =- PARSER.parse(xpathExpr);- serializer.startDocument();//The MatchingContentHandler does not invoke startDocument. See https://lists.apache.org/thread.html/5ec63e104e564a2363e45f74d5aced6520b7d32b4b625762ef56cb86%401226775505%40%3Cdev.tika.apache.org%3E+ Matcher matcher = PARSER.parse(xpathExpr);+ serializer+ .startDocument(); // The MatchingContentHandler does not invoke startDocument. See+ // https://lists.apache.org/thread.html/5ec63e104e564a2363e45f74d5aced6520b7d32b4b625762ef56cb86%401226775505%40%3Cdev.tika.apache.org%3EparsingHandler = new MatchingContentHandler(serializer, matcher);} else {parsingHandler = serializer;}} else if (xpathExpr != null) {- Matcher matcher =- PARSER.parse(xpathExpr);+ Matcher matcher = PARSER.parse(xpathExpr);parsingHandler = new MatchingContentHandler(handler, matcher);- } //else leave it as is+ } // else leave it as is- try{- //potentially use a wrapper handler for parsing, but we still need the SolrContentHandler for getting the document.+ try {+ // potentially use a wrapper handler for parsing, but we still need the SolrContentHandler+ // for getting the document.ParseContext context = parseContextConfig.create();-context.set(Parser.class, parser);context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE);// Password handlingRegexRulesPasswordProvider epp = new RegexRulesPasswordProvider();String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE);- if(pwMapFile != null && pwMapFile.length() > 0) {+ if (pwMapFile != null && pwMapFile.length() > 0) {InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile);- if(is != null) {+ if (is != null) {log.debug("Password file supplied: {}", pwMapFile);epp.parse(is);}}context.set(PasswordProvider.class, epp);String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD);- if(resourcePassword != null) {+ if (resourcePassword != null) {epp.setExplicitPassword(resourcePassword);log.debug("Literal password supplied for file {}", resourceName);}parser.parse(inputStream, parsingHandler, metadata, context);} catch (TikaException e) {- if(ignoreTikaException)- log.warn(new StringBuilder("skip extracting text due to ").append(e.getLocalizedMessage())- .append(". metadata=").append(metadata.toString()).toString()); // nowarn- else- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ if (ignoreTikaException)+ log.warn(+ new StringBuilder("skip extracting text due to ")+ .append(e.getLocalizedMessage())+ .append(". metadata=")+ .append(metadata.toString())+ .toString()); // nowarn+ else throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);}if (extractOnly == false) {addDoc(handler);} else {- //serializer is not null, so we need to call endDoc on it if using xpath- if (xpathExpr != null){+ // serializer is not null, so we need to call endDoc on it if using xpath+ if (xpathExpr != null) {serializer.endDocument();}rsp.add(stream.getName(), writer.toString());@@ -256,20 +250,26 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {IOUtils.closeQuietly(inputStream);}} else {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stream type of " + streamType + " didn't match any known parsers. Please supply the " + ExtractingParams.STREAM_TYPE + " parameter.");+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Stream type of "+ + streamType+ + " didn't match any known parsers. Please supply the "+ + ExtractingParams.STREAM_TYPE+ + " parameter.");}}public static class MostlyPassthroughHtmlMapper implements HtmlMapper {public static final HtmlMapper INSTANCE = new MostlyPassthroughHtmlMapper();- /**+ /*** Keep all elements and their content.- *- * Apparently <SCRIPT> and <STYLE> elements are blocked elsewhere+ *+ * <p>Apparently <SCRIPT> and <STYLE> elements are blocked elsewhere*/@Override- public boolean isDiscardElement(String name) {+ public boolean isDiscardElement(String name) {return false;}@@ -280,15 +280,14 @@ public class ExtractingDocumentLoader extends ContentStreamLoader {}/**- * Lowercases the element name, but returns null for <BR>,- * which suppresses the start-element event for lt;BR> tags.- * This also suppresses the <BODY> tags because those- * are handled internally by Tika's XHTMLContentHandler.+ * Lowercases the element name, but returns null for <BR>, which suppresses the+ * start-element event for lt;BR> tags. This also suppresses the <BODY> tags because+ * those are handled internally by Tika's XHTMLContentHandler.*/@Overridepublic String mapSafeElement(String name) {String lowerName = name.toLowerCase(Locale.ROOT);return (lowerName.equals("br") || lowerName.equals("body")) ? null : lowerName;}- }- }\ No newline at end of file+ }+}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.javaindex 71aced1b69c..0a72edca0ec 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingMetadataConstants.java@@ -16,11 +16,7 @@*/package org.apache.solr.handler.extraction;--/**- * Constants used internally by the {@link ExtractingRequestHandler}.- *- **/+/** Constants used internally by the {@link ExtractingRequestHandler}. */public interface ExtractingMetadataConstants {String STREAM_NAME = "stream_name";String STREAM_SOURCE_INFO = "stream_source_info";diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.javaindex f7917bb399a..a7d159678f1 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingParams.java@@ -16,86 +16,75 @@*/package org.apache.solr.handler.extraction;--/**- * The various Solr Parameters names to use when extracting content.- *- **/+/** The various Solr Parameters names to use when extracting content. */public interface ExtractingParams {- /**- * Map all generated attribute names to field names with lowercase and underscores.- */+ /** Map all generated attribute names to field names with lowercase and underscores. */public static final String LOWERNAMES = "lowernames";- /**- * if true, ignore TikaException (give up to extract text but index meta data)- */+ /** if true, ignore TikaException (give up to extract text but index meta data) */public static final String IGNORE_TIKA_EXCEPTION = "ignoreTikaException";-/*** The param prefix for mapping Tika metadata to Solr fields.- * <p>- * To map a field, add a name like:- * <pre>fmap.title=solr.title</pre>*- * In this example, the tika "title" metadata value will be added to a Solr field named "solr.title"+ * <p>To map a field, add a name like:*+ * <pre>fmap.title=solr.title</pre>*+ * In this example, the tika "title" metadata value will be added to a Solr field named+ * "solr.title"*/public static final String MAP_PREFIX = "fmap.";/*** Pass in literal values to be added to the document, as in+ ** <pre>- * literal.myField=Foo+ * literal.myField=Foo* </pre>- **/public static final String LITERALS_PREFIX = "literal.";-/**- * Restrict the extracted parts of a document to be indexed- * by passing in an XPath expression. All content that satisfies the XPath expr.- * will be passed to the {@link SolrContentHandler}.- * <p>- * See Tika's docs for what the extracted document looks like.+ * Restrict the extracted parts of a document to be indexed by passing in an XPath expression. All+ * content that satisfies the XPath expr. will be passed to the {@link SolrContentHandler}.+ *+ * <p>See Tika's docs for what the extracted document looks like.+ ** @see #CAPTURE_ELEMENTS*/public static final String XPATH_EXPRESSION = "xpath";-- /**- * Only extract and return the content, do not index it.- */+ /** Only extract and return the content, do not index it. */public static final String EXTRACT_ONLY = "extractOnly";- /**- * Content output format if extractOnly is true. Default is "xml", alternative is "text".- */+ /** Content output format if extractOnly is true. Default is "xml", alternative is "text". */public static final String EXTRACT_FORMAT = "extractFormat";/**- * Capture attributes separately according to the name of the element, instead of just adding them to the string buffer+ * Capture attributes separately according to the name of the element, instead of just adding them+ * to the string buffer*/public static final String CAPTURE_ATTRIBUTES = "captureAttr";/**- * Literal field values will by default override other values such as metadata and content. Set this to false to revert to pre-4.0 behaviour+ * Literal field values will by default override other values such as metadata and content. Set+ * this to false to revert to pre-4.0 behaviour*/public static final String LITERALS_OVERRIDE = "literalsOverride";/**- * Capture the specified fields (and everything included below it that isn't capture by some other capture field) separately from the default. This is different- * then the case of passing in an XPath expression.- * <p>- * The Capture field is based on the localName returned to the {@link SolrContentHandler}- * by Tika, not to be confused by the mapped field. The field name can then- * be mapped into the index schema.- * <p>- * For instance, a Tika document may look like:+ * Capture the specified fields (and everything included below it that isn't capture by some other+ * capture field) separately from the default. This is different then the case of passing in an+ * XPath expression.+ *+ * <p>The Capture field is based on the localName returned to the {@link SolrContentHandler} by+ * Tika, not to be confused by the mapped field. The field name can then be mapped into the index+ * schema.+ *+ * <p>For instance, a Tika document may look like:+ ** <pre>* <html>* ...@@ -104,48 +93,47 @@ public interface ExtractingParams {* Some more text* </body>* </pre>- * By passing in the p tag, you could capture all P tags separately from the rest of the t- * Thus, in the example, the capture of the P tag would be: "some text here. more text"*+ * By passing in the p tag, you could capture all P tags separately from the rest of the t Thus,+ * in the example, the capture of the P tag would be: "some text here. more text"*/public static final String CAPTURE_ELEMENTS = "capture";- /**- * The type of the stream. If not specified, Tika will use mime type detection.- */+ /** The type of the stream. If not specified, Tika will use mime type detection. */public static final String STREAM_TYPE = "stream.type";-/**- * Optional. The file name. If specified, Tika can take this into account while- * guessing the MIME type.+ * Optional. The file name. If specified, Tika can take this into account while guessing the MIME+ * type.*/public static final String RESOURCE_NAME = "resource.name";/**- * Optional. The password for this resource. Will be used instead of the rule based password lookup mechanisms+ * Optional. The password for this resource. Will be used instead of the rule based password+ * lookup mechanisms*/public static final String RESOURCE_PASSWORD = "resource.password";/**- * Optional. If specified, the prefix will be prepended to all Metadata, such that it would be possible- * to setup a dynamic field to automatically capture it+ * Optional. If specified, the prefix will be prepended to all Metadata, such that it would be+ * possible to setup a dynamic field to automatically capture it*/public static final String UNKNOWN_FIELD_PREFIX = "uprefix";/**- * Optional. If specified and the name of a potential field cannot be determined, the default Field specified- * will be used instead.+ * Optional. If specified and the name of a potential field cannot be determined, the default+ * Field specified will be used instead.*/public static final String DEFAULT_FIELD = "defaultField";/**- * Optional. If specified, loads the file as a source for password lookups for Tika encrypted documents.- * <p>- * File format is Java properties format with one key=value per line.- * The key is evaluated as a regex against the file name, and the value is the password- * The rules are evaluated top-bottom, i.e. the first match will be used- * If you want a fallback password to be always used, supply a .*=<defaultmypassword> at the end+ * Optional. If specified, loads the file as a source for password lookups for Tika encrypted+ * documents.+ *+ * <p>File format is Java properties format with one key=value per line. The key is evaluated as a+ * regex against the file name, and the value is the password The rules are evaluated top-bottom,+ * i.e. the first match will be used If you want a fallback password to be always used, supply a+ * .*=<defaultmypassword> at the end*/public static final String PASSWORD_MAP_FILE = "passwordsFile";}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.javaindex 340ca8f0eff..99222c04964 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ExtractingRequestHandler.java@@ -18,7 +18,6 @@ package org.apache.solr.handler.extraction;import java.io.File;import java.io.InputStream;-import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.core.SolrCore;@@ -32,10 +31,11 @@ import org.apache.solr.util.plugin.SolrCoreAware;import org.apache.tika.config.TikaConfig;/**- * Handler for rich documents like PDF or Word or any other file format that Tika handles that need the text to be extracted- * first from the document.+ * Handler for rich documents like PDF or Word or any other file format that Tika handles that need+ * the text to be extracted first from the document.*/-public class ExtractingRequestHandler extends ContentStreamHandlerBase implements SolrCoreAware , PermissionNameProvider {+public class ExtractingRequestHandler extends ContentStreamHandlerBase+ implements SolrCoreAware, PermissionNameProvider {public static final String PARSE_CONTEXT_CONFIG = "parseContext.config";public static final String CONFIG_LOCATION = "tika.config";@@ -74,7 +74,8 @@ public class ExtractingRequestHandler extends ContentStreamHandlerBase implementif (parseContextConfigLoc == null) { // default:parseContextConfig = new ParseContextConfig();} else {- parseContextConfig = new ParseContextConfig(core.getResourceLoader(), parseContextConfigLoc);+ parseContextConfig =+ new ParseContextConfig(core.getResourceLoader(), parseContextConfigLoc);}} catch (Exception e) {throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to load Tika Config", e);@@ -97,4 +98,4 @@ public class ExtractingRequestHandler extends ContentStreamHandlerBase implementpublic String getDescription() {return "Add/Update Rich document";}-}\ No newline at end of file+}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.javaindex 4fdba823bde..327fe3e606a 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/ParseContextConfig.java@@ -26,7 +26,6 @@ import java.lang.reflect.Method;import java.util.HashMap;import java.util.Locale;import java.util.Map;-import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.util.SafeXMLParsing;import org.apache.tika.parser.ParseContext;@@ -40,24 +39,29 @@ import org.w3c.dom.NodeList;public class ParseContextConfig {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-+private final Map<Class<?>, Object> entries = new HashMap<>();/** Creates an empty Config without any settings (used as placeholder). */- public ParseContextConfig() {- }+ public ParseContextConfig() {}/** Creates a {@code ParseContextConfig} from the given XML DOM element. */public ParseContextConfig(SolrResourceLoader resourceLoader, Element element) throws Exception {extract(element, resourceLoader);}- /** Creates a {@code ParseContextConfig} from the given XML file, loaded from the given {@link SolrResourceLoader}. */- public ParseContextConfig(SolrResourceLoader resourceLoader, String parseContextConfigLoc) throws Exception {- this(resourceLoader, loadConfigFile(resourceLoader, parseContextConfigLoc).getDocumentElement());+ /**+ * Creates a {@code ParseContextConfig} from the given XML file, loaded from the given {@link+ * SolrResourceLoader}.+ */+ public ParseContextConfig(SolrResourceLoader resourceLoader, String parseContextConfigLoc)+ throws Exception {+ this(+ resourceLoader, loadConfigFile(resourceLoader, parseContextConfigLoc).getDocumentElement());}-- private static Document loadConfigFile(SolrResourceLoader resourceLoader, String parseContextConfigLoc) throws Exception {++ private static Document loadConfigFile(+ SolrResourceLoader resourceLoader, String parseContextConfigLoc) throws Exception {return SafeXMLParsing.parseConfigXML(log, resourceLoader, parseContextConfigLoc);}@@ -68,11 +72,13 @@ public class ParseContextConfig {final String className = xmlEntryAttributes.getNamedItem("class").getNodeValue();final String implementationName = xmlEntryAttributes.getNamedItem("impl").getNodeValue();- final NodeList xmlProperties = ((Element)xmlEntries.item(i)).getElementsByTagName("property");+ final NodeList xmlProperties =+ ((Element) xmlEntries.item(i)).getElementsByTagName("property");final Class<?> interfaceClass = loader.findClass(className, Object.class);- final BeanInfo beanInfo = Introspector.getBeanInfo(interfaceClass, Introspector.IGNORE_ALL_BEANINFO);-+ final BeanInfo beanInfo =+ Introspector.getBeanInfo(interfaceClass, Introspector.IGNORE_ALL_BEANINFO);+final HashMap<String, PropertyDescriptor> descriptorMap = new HashMap<>();for (final PropertyDescriptor pd : beanInfo.getPropertyDescriptors()) {descriptorMap.put(pd.getName(), pd);@@ -80,7 +86,8 @@ public class ParseContextConfig {final Object instance = loader.newInstance(implementationName, Object.class);if (!interfaceClass.isInstance(instance)) {- throw new IllegalArgumentException("Implementation class does not extend " + interfaceClass.getName());+ throw new IllegalArgumentException(+ "Implementation class does not extend " + interfaceClass.getName());}for (int j = 0, c2 = xmlProperties.getLength(); j < c2; j++) {@@ -92,15 +99,24 @@ public class ParseContextConfig {final PropertyDescriptor propertyDescriptor = descriptorMap.get(propertyName);if (propertyDescriptor == null) {- throw new IllegalArgumentException(String.format(Locale.ENGLISH, "Unknown bean property %s in class %s",- propertyName, interfaceClass.getName()));+ throw new IllegalArgumentException(+ String.format(+ Locale.ENGLISH,+ "Unknown bean property %s in class %s",+ propertyName,+ interfaceClass.getName()));}final Method method = propertyDescriptor.getWriteMethod();if (method == null) {- throw new IllegalArgumentException(String.format(Locale.ENGLISH, "Cannot set bean property %s in class %s (no write method available)",- propertyName, interfaceClass.getName()));+ throw new IllegalArgumentException(+ String.format(+ Locale.ENGLISH,+ "Cannot set bean property %s in class %s (no write method available)",+ propertyName,+ interfaceClass.getName()));}- method.invoke(instance, getValueFromString(propertyDescriptor.getPropertyType(), propertyValue));+ method.invoke(+ instance, getValueFromString(propertyDescriptor.getPropertyType(), propertyValue));}entries.put(interfaceClass, instance);@@ -120,7 +136,7 @@ public class ParseContextConfig {public ParseContext create() {final ParseContext result = new ParseContext();- for (Map.Entry<Class<?>, Object> entry : entries.entrySet()){+ for (Map.Entry<Class<?>, Object> entry : entries.entrySet()) {result.set((Class) entry.getKey(), entry.getValue());}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.javaindex 5ef5d3f1f3a..b1f1bc5038e 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java@@ -25,7 +25,6 @@ import java.util.LinkedHashMap;import java.util.Map.Entry;import java.util.regex.Pattern;import java.util.regex.PatternSyntaxException;-import org.apache.lucene.util.IOUtils;import org.apache.tika.metadata.Metadata;import org.apache.tika.metadata.TikaMetadataKeys;@@ -34,69 +33,67 @@ import org.slf4j.Logger;import org.slf4j.LoggerFactory;/**- * Password provider for Extracting request handler which finds correct- * password based on file name matching against a list of regular expressions.- * The list of passwords is supplied in an optional Map.- * If an explicit password is set, it will be used.+ * Password provider for Extracting request handler which finds correct password based on file name+ * matching against a list of regular expressions. The list of passwords is supplied in an optional+ * Map. If an explicit password is set, it will be used.*/public class RegexRulesPasswordProvider implements PasswordProvider {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-- private LinkedHashMap<Pattern,String> passwordMap = new LinkedHashMap<>();- private String explicitPassword;-++ private LinkedHashMap<Pattern, String> passwordMap = new LinkedHashMap<>();+ private String explicitPassword;+@Overridepublic String getPassword(Metadata meta) {- if(getExplicitPassword() != null) {+ if (getExplicitPassword() != null) {return getExplicitPassword();}-- if(passwordMap.size() > 0)++ if (passwordMap.size() > 0)return lookupPasswordFromMap(meta.get(TikaMetadataKeys.RESOURCE_NAME_KEY));-+return null;}private String lookupPasswordFromMap(String fileName) {- if(fileName != null && fileName.length() > 0) {- for(Entry<Pattern,String> e : passwordMap.entrySet()) {- if(e.getKey().matcher(fileName).matches()) {+ if (fileName != null && fileName.length() > 0) {+ for (Entry<Pattern, String> e : passwordMap.entrySet()) {+ if (e.getKey().matcher(fileName).matches()) {return e.getValue();}}}return null;}-+/*** Parses rule file from stream and returns a Map of all rules found+ ** @param is input stream for the file*/- public static LinkedHashMap<Pattern,String> parseRulesFile(InputStream is) {- LinkedHashMap<Pattern,String> rules = new LinkedHashMap<>();+ public static LinkedHashMap<Pattern, String> parseRulesFile(InputStream is) {+ LinkedHashMap<Pattern, String> rules = new LinkedHashMap<>();BufferedReader br = new BufferedReader(IOUtils.getDecodingReader(is, StandardCharsets.UTF_8));String line;try {int linenum = 0;- while ((line = br.readLine()) != null) {+ while ((line = br.readLine()) != null) {linenum++;// Remove commentsString[] arr = line.split("#");- if(arr.length > 0)- line = arr[0].trim();- if(line.length() == 0)- continue;+ if (arr.length > 0) line = arr[0].trim();+ if (line.length() == 0) continue;int sep = line.indexOf("=");- if(sep <= 0) {+ if (sep <= 0) {log.warn("Wrong format of password line {}", linenum);continue;}- String pass = line.substring(sep+1).trim();+ String pass = line.substring(sep + 1).trim();String regex = line.substring(0, sep).trim();try {Pattern pattern = Pattern.compile(regex);- rules.put(pattern, pass);- } catch(PatternSyntaxException pse) {+ rules.put(pattern, pass);+ } catch (PatternSyntaxException pse) {log.warn("Key of line {} was not a valid regex pattern{}", linenum, pse);continue;}@@ -111,22 +108,24 @@ public class RegexRulesPasswordProvider implements PasswordProvider {/*** Initialize rules through file input stream. This is a convenience for first calling* setPasswordMap(parseRulesFile(is)).+ ** @param is the input stream with rules file, one line per rule on format regex=password*/public void parse(InputStream is) {setPasswordMap(parseRulesFile(is));}-- public LinkedHashMap<Pattern,String> getPasswordMap() {++ public LinkedHashMap<Pattern, String> getPasswordMap() {return passwordMap;}- public void setPasswordMap(LinkedHashMap<Pattern,String> linkedHashMap) {+ public void setPasswordMap(LinkedHashMap<Pattern, String> linkedHashMap) {this.passwordMap = linkedHashMap;}/*** Gets the explicit password, if set+ ** @return the password, or null if not set*/public String getExplicitPassword() {@@ -135,17 +134,15 @@ public class RegexRulesPasswordProvider implements PasswordProvider {/*** Sets an explicit password which will be used instead of password map+ ** @param explicitPassword the password to use*/public void setExplicitPassword(String explicitPassword) {this.explicitPassword = explicitPassword;}-- /**- * Resets explicit password, so that map will be used for lookups- */++ /** Resets explicit password, so that map will be used for lookups */public void resetExplicitPassword() {this.explicitPassword = null;}-}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.javaindex 8d871a432e5..43d0de9f7d5 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandler.java@@ -25,7 +25,6 @@ import java.util.HashSet;import java.util.Iterator;import java.util.Map;import java.util.Set;-import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.params.SolrParams;import org.apache.solr.schema.IndexSchema;@@ -38,14 +37,13 @@ import org.xml.sax.Attributes;import org.xml.sax.SAXException;import org.xml.sax.helpers.DefaultHandler;-/**- * The class responsible for handling Tika events and translating them into {@link org.apache.solr.common.SolrInputDocument}s.- * <B>This class is not thread-safe.</B>- * <p>- * This class cannot be reused, you have to create a new instance per document!- * <p>- * User's may wish to override this class to provide their own functionality.+ * The class responsible for handling Tika events and translating them into {@link+ * org.apache.solr.common.SolrInputDocument}s. <B>This class is not thread-safe.</B>+ *+ * <p>This class cannot be reused, you have to create a new instance per document!+ *+ * <p>User's may wish to override this class to provide their own functionality.** @see org.apache.solr.handler.extraction.SolrContentHandlerFactory* @see org.apache.solr.handler.extraction.ExtractingRequestHandler@@ -67,14 +65,13 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParaprotected final boolean captureAttribs;protected final boolean lowerNames;-+protected final String unknownFieldPrefix;protected final String defaultField;private final boolean literalsOverride;-- private Set<String> literalFieldNames = null;+ private Set<String> literalFieldNames = null;public SolrContentHandler(Metadata metadata, SolrParams params, IndexSchema schema) {this.document = new SolrInputDocument();@@ -87,7 +84,7 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParathis.literalsOverride = params.getBool(LITERALS_OVERRIDE, true);this.unknownFieldPrefix = params.get(UNKNOWN_FIELD_PREFIX, "");this.defaultField = params.get(DEFAULT_FIELD, "");-+String[] captureFields = params.getParams(CAPTURE_ELEMENTS);if (captureFields != null && captureFields.length > 0) {fieldBuilders = new HashMap<>();@@ -100,30 +97,29 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParabldrStack.add(catchAllBuilder);}-/**- * This is called by a consumer when it is ready to deal with a new SolrInputDocument. Overriding- * classes can use this hook to add in or change whatever they deem fit for the document at that time.- * The base implementation adds the metadata as fields, allowing for potential remapping.+ * This is called by a consumer when it is ready to deal with a new SolrInputDocument. Overriding+ * classes can use this hook to add in or change whatever they deem fit for the document at that+ * time. The base implementation adds the metadata as fields, allowing for potential remapping.** @return The {@link org.apache.solr.common.SolrInputDocument}.- ** @see #addMetadata()* @see #addCapturedContent()* @see #addContent()* @see #addLiterals()*/public SolrInputDocument newDocument() {- //handle the literals from the params. NOTE: This MUST be called before the others in order for literals to override other values+ // handle the literals from the params. NOTE: This MUST be called before the others in order for+ // literals to override other valuesaddLiterals();- //handle the metadata extracted from the document+ // handle the metadata extracted from the documentaddMetadata();- //add in the content+ // add in the contentaddContent();- //add in the captured content+ // add in the captured contentaddCapturedContent();if (log.isDebugEnabled()) {@@ -133,26 +129,25 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara}/**- * Add the per field captured content to the Solr Document. Default implementation uses the- * {@link #fieldBuilders} info+ * Add the per field captured content to the Solr Document. Default implementation uses the {@link+ * #fieldBuilders} info*/protected void addCapturedContent() {for (Map.Entry<String, StringBuilder> entry : fieldBuilders.entrySet()) {if (entry.getValue().length() > 0) {String fieldName = entry.getKey();- if (literalsOverride && literalFieldNames.contains(fieldName))- continue;- addField(fieldName, entry.getValue().toString(), null); }+ if (literalsOverride && literalFieldNames.contains(fieldName)) continue;+ addField(fieldName, entry.getValue().toString(), null);+ }}}/**- * Add in the catch all content to the field. Default impl. uses the {@link #contentFieldName}- * and the {@link #catchAllBuilder}+ * Add in the catch all content to the field. Default impl. uses the {@link #contentFieldName} and+ * the {@link #catchAllBuilder}*/protected void addContent() {- if (literalsOverride && literalFieldNames.contains(contentFieldName))- return;+ if (literalsOverride && literalFieldNames.contains(contentFieldName)) return;addField(contentFieldName, catchAllBuilder.toString(), null);}@@ -172,13 +167,10 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara}}- /**- * Add in any metadata using {@link #metadata} as the source.- */+ /** Add in any metadata using {@link #metadata} as the source. */protected void addMetadata() {for (String name : metadata.names()) {- if (literalsOverride && literalFieldNames.contains(name))- continue;+ if (literalsOverride && literalFieldNames.contains(name)) continue;String[] vals = metadata.getValues(name);addField(name, null, vals);}@@ -191,21 +183,24 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParaprotected void addField(String fname, String fval, String[] vals) {if (lowerNames) {StringBuilder sb = new StringBuilder();- for (int i=0; i<fname.length(); i++) {+ for (int i = 0; i < fname.length(); i++) {char ch = fname.charAt(i);- if (!Character.isLetterOrDigit(ch)) ch='_';- else ch=Character.toLowerCase(ch);+ if (!Character.isLetterOrDigit(ch)) ch = '_';+ else ch = Character.toLowerCase(ch);sb.append(ch);}fname = sb.toString();- }+ }String name = findMappedName(fname);SchemaField sf = schema.getFieldOrNull(name);- if (sf==null && unknownFieldPrefix.length() > 0) {+ if (sf == null && unknownFieldPrefix.length() > 0) {name = unknownFieldPrefix + name;sf = schema.getFieldOrNull(name);- } else if (sf == null && defaultField.length() > 0 && name.equals(TikaMetadataKeys.RESOURCE_NAME_KEY) == false /*let the fall through below handle this*/){+ } else if (sf == null+ && defaultField.length() > 0+ && name.equals(TikaMetadataKeys.RESOURCE_NAME_KEY)+ == false /*let the fall through below handle this*/) {name = defaultField;sf = schema.getFieldOrNull(name);}@@ -215,12 +210,14 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara// ExtractingDocumentLoader.load(). You shouldn't have to define a mapping for this// field just because you specified a resource.name parameter to the handler, should// you?- if (sf == null && unknownFieldPrefix.length()==0 && name == TikaMetadataKeys.RESOURCE_NAME_KEY) {+ if (sf == null+ && unknownFieldPrefix.length() == 0+ && name == TikaMetadataKeys.RESOURCE_NAME_KEY) {return;}// normalize val params so vals.length>1- if (vals != null && vals.length==1) {+ if (vals != null && vals.length == 1) {fval = vals[0];vals = null;}@@ -228,17 +225,17 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara// single valued field with multiple values... catenate them.if (sf != null && !sf.multiValued() && vals != null) {StringBuilder builder = new StringBuilder();- boolean first=true;+ boolean first = true;for (String val : vals) {if (first) {- first=false;+ first = false;} else {builder.append(' ');}builder.append(val);}fval = builder.toString();- vals=null;+ vals = null;}if (fval != null) {@@ -256,10 +253,11 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingPara}@Override- public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {+ public void startElement(String uri, String localName, String qName, Attributes attributes)+ throws SAXException {StringBuilder theBldr = fieldBuilders.get(localName);if (theBldr != null) {- //we need to switch the currentBuilder+ // we need to switch the currentBuilderbldrStack.add(theBldr);}if (captureAttribs == true) {@@ -278,22 +276,19 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParapublic void endElement(String uri, String localName, String qName) throws SAXException {StringBuilder theBldr = fieldBuilders.get(localName);if (theBldr != null) {- //pop the stack+ // pop the stackbldrStack.removeLast();assert (bldrStack.size() >= 1);}bldrStack.getLast().append(' ');}-@Overridepublic void characters(char[] chars, int offset, int length) throws SAXException {bldrStack.getLast().append(chars, offset, length);}- /**- * Treat the same as any other characters- */+ /** Treat the same as any other characters */@Overridepublic void ignorableWhitespace(char[] chars, int offset, int length) throws SAXException {characters(chars, offset, length);@@ -308,5 +303,4 @@ public class SolrContentHandler extends DefaultHandler implements ExtractingParaprotected String findMappedName(String name) {return params.get(MAP_PREFIX + name, name);}-}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.javaindex f95125e2aee..1070e744d84 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/SolrContentHandlerFactory.java@@ -16,20 +16,17 @@*/package org.apache.solr.handler.extraction;-import org.apache.tika.metadata.Metadata;import org.apache.solr.common.params.SolrParams;import org.apache.solr.schema.IndexSchema;+import org.apache.tika.metadata.Metadata;--/**- *- *- **/+/** */public class SolrContentHandlerFactory {- public SolrContentHandlerFactory() { }+ public SolrContentHandlerFactory() {}- public SolrContentHandler createSolrContentHandler(Metadata metadata, SolrParams params, IndexSchema schema) {+ public SolrContentHandler createSolrContentHandler(+ Metadata metadata, SolrParams params, IndexSchema schema) {return new SolrContentHandler(metadata, params, schema);}}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.javaindex d1e5dda4dac..21dee630788 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java@@ -29,7 +29,6 @@ import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;-import org.apache.lucene.index.IndexableField;import org.apache.poi.ss.usermodel.Cell;import org.apache.poi.ss.usermodel.FillPatternType;@@ -52,13 +51,14 @@ import org.apache.solr.search.ReturnFields;public class XLSXResponseWriter extends RawResponseWriter {@Override- public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {+ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp)+ throws IOException {// throw away arraywriter just to satisfy super requirements; we're grabbing// all writes before they go to it anywayXLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp);- LinkedHashMap<String,String> reqNamesMap = new LinkedHashMap<>();- LinkedHashMap<String,Integer> reqWidthsMap = new LinkedHashMap<>();+ LinkedHashMap<String, String> reqNamesMap = new LinkedHashMap<>();+ LinkedHashMap<String, Integer> reqWidthsMap = new LinkedHashMap<>();Iterator<String> paramNamesIter = req.getParams().getParameterNamesIterator();while (paramNamesIter.hasNext()) {@@ -102,12 +102,12 @@ class XLSXWriter extends TabularResponseWriter {this.rowIndex = 0;- this.headerStyle = (XSSFCellStyle)swb.createCellStyle();+ this.headerStyle = (XSSFCellStyle) swb.createCellStyle();this.headerStyle.setFillBackgroundColor(IndexedColors.BLACK.getIndex());- //solid fill+ // solid fillthis.headerStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);Font headerFont = swb.createFont();- headerFont.setFontHeightInPoints((short)14);+ headerFont.setFontHeightInPoints((short) 14);headerFont.setBold(true);headerFont.setColor(IndexedColors.WHITE.getIndex());this.headerStyle.setFont(headerFont);@@ -119,18 +119,18 @@ class XLSXWriter extends TabularResponseWriter {}void setHeaderRow() {- curRow.setHeightInPoints((short)21);+ curRow.setHeightInPoints((short) 21);}- //sets last created cell to have header style+ // sets last created cell to have header stylevoid setHeaderCell() {curRow.getCell(cellIndex - 1).setCellStyle(this.headerStyle);}- //set the width of the most recently created column+ // set the width of the most recently created columnvoid setColWidth(int charWidth) {- //width in poi is units of 1/256th of a character width for some reason- this.sh.setColumnWidth(cellIndex - 1, 256*charWidth);+ // width in poi is units of 1/256th of a character width for some reason+ this.sh.setColumnWidth(cellIndex - 1, 256 * charWidth);}void writeCell(String value) {@@ -145,7 +145,7 @@ class XLSXWriter extends TabularResponseWriter {StringWriter sw = new StringWriter();e.printStackTrace(new PrintWriter(sw));String stacktrace = sw.toString();- }finally {+ } finally {swb.dispose();}}@@ -158,14 +158,17 @@ class XLSXWriter extends TabularResponseWriter {SchemaField sf;}- private Map<String,XLField> xlFields = new LinkedHashMap<String,XLField>();+ private Map<String, XLField> xlFields = new LinkedHashMap<String, XLField>();public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {super(writer, req, rsp);}- public void writeResponse(OutputStream out, LinkedHashMap<String, String> colNamesMap,- LinkedHashMap<String, Integer> colWidthsMap) throws IOException {+ public void writeResponse(+ OutputStream out,+ LinkedHashMap<String, String> colNamesMap,+ LinkedHashMap<String, Integer> colWidthsMap)+ throws IOException {Collection<String> fields = getFields();for (String field : fields) {@@ -194,10 +197,8 @@ class XLSXWriter extends TabularResponseWriter {xlFields.put(field, xlField);}--wb.addRow();- //write header+ // write headerfor (XLField xlField : xlFields.values()) {String printName = xlField.name;int colWidth = 14;@@ -230,11 +231,12 @@ class XLSXWriter extends TabularResponseWriter {super.close();}- //NOTE: a document cannot currently contain another document+ // NOTE: a document cannot currently contain another documentList<Object> tmpList;@Override- public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx)+ throws IOException {if (tmpList == null) {tmpList = new ArrayList<>(1);tmpList.add(null);@@ -242,7 +244,7 @@ class XLSXWriter extends TabularResponseWriter {for (XLField xlField : xlFields.values()) {Object val = doc.getFieldValue(xlField.name);- int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1);+ int nVals = val instanceof Collection ? ((Collection) val).size() : (val == null ? 0 : 1);if (nVals == 0) {writeNull(xlField.name);continue;@@ -252,7 +254,7 @@ class XLSXWriter extends TabularResponseWriter {Collection<?> values;// normalize to a collectionif (val instanceof Collection) {- values = (Collection<?>)val;+ values = (Collection<?>) val;} else {tmpList.set(0, val);values = tmpList;@@ -263,7 +265,7 @@ class XLSXWriter extends TabularResponseWriter {} else {// normalize to first valueif (val instanceof Collection) {- Collection<?> values = (Collection<?>)val;+ Collection<?> values = (Collection<?>) val;val = values.iterator().next();}writeVal(xlField.name, val);@@ -284,7 +286,7 @@ class XLSXWriter extends TabularResponseWriter {while (val.hasNext()) {Object v = val.next();if (v instanceof IndexableField) {- IndexableField f = (IndexableField)v;+ IndexableField f = (IndexableField) v;if (v instanceof Date) {output.append(((Date) val).toInstant().toString()).append("; ");} else {@@ -295,8 +297,8 @@ class XLSXWriter extends TabularResponseWriter {}}if (output.length() > 0) {- output.deleteCharAt(output.length()-1);- output.deleteCharAt(output.length()-1);+ output.deleteCharAt(output.length() - 1);+ output.deleteCharAt(output.length() - 1);}writeStr(name, output.toString(), false);}@@ -335,4 +337,4 @@ class XLSXWriter extends TabularResponseWriter {public void writeDate(String name, String val) throws IOException {wb.writeCell(val);}-}\ No newline at end of file+}diff --git a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/package-info.java b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/package-info.javaindex 729a2767964..48d84cbcb28 100644--- a/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/package-info.java+++ b/solr/modules/extraction/src/java/org/apache/solr/handler/extraction/package-info.java@@ -14,12 +14,6 @@* See the License for the specific language governing permissions and* limitations under the License.*/--/**- * {@link org.apache.solr.handler.extraction.ExtractingRequestHandler} and related code.- */-package org.apache.solr.handler.extraction;---+/** {@link org.apache.solr.handler.extraction.ExtractingRequestHandler} and related code. */+package org.apache.solr.handler.extraction;diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.javaindex 91ebc8234b6..7b7f924b79a 100644--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java@@ -20,7 +20,6 @@ import java.util.ArrayList;import java.util.List;import java.util.Locale;import java.util.TimeZone;-import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.SolrException;import org.apache.solr.common.util.ContentStream;@@ -35,21 +34,21 @@ import org.junit.Before;import org.junit.BeforeClass;import org.junit.Test;--/**- *- *- **/+/** */public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@BeforeClasspublic static void beforeClass() throws Exception {// Is the JDK/env affected by a known bug?- final String tzDisplayName = TimeZone.getDefault().getDisplayName(false, TimeZone.SHORT, Locale.US);+ final String tzDisplayName =+ TimeZone.getDefault().getDisplayName(false, TimeZone.SHORT, Locale.US);if (!tzDisplayName.matches("[A-Za-z]{3,}([+-]\\d\\d(:\\d\\d)?)?")) {- assertTrue("Is some other JVM affected? Or bad regex? TzDisplayName: " + tzDisplayName,+ assertTrue(+ "Is some other JVM affected? Or bad regex? TzDisplayName: " + tzDisplayName,System.getProperty("java.version").startsWith("11"));- assumeTrue("SOLR-12759 JDK 11 (1st release) and Tika 1.x can result in extracting dates in a bad format.", false);+ assumeTrue(+ "SOLR-12759 JDK 11 (1st release) and Tika 1.x can result in extracting dates in a bad format.",+ false);}initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath());@@ -65,159 +64,265 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testExtraction() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- loadLocal("extraction/solr-word.pdf",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "extractedContent",- "literal.id", "one",- "fmap.Last-Modified", "extractedDate"- );+ loadLocal(+ "extraction/solr-word.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "extractedContent",+ "literal.id",+ "one",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("title:solr-word"), "//*[@numFound='0']");assertU(commit());assertQ(req("title:solr-word"), "//*[@numFound='1']");-- loadLocal("extraction/simple.html", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "fmap.language", "extractedLanguage",- "literal.id", "two",- "uprefix", "ignored_",- "fmap.content", "extractedContent",- "fmap.Last-Modified", "extractedDate"- );+ loadLocal(+ "extraction/simple.html",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "fmap.language",+ "extractedLanguage",+ "literal.id",+ "two",+ "uprefix",+ "ignored_",+ "fmap.content",+ "extractedContent",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("title:Welcome"), "//*[@numFound='0']");assertU(commit());assertQ(req("title:Welcome"), "//*[@numFound='1']");- assertQ(req("extractedContent:distinctwords"), "//*[@numFound='0']");- assertQ(req("extractedContent:distinct"), "//*[@numFound='1']");- assertQ(req("extractedContent:words"), "//*[@numFound='2']");+ assertQ(req("extractedContent:distinctwords"), "//*[@numFound='0']");+ assertQ(req("extractedContent:distinct"), "//*[@numFound='1']");+ assertQ(req("extractedContent:words"), "//*[@numFound='2']");assertQ(req("extractedContent:\"distinct words\""), "//*[@numFound='1']");- loadLocal("extraction/simple.html",- "literal.id","simple2",- "uprefix", "t_",- "lowernames", "true",- "captureAttr", "true",- "fmap.a","t_href",- "fmap.content_type", "abcxyz", // test that lowernames is applied before mapping, and uprefix is applied after mapping- "commit", "true" // test immediate commit- );+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "simple2",+ "uprefix",+ "t_",+ "lowernames",+ "true",+ "captureAttr",+ "true",+ "fmap.a",+ "t_href",+ "fmap.content_type",+ "abcxyz", // test that lowernames is applied before mapping, and uprefix is applied after+ // mapping+ "commit",+ "true" // test immediate commit+ );// test that purposely causes a failure to print out the doc for test debugging// assertQ(req("q","id:simple2","indent","true"), "//*[@numFound='0']");// test both lowernames and unknown field mapping- //assertQ(req("+id:simple2 +t_content_type:[* TO *]"), "//*[@numFound='1']");+ // assertQ(req("+id:simple2 +t_content_type:[* TO *]"), "//*[@numFound='1']");assertQ(req("+id:simple2 +t_href:[* TO *]"), "//*[@numFound='1']");assertQ(req("+id:simple2 +t_abcxyz:[* TO *]"), "//*[@numFound='1']");- assertQ(req("+id:simple2 +t_content:serif"), "//*[@numFound='0']"); // make sure <style> content is excluded- assertQ(req("+id:simple2 +t_content:blur"), "//*[@numFound='0']"); // make sure <script> content is excluded+ assertQ(+ req("+id:simple2 +t_content:serif"),+ "//*[@numFound='0']"); // make sure <style> content is excluded+ assertQ(+ req("+id:simple2 +t_content:blur"),+ "//*[@numFound='0']"); // make sure <script> content is excluded// make sure the fact there is an index-time boost does not fail the parsing- loadLocal("extraction/simple.html",- "literal.id","simple3",- "uprefix", "t_",- "lowernames", "true",- "captureAttr", "true", "fmap.a","t_href",- "commit", "true"-- ,"boost.t_href", "100.0"- );+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "simple3",+ "uprefix",+ "t_",+ "lowernames",+ "true",+ "captureAttr",+ "true",+ "fmap.a",+ "t_href",+ "commit",+ "true",+ "boost.t_href",+ "100.0");assertQ(req("t_href:http"), "//*[@numFound='2']");assertQ(req("t_href:http"), "//doc[2]/str[.='simple3']");- assertQ(req("+id:simple3 +t_content_type:[* TO *]"), "//*[@numFound='1']");//test lowercase and then uprefix-- loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "three",- "uprefix", "ignored_",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Last-Modified", "extractedDate"- );+ assertQ(+ req("+id:simple3 +t_content_type:[* TO *]"),+ "//*[@numFound='1']"); // test lowercase and then uprefix++ loadLocal(+ "extraction/version_control.xml",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "three",+ "uprefix",+ "ignored_",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("stream_name:version_control.xml"), "//*[@numFound='0']");assertU(commit());assertQ(req("stream_name:version_control.xml"), "//*[@numFound='1']");- loadLocal("extraction/word2003.doc", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "four",- "uprefix", "ignored_",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Last-Modified", "extractedDate"- );+ loadLocal(+ "extraction/word2003.doc",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "four",+ "uprefix",+ "ignored_",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("title:\"Word 2003 Title\""), "//*[@numFound='0']");// There is already a PDF file with this content:- assertQ(req("extractedContent:\"This is a test of PDF and Word extraction in Solr, it is only a test\""), "//*[@numFound='1']");+ assertQ(+ req(+ "extractedContent:\"This is a test of PDF and Word extraction in Solr, it is only a test\""),+ "//*[@numFound='1']");assertU(commit());assertQ(req("title:\"Word 2003 Title\""), "//*[@numFound='1']");// now 2 of them:- assertQ(req("extractedContent:\"This is a test of PDF and Word extraction in Solr, it is only a test\""), "//*[@numFound='2']");+ assertQ(+ req(+ "extractedContent:\"This is a test of PDF and Word extraction in Solr, it is only a test\""),+ "//*[@numFound='2']");// compressed file- loadLocal("extraction/tiny.txt.gz",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "uprefix", "ignored_",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Last-Modified", "extractedDate",- "literal.id", "tiny.txt.gz");+ loadLocal(+ "extraction/tiny.txt.gz",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "uprefix",+ "ignored_",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Last-Modified",+ "extractedDate",+ "literal.id",+ "tiny.txt.gz");assertU(commit());- assertQ(req("id:tiny.txt.gz")- , "//*[@numFound='1']"- , "//*/arr[@name='stream_name']/str[.='tiny.txt.gz']"- );+ assertQ(+ req("id:tiny.txt.gz"),+ "//*[@numFound='1']",+ "//*/arr[@name='stream_name']/str[.='tiny.txt.gz']");// compressed file- loadLocal("extraction/open-document.odt",- "uprefix", "ignored_",- "fmap.content", "extractedContent",- "literal.id", "open-document");+ loadLocal(+ "extraction/open-document.odt",+ "uprefix",+ "ignored_",+ "fmap.content",+ "extractedContent",+ "literal.id",+ "open-document");assertU(commit());- assertQ(req("extractedContent:\"Práctica sobre GnuPG\"")- , "//*[@numFound='1']"- , "//*/arr[@name='stream_name']/str[.='open-document.odt']"- );+ assertQ(+ req("extractedContent:\"Práctica sobre GnuPG\""),+ "//*[@numFound='1']",+ "//*/arr[@name='stream_name']/str[.='open-document.odt']");}@Testpublic void testCapture() throws Exception {- loadLocal("extraction/simple.html",- "literal.id","capture1",- "uprefix","t_",- "capture","div",- "fmap.div", "foo_t",- "commit", "true"- );+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "capture1",+ "uprefix",+ "t_",+ "capture",+ "div",+ "fmap.div",+ "foo_t",+ "commit",+ "true");assertQ(req("+id:capture1 +t_content:Solr"), "//*[@numFound='1']");assertQ(req("+id:capture1 +foo_t:\"here is some text in a div\""), "//*[@numFound='1']");- loadLocal("extraction/simple.html",- "literal.id", "capture2",- "captureAttr", "true",- "defaultField", "text",- "fmap.div", "div_t",- "fmap.a", "anchor_t",- "capture", "div",- "capture", "a",- "commit", "true"- );+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "capture2",+ "captureAttr",+ "true",+ "defaultField",+ "text",+ "fmap.div",+ "div_t",+ "fmap.a",+ "anchor_t",+ "capture",+ "div",+ "capture",+ "a",+ "commit",+ "true");assertQ(req("+id:capture2 +text:Solr"), "//*[@numFound='1']");assertQ(req("+id:capture2 +div_t:\"here is some text in a div\""), "//*[@numFound='1']");assertQ(req("+id:capture2 +anchor_t:http\\://www.apache.org"), "//*[@numFound='1']");@@ -226,68 +331,106 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testDefaultField() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertNotNull("handler is null and it shouldn't be", handler);try {ignoreException("unknown field 'a'");- ignoreException("unknown field 'meta'"); // TODO: should this exception be happening?- expectThrows(SolrException.class, () -> {- loadLocal("extraction/simple.html",- "literal.id", "simple2",- "lowernames", "true",- "captureAttr", "true",- //"fmap.content_type", "abcxyz",- "commit", "true" // test immediate commit- );- });+ ignoreException("unknown field 'meta'"); // TODO: should this exception be happening?+ expectThrows(+ SolrException.class,+ () -> {+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "simple2",+ "lowernames",+ "true",+ "captureAttr",+ "true",+ // "fmap.content_type", "abcxyz",+ "commit",+ "true" // test immediate commit+ );+ });} finally {resetExceptionIgnores();}--- loadLocal("extraction/simple.html",- "literal.id","simple2",- ExtractingParams.DEFAULT_FIELD, "defaultExtr",//test that unmapped fields go to the text field when no uprefix is specified- "lowernames", "true",- "captureAttr", "true",- //"fmap.content_type", "abcxyz",- "commit", "true" // test immediate commit- );++ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "simple2",+ ExtractingParams.DEFAULT_FIELD,+ "defaultExtr", // test that unmapped fields go to the text field when no uprefix is+ // specified+ "lowernames",+ "true",+ "captureAttr",+ "true",+ // "fmap.content_type", "abcxyz",+ "commit",+ "true" // test immediate commit+ );assertQ(req("id:simple2"), "//*[@numFound='1']");assertQ(req("defaultExtr:http\\:\\/\\/www.apache.org"), "//*[@numFound='1']");- //Test when both uprefix and default are specified.- loadLocal("extraction/simple.html",- "literal.id","simple2",- ExtractingParams.DEFAULT_FIELD, "defaultExtr",//test that unmapped fields go to the text field when no uprefix is specified- ExtractingParams.UNKNOWN_FIELD_PREFIX, "t_",- "lowernames", "true",- "captureAttr", "true",- "fmap.a","t_href",- //"fmap.content_type", "abcxyz",- "commit", "true" // test immediate commit- );+ // Test when both uprefix and default are specified.+ loadLocal(+ "extraction/simple.html",+ "literal.id",+ "simple2",+ ExtractingParams.DEFAULT_FIELD,+ "defaultExtr", // test that unmapped fields go to the text field when no uprefix is+ // specified+ ExtractingParams.UNKNOWN_FIELD_PREFIX,+ "t_",+ "lowernames",+ "true",+ "captureAttr",+ "true",+ "fmap.a",+ "t_href",+ // "fmap.content_type", "abcxyz",+ "commit",+ "true" // test immediate commit+ );assertQ(req("+id:simple2 +t_href:[* TO *]"), "//*[@numFound='1']");}@Testpublic void testLiterals() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- //test literal- loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "fmap.content", "extractedContent",- "literal.id", "one",- "uprefix", "ignored_",- "fmap.language", "extractedLanguage",- "literal.extractionLiteralMV", "one",- "literal.extractionLiteralMV", "two",- "fmap.Last-Modified", "extractedDate"-- );+ // test literal+ loadLocal(+ "extraction/version_control.xml",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "extractedContent",+ "literal.id",+ "one",+ "uprefix",+ "ignored_",+ "fmap.language",+ "extractedLanguage",+ "literal.extractionLiteralMV",+ "one",+ "literal.extractionLiteralMV",+ "two",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("stream_name:version_control.xml"), "//*[@numFound='0']");assertU(commit());assertQ(req("stream_name:version_control.xml"), "//*[@numFound='1']");@@ -297,121 +440,170 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {try {// TODO: original author did not specify why an exception should be thrown... how to fix?- loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "fmap.content", "extractedContent",- "literal.id", "two",- "fmap.language", "extractedLanguage",- "literal.extractionLiteral", "one",- "literal.extractionLiteral", "two",- "fmap.X-Parsed-By", "ignored_parser",- "fmap.Last-Modified", "extractedDate"- );+ loadLocal(+ "extraction/version_control.xml",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "extractedContent",+ "literal.id",+ "two",+ "fmap.language",+ "extractedLanguage",+ "literal.extractionLiteral",+ "one",+ "literal.extractionLiteral",+ "two",+ "fmap.X-Parsed-By",+ "ignored_parser",+ "fmap.Last-Modified",+ "extractedDate");// TODO: original author did not specify why an exception should be thrown... how to fix?// assertTrue("Exception should have been thrown", false);} catch (SolrException e) {- //nothing to see here, move along+ // nothing to see here, move along}- loadLocal("extraction/version_control.xml", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "fmap.content", "extractedContent",- "literal.id", "three",- "fmap.language", "extractedLanguage",- "literal.extractionLiteral", "one",- "fmap.X-Parsed-By", "ignored_parser",- "fmap.Last-Modified", "extractedDate"- );+ loadLocal(+ "extraction/version_control.xml",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "extractedContent",+ "literal.id",+ "three",+ "fmap.language",+ "extractedLanguage",+ "literal.extractionLiteral",+ "one",+ "fmap.X-Parsed-By",+ "ignored_parser",+ "fmap.Last-Modified",+ "extractedDate");assertU(commit());assertQ(req("extractionLiteral:one"), "//*[@numFound='1']");-}public void testLiteralDefaults() throws Exception {// sanity check config- loadLocalFromHandler("/update/extract/lit-def",- "extraction/simple.html",- "literal.id", "lit-def-simple");+ loadLocalFromHandler(+ "/update/extract/lit-def", "extraction/simple.html", "literal.id", "lit-def-simple");assertU(commit());- assertQ(req("q", "id:lit-def-simple")- , "//*[@numFound='1']"- , "count(//arr[@name='foo_s']/str)=1"- , "//arr[@name='foo_s']/str[.='x']"- , "count(//arr[@name='bar_s']/str)=1"- , "//arr[@name='bar_s']/str[.='y']"- , "count(//arr[@name='zot_s']/str)=1"- , "//arr[@name='zot_s']/str[.='z']"- );-+ assertQ(+ req("q", "id:lit-def-simple"),+ "//*[@numFound='1']",+ "count(//arr[@name='foo_s']/str)=1",+ "//arr[@name='foo_s']/str[.='x']",+ "count(//arr[@name='bar_s']/str)=1",+ "//arr[@name='bar_s']/str[.='y']",+ "count(//arr[@name='zot_s']/str)=1",+ "//arr[@name='zot_s']/str[.='z']");+// override the default foo_s- loadLocalFromHandler("/update/extract/lit-def",- "extraction/simple.html",- "literal.foo_s", "1111",- "literal.id", "lit-def-simple");+ loadLocalFromHandler(+ "/update/extract/lit-def",+ "extraction/simple.html",+ "literal.foo_s",+ "1111",+ "literal.id",+ "lit-def-simple");assertU(commit());- assertQ(req("q", "id:lit-def-simple")- , "//*[@numFound='1']"- , "count(//arr[@name='foo_s']/str)=1"- , "//arr[@name='foo_s']/str[.='1111']"- , "count(//arr[@name='bar_s']/str)=1"- , "//arr[@name='bar_s']/str[.='y']"- , "count(//arr[@name='zot_s']/str)=1"- , "//arr[@name='zot_s']/str[.='z']"- );+ assertQ(+ req("q", "id:lit-def-simple"),+ "//*[@numFound='1']",+ "count(//arr[@name='foo_s']/str)=1",+ "//arr[@name='foo_s']/str[.='1111']",+ "count(//arr[@name='bar_s']/str)=1",+ "//arr[@name='bar_s']/str[.='y']",+ "count(//arr[@name='zot_s']/str)=1",+ "//arr[@name='zot_s']/str[.='z']");// pre-pend the bar_s- loadLocalFromHandler("/update/extract/lit-def",- "extraction/simple.html",- "literal.bar_s", "2222",- "literal.id", "lit-def-simple");+ loadLocalFromHandler(+ "/update/extract/lit-def",+ "extraction/simple.html",+ "literal.bar_s",+ "2222",+ "literal.id",+ "lit-def-simple");assertU(commit());- assertQ(req("q", "id:lit-def-simple")- , "//*[@numFound='1']"- , "count(//arr[@name='foo_s']/str)=1"- , "//arr[@name='foo_s']/str[.='x']"- , "count(//arr[@name='bar_s']/str)=2"- , "//arr[@name='bar_s']/str[.='2222']"- , "//arr[@name='bar_s']/str[.='y']"- , "count(//arr[@name='zot_s']/str)=1"- , "//arr[@name='zot_s']/str[.='z']"- );+ assertQ(+ req("q", "id:lit-def-simple"),+ "//*[@numFound='1']",+ "count(//arr[@name='foo_s']/str)=1",+ "//arr[@name='foo_s']/str[.='x']",+ "count(//arr[@name='bar_s']/str)=2",+ "//arr[@name='bar_s']/str[.='2222']",+ "//arr[@name='bar_s']/str[.='y']",+ "count(//arr[@name='zot_s']/str)=1",+ "//arr[@name='zot_s']/str[.='z']");// invariant zot_s can not be changed- loadLocalFromHandler("/update/extract/lit-def",- "extraction/simple.html",- "literal.zot_s", "3333",- "literal.id", "lit-def-simple");+ loadLocalFromHandler(+ "/update/extract/lit-def",+ "extraction/simple.html",+ "literal.zot_s",+ "3333",+ "literal.id",+ "lit-def-simple");assertU(commit());- assertQ(req("q", "id:lit-def-simple")- , "//*[@numFound='1']"- , "count(//arr[@name='foo_s']/str)=1"- , "//arr[@name='foo_s']/str[.='x']"- , "count(//arr[@name='bar_s']/str)=1"- , "//arr[@name='bar_s']/str[.='y']"- , "count(//arr[@name='zot_s']/str)=1"- , "//arr[@name='zot_s']/str[.='z']"- );-+ assertQ(+ req("q", "id:lit-def-simple"),+ "//*[@numFound='1']",+ "count(//arr[@name='foo_s']/str)=1",+ "//arr[@name='foo_s']/str[.='x']",+ "count(//arr[@name='bar_s']/str)=1",+ "//arr[@name='bar_s']/str[.='y']",+ "count(//arr[@name='zot_s']/str)=1",+ "//arr[@name='zot_s']/str[.='z']");}@Testpublic void testPlainTextSpecifyingMimeType() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);// Load plain text specifying MIME type:- loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "one",- "fmap.language", "extractedLanguage",- "fmap.X-Parsed-By", "ignored_parser",- "fmap.content", "extractedContent",- ExtractingParams.STREAM_TYPE, "text/plain"- );+ loadLocal(+ "extraction/version_control.txt",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "one",+ "fmap.language",+ "extractedLanguage",+ "fmap.X-Parsed-By",+ "ignored_parser",+ "fmap.content",+ "extractedContent",+ ExtractingParams.STREAM_TYPE,+ "text/plain");assertQ(req("extractedContent:Apache"), "//*[@numFound='0']");assertU(commit());assertQ(req("extractedContent:Apache"), "//*[@numFound='1']");@@ -419,19 +611,33 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testPlainTextSpecifyingResourceName() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);// Load plain text specifying filename- loadLocal("extraction/version_control.txt", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "one",- "fmap.language", "extractedLanguage",- "fmap.X-Parsed-By", "ignored_parser",- "fmap.content", "extractedContent",- ExtractingParams.RESOURCE_NAME, "extraction/version_control.txt"- );+ loadLocal(+ "extraction/version_control.txt",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "one",+ "fmap.language",+ "extractedLanguage",+ "fmap.X-Parsed-By",+ "ignored_parser",+ "fmap.content",+ "extractedContent",+ ExtractingParams.RESOURCE_NAME,+ "extraction/version_control.txt");assertQ(req("extractedContent:Apache"), "//*[@numFound='0']");assertU(commit());assertQ(req("extractedContent:Apache"), "//*[@numFound='1']");@@ -439,18 +645,24 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testCommitWithin() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);-- SolrQueryRequest req = req("literal.id", "one",- ExtractingParams.RESOURCE_NAME, "extraction/version_control.txt",- "commitWithin", "200"- );++ SolrQueryRequest req =+ req(+ "literal.id",+ "one",+ ExtractingParams.RESOURCE_NAME,+ "extraction/version_control.txt",+ "commitWithin",+ "200");SolrQueryResponse rsp = new SolrQueryResponse();BufferingRequestProcessor p = new BufferingRequestProcessor(null);ExtractingDocumentLoader loader = (ExtractingDocumentLoader) handler.newLoader(req, p);- loader.load(req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")),p);+ loader.load(+ req, rsp, new ContentStreamBase.FileStream(getFile("extraction/version_control.txt")), p);AddUpdateCommand add = p.addCommands.get(0);assertEquals(200, add.commitWithin);@@ -458,20 +670,25 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {req.close();}- // Note: If you load a plain text file specifying neither MIME type nor filename, extraction will silently fail. This is because Tika's- // automatic MIME type detection will fail, and it will default to using an empty-string-returning default parser+ // Note: If you load a plain text file specifying neither MIME type nor filename, extraction will+ // silently fail. This is because Tika's+ // automatic MIME type detection will fail, and it will default to using an empty-string-returning+ // default parser@Testpublic void testExtractOnly() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- SolrQueryResponse rsp = loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true");+ SolrQueryResponse rsp =+ loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true");assertTrue("rsp is null and it shouldn't be", rsp != null);NamedList<?> list = rsp.getValues();String extraction = (String) list.get("solr-word.pdf");assertTrue("extraction is null and it shouldn't be", extraction != null);- assertTrue(extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);+ assertTrue(+ extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);NamedList<?> nl = (NamedList<?>) list.get("solr-word.pdf_metadata");assertTrue("nl is null and it shouldn't be", nl != null);@@ -479,66 +696,95 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {assertTrue("title is null and it shouldn't be", title != null);assertTrue(extraction.indexOf("<?xml") != -1);- rsp = loadLocal("extraction/solr-word.pdf", ExtractingParams.EXTRACT_ONLY, "true",- ExtractingParams.EXTRACT_FORMAT, ExtractingDocumentLoader.TEXT_FORMAT);+ rsp =+ loadLocal(+ "extraction/solr-word.pdf",+ ExtractingParams.EXTRACT_ONLY,+ "true",+ ExtractingParams.EXTRACT_FORMAT,+ ExtractingDocumentLoader.TEXT_FORMAT);assertTrue("rsp is null and it shouldn't be", rsp != null);list = rsp.getValues();extraction = (String) list.get("solr-word.pdf");assertTrue("extraction is null and it shouldn't be", extraction != null);- assertTrue(extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);+ assertTrue(+ extraction + " does not contain " + "solr-word", extraction.indexOf("solr-word") != -1);assertTrue(extraction.indexOf("<?xml") == -1);nl = (NamedList<?>) list.get("solr-word.pdf_metadata");assertTrue("nl is null and it shouldn't be", nl != null);title = nl.get("title");assertTrue("title is null and it shouldn't be", title != null);---}@Testpublic void testXPath() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- SolrQueryResponse rsp = loadLocal("extraction/example.html",- ExtractingParams.XPATH_EXPRESSION, "/xhtml:html/xhtml:body/xhtml:a/descendant::node()",- ExtractingParams.EXTRACT_ONLY, "true"- );+ SolrQueryResponse rsp =+ loadLocal(+ "extraction/example.html",+ ExtractingParams.XPATH_EXPRESSION,+ "/xhtml:html/xhtml:body/xhtml:a/descendant::node()",+ ExtractingParams.EXTRACT_ONLY,+ "true");assertTrue("rsp is null and it shouldn't be", rsp != null);NamedList<?> list = rsp.getValues();String val = (String) list.get("example.html");- assertEquals("News", val.trim()); //there is only one matching <a> tag-- loadLocal("extraction/example.html",- "literal.id", "example1",- "captureAttr", "true",- "defaultField", "text",- "capture", "div",- "fmap.div", "foo_t",- "boost.foo_t", "3",- "xpath", "/xhtml:html/xhtml:body/xhtml:div//node()",- "commit", "true"- );+ assertEquals("News", val.trim()); // there is only one matching <a> tag++ loadLocal(+ "extraction/example.html",+ "literal.id",+ "example1",+ "captureAttr",+ "true",+ "defaultField",+ "text",+ "capture",+ "div",+ "fmap.div",+ "foo_t",+ "boost.foo_t",+ "3",+ "xpath",+ "/xhtml:html/xhtml:body/xhtml:div//node()",+ "commit",+ "true");assertQ(req("+id:example1 +foo_t:\"here is some text in a div\""), "//*[@numFound='1']");}/** test arabic PDF extraction is functional */@Testpublic void testArabicPDF() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler)- h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- loadLocal("extraction/arabic.pdf", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "fmap.Author", "extractedAuthor",- "uprefix", "ignored_",- "fmap.content", "wdf_nocase",- "literal.id", "one",- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/arabic.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "fmap.Author",+ "extractedAuthor",+ "uprefix",+ "ignored_",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "one",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("wdf_nocase:السلم"), "//result[@numFound=0]");assertU(commit());assertQ(req("wdf_nocase:السلم"), "//result[@numFound=1]");@@ -546,133 +792,215 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testTikaExceptionHandling() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler)- h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- expectThrows(Exception.class, () -> {- loadLocal("extraction/password-is-solrcell.docx", "literal.id", "one");- });+ expectThrows(+ Exception.class,+ () -> {+ loadLocal("extraction/password-is-solrcell.docx", "literal.id", "one");+ });assertU(commit());assertQ(req("*:*"), "//result[@numFound=0]");- try{- loadLocal("extraction/password-is-solrcell.docx", "fmap.created", "extractedDate", "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator", "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "literal.id", "one",- "ignoreTikaException", "true", // set ignore flag- "fmap.Last-Modified", "extractedDate");- }- catch(Exception e){+ try {+ loadLocal(+ "extraction/password-is-solrcell.docx",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "one",+ "ignoreTikaException",+ "true", // set ignore flag+ "fmap.Last-Modified",+ "extractedDate");+ } catch (Exception e) {fail("TikaException should be ignored.");}assertU(commit());assertQ(req("*:*"), "//result[@numFound=1]");}-+@Testpublic void testWrongStreamType() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);- expectThrows(Exception.class, () -> {- // Load plain text specifying another mime type, should fail- loadLocal("extraction/version_control.txt",- "literal.id", "one",- ExtractingParams.STREAM_TYPE, "application/pdf"- );- });-- expectThrows(Exception.class, () -> {- // Load plain text specifying non existing mimetype, should fail- loadLocal("extraction/version_control.txt",- "literal.id", "one",- ExtractingParams.STREAM_TYPE, "foo/bar"- );- });+ expectThrows(+ Exception.class,+ () -> {+ // Load plain text specifying another mime type, should fail+ loadLocal(+ "extraction/version_control.txt",+ "literal.id",+ "one",+ ExtractingParams.STREAM_TYPE,+ "application/pdf");+ });++ expectThrows(+ Exception.class,+ () -> {+ // Load plain text specifying non existing mimetype, should fail+ loadLocal(+ "extraction/version_control.txt",+ "literal.id",+ "one",+ ExtractingParams.STREAM_TYPE,+ "foo/bar");+ });}public void testLiteralsOverride() throws Exception {- ExtractingRequestHandler handler = (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");+ ExtractingRequestHandler handler =+ (ExtractingRequestHandler) h.getCore().getRequestHandler("/update/extract");assertTrue("handler is null and it shouldn't be", handler != null);-+assertQ(req("*:*"), "//*[@numFound='0']");// Here Tika should parse out a title for this document:- loadLocal("extraction/solr-word.pdf",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "three",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/solr-word.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "three",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Last-Modified",+ "extractedDate");// Here the literal value should override the Tika-parsed title:- loadLocal("extraction/solr-word.pdf",- "literal.title", "wolf-man",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "four",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/solr-word.pdf",+ "literal.title",+ "wolf-man",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "four",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Last-Modified",+ "extractedDate");// Here we mimic the old behaviour where literals are added, not overridden- loadLocal("extraction/solr-word.pdf",- "literalsOverride", "false",- // Trick - we first map the metadata-title to an ignored field before we replace with literal title- "fmap.title", "ignored_a",- "literal.title", "old-behaviour",- "literal.extractedKeywords", "literalkeyword",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Author", "extractedAuthor",- "literal.id", "five",- "fmap.content", "extractedContent",- "fmap.language", "extractedLanguage",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/solr-word.pdf",+ "literalsOverride",+ "false",+ // Trick - we first map the metadata-title to an ignored field before we replace with+ // literal title+ "fmap.title",+ "ignored_a",+ "literal.title",+ "old-behaviour",+ "literal.extractedKeywords",+ "literalkeyword",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Author",+ "extractedAuthor",+ "literal.id",+ "five",+ "fmap.content",+ "extractedContent",+ "fmap.language",+ "extractedLanguage",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Last-Modified",+ "extractedDate");assertU(commit());assertQ(req("title:solr-word"), "//*[@numFound='1']");assertQ(req("title:wolf-man"), "//*[@numFound='1']");- assertQ(req("extractedKeywords:(solr AND word AND pdf AND literalkeyword)"), "//*[@numFound='1']");+ assertQ(+ req("extractedKeywords:(solr AND word AND pdf AND literalkeyword)"), "//*[@numFound='1']");}@Testpublic void testPdfWithImages() throws Exception {- //Tests possibility to configure ParseContext (by example to extract embedded images from pdf)- loadLocal("extraction/pdf-with-image.pdf",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "literal.id", "pdfWithImage",- "resource.name", "pdf-with-image.pdf",- "resource.password", "solrRules",- "fmap.Last-Modified", "extractedDate");+ // Tests possibility to configure ParseContext (by example to extract embedded images from pdf)+ loadLocal(+ "extraction/pdf-with-image.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "pdfWithImage",+ "resource.name",+ "pdf-with-image.pdf",+ "resource.password",+ "solrRules",+ "fmap.Last-Modified",+ "extractedDate");assertQ(req("wdf_nocase:\"embedded:image0.jpg\""), "//*[@numFound='0']");assertU(commit());@@ -682,74 +1010,126 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {@Testpublic void testPasswordProtected() throws Exception {// PDF, Passwords from resource.password- loadLocal("extraction/encrypted-password-is-solrRules.pdf",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "literal.id", "pdfpwliteral",- "resource.name", "encrypted-password-is-solrRules.pdf",- "resource.password", "solrRules",- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/encrypted-password-is-solrRules.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "pdfpwliteral",+ "resource.name",+ "encrypted-password-is-solrRules.pdf",+ "resource.password",+ "solrRules",+ "fmap.Last-Modified",+ "extractedDate");// PDF, Passwords from passwords property file- loadLocal("extraction/encrypted-password-is-solrRules.pdf",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "literal.id", "pdfpwfile",- "resource.name", "encrypted-password-is-solrRules.pdf",- "passwordsFile", "passwordRegex.properties", // Passwords-file- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/encrypted-password-is-solrRules.pdf",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "pdfpwfile",+ "resource.name",+ "encrypted-password-is-solrRules.pdf",+ "passwordsFile",+ "passwordRegex.properties", // Passwords-file+ "fmap.Last-Modified",+ "extractedDate");// DOCX, Explicit password- loadLocal("extraction/password-is-Word2010.docx",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "uprefix", "ignored_",- "literal.id", "docxpwliteral",- "resource.name", "password-is-Word2010.docx",- "resource.password", "Word2010", // Explicit password- "fmap.Last-Modified", "extractedDate");+ loadLocal(+ "extraction/password-is-Word2010.docx",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "uprefix",+ "ignored_",+ "literal.id",+ "docxpwliteral",+ "resource.name",+ "password-is-Word2010.docx",+ "resource.password",+ "Word2010", // Explicit password+ "fmap.Last-Modified",+ "extractedDate");// DOCX, Passwords from file- loadLocal("extraction/password-is-Word2010.docx",- "fmap.created", "extractedDate",- "fmap.producer", "extractedProducer",- "fmap.creator", "extractedCreator",- "fmap.Keywords", "extractedKeywords",- "fmap.Creation-Date", "extractedDate",- "uprefix", "ignored_",- "fmap.Author", "extractedAuthor",- "fmap.content", "wdf_nocase",- "literal.id", "docxpwfile",- "resource.name", "password-is-Word2010.docx",- "passwordsFile", "passwordRegex.properties", // Passwords-file- "fmap.Last-Modified", "extractedDate");-+ loadLocal(+ "extraction/password-is-Word2010.docx",+ "fmap.created",+ "extractedDate",+ "fmap.producer",+ "extractedProducer",+ "fmap.creator",+ "extractedCreator",+ "fmap.Keywords",+ "extractedKeywords",+ "fmap.Creation-Date",+ "extractedDate",+ "uprefix",+ "ignored_",+ "fmap.Author",+ "extractedAuthor",+ "fmap.content",+ "wdf_nocase",+ "literal.id",+ "docxpwfile",+ "resource.name",+ "password-is-Word2010.docx",+ "passwordsFile",+ "passwordRegex.properties", // Passwords-file+ "fmap.Last-Modified",+ "extractedDate");+assertU(commit());Thread.sleep(100);assertQ(req("wdf_nocase:\"This is a test of PDF\""), "//*[@numFound='2']");assertQ(req("wdf_nocase:\"Test password protected word doc\""), "//*[@numFound='2']");}-- SolrQueryResponse loadLocalFromHandler(String handler, String filename,- String... args) throws Exception {-++ SolrQueryResponse loadLocalFromHandler(String handler, String filename, String... args)+ throws Exception {+LocalSolrQueryRequest req = (LocalSolrQueryRequest) req(args);try {// TODO: stop using locally defined streams once stream.file and@@ -766,6 +1146,4 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {SolrQueryResponse loadLocal(String filename, String... args) throws Exception {return loadLocalFromHandler("/update/extract", filename, args);}--}diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ParseContextConfigTest.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ParseContextConfigTest.javaindex 8aeeaad1a42..680ba14ebc2 100644--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ParseContextConfigTest.java+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/ParseContextConfigTest.java@@ -16,9 +16,8 @@*/package org.apache.solr.handler.extraction;-import javax.xml.parsers.DocumentBuilderFactory;import java.nio.file.Paths;-+import javax.xml.parsers.DocumentBuilderFactory;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.core.SolrResourceLoader;import org.apache.tika.parser.ParseContext;@@ -28,12 +27,11 @@ import org.w3c.dom.Element;public class ParseContextConfigTest extends SolrTestCaseJ4 {- public void testAll() throws Exception {+ public void testAll() throws Exception {Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();Element entries = document.createElement("entries");Element entry = document.createElement("entry");-entry.setAttribute("class", "org.apache.tika.parser.pdf.PDFParserConfig");entry.setAttribute("impl", "org.apache.tika.parser.pdf.PDFParserConfig");@@ -44,11 +42,11 @@ public class ParseContextConfigTest extends SolrTestCaseJ4 {entry.appendChild(property);entries.appendChild(entry);- ParseContext parseContext = new ParseContextConfig(new SolrResourceLoader(Paths.get(".")), entries).create();+ ParseContext parseContext =+ new ParseContextConfig(new SolrResourceLoader(Paths.get(".")), entries).create();PDFParserConfig pdfParserConfig = parseContext.get(PDFParserConfig.class);assertEquals(true, pdfParserConfig.getExtractInlineImages());}-}diff --git a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.javaindex 78f2df37269..56b89451a34 100644--- a/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java+++ b/solr/modules/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java@@ -23,7 +23,6 @@ import java.util.Arrays;import java.util.Date;import java.util.List;import java.util.stream.Collectors;-import org.apache.poi.ss.usermodel.Cell;import org.apache.poi.ss.usermodel.Row;import org.apache.poi.xssf.usermodel.XSSFSheet;@@ -48,9 +47,9 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {@BeforeClasspublic static void beforeClass() throws Exception {System.setProperty("enable.update.log", "false");- initCore("solrconfig.xml","schema.xml",getFile("extraction/solr").getAbsolutePath());+ initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath());createIndex();- //find a reference to the default response writer so we can redirect its output later+ // find a reference to the default response writer so we can redirect its output laterSolrCore testCore = h.getCore();QueryResponseWriter writer = testCore.getQueryResponseWriter("xlsx");if (writer instanceof XLSXResponseWriter) {@@ -61,11 +60,41 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {}public static void createIndex() {- assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt1","2000-01-02T03:04:05Z"));- assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo"));- assertU(adoc("id","3", "shouldbeunstored","foo"));- assertU(adoc("id","4", "foo_s1","foo"));- assertU(adoc("id","5", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));+ assertU(+ adoc(+ "id",+ "1",+ "foo_i",+ "-1",+ "foo_s",+ "hi",+ "foo_l",+ "12345678987654321",+ "foo_b",+ "false",+ "foo_f",+ "1.414",+ "foo_d",+ "-1.0E300",+ "foo_dt1",+ "2000-01-02T03:04:05Z"));+ assertU(+ adoc(+ "id",+ "2",+ "v_ss",+ "hi",+ "v_ss",+ "there",+ "v2_ss",+ "nice",+ "v2_ss",+ "output",+ "shouldbeunstored",+ "foo"));+ assertU(adoc("id", "3", "shouldbeunstored", "foo"));+ assertU(adoc("id", "4", "foo_s1", "foo"));+ assertU(adoc("id", "5", "pubyear_ii", "123", "store_iis", "12", "price_ff", "1.3"));assertU(commit());}@@ -85,212 +114,266 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {SolrQueryRequest r = req();// check Content-Type- assertEquals("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", writerXlsx.getContentType(r, rsp));+ assertEquals(+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ writerXlsx.getContentType(r, rsp));// test our basic types,and that fields come back in the requested order- XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1"));+ XSSFSheet resultSheet =+ getWSResultForQuery(+ req("q", "id:1", "wt", "xlsx", "fl", "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1"));- assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1\n1,hi,-1,12345678987654321,F,1.414,-1.0E300,2000-01-02T03:04:05Z\n"- , getStringFromSheet(resultSheet));+ assertEquals(+ "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1\n1,hi,-1,12345678987654321,F,1.414,-1.0E300,2000-01-02T03:04:05Z\n",+ getStringFromSheet(resultSheet));- resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "fl","id,score,foo_s"));+ resultSheet = getWSResultForQuery(req("q", "id:1^0", "wt", "xlsx", "fl", "id,score,foo_s"));// test retrieving scoreassertEquals("id,score,foo_s\n1,0.0,hi\n", getStringFromSheet(resultSheet));- resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "colname.id", "I.D.", "colwidth.id", "10",- "fl","id,score,foo_s"));+ resultSheet =+ getWSResultForQuery(+ req(+ "q",+ "id:1^0",+ "wt",+ "xlsx",+ "colname.id",+ "I.D.",+ "colwidth.id",+ "10",+ "fl",+ "id,score,foo_s"));// test override colname/widthassertEquals("I.D.,score,foo_s\n1,0.0,hi\n", getStringFromSheet(resultSheet));// test colwidth (value returned is in 256ths of a character as per excel standard)- assertEquals(10*256, resultSheet.getColumnWidth(0));+ assertEquals(10 * 256, resultSheet.getColumnWidth(0));- resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,v_ss"));+ resultSheet = getWSResultForQuery(req("q", "id:2", "wt", "xlsx", "fl", "id,v_ss"));// test multivaluedassertEquals("id,v_ss\n2,hi; there\n", getStringFromSheet(resultSheet));// test retrieving fields from index- resultSheet = getWSResultForQuery(req("q","*:*", "wt","xslx", "fl","*,score"));+ resultSheet = getWSResultForQuery(req("q", "*:*", "wt", "xslx", "fl", "*,score"));String result = getStringFromSheet(resultSheet);- for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss,score".split(",")) {+ for (String field :+ "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss,score".split(",")) {assertTrue(result.indexOf(field) >= 0);}// test null values- resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,foo_s,v_ss"));+ resultSheet = getWSResultForQuery(req("q", "id:2", "wt", "xlsx", "fl", "id,foo_s,v_ss"));assertEquals("id,foo_s,v_ss\n2,,hi; there\n", getStringFromSheet(resultSheet));// now test SolrDocumentListSolrDocument d = new SolrDocument();SolrDocument d1 = d;- d.addField("id","1");- d.addField("foo_i",-1);- d.addField("foo_s","hi");- d.addField("foo_l","12345678987654321L");- d.addField("foo_b",false);- d.addField("foo_f",1.414f);- d.addField("foo_d",-1.0E300);+ d.addField("id", "1");+ d.addField("foo_i", -1);+ d.addField("foo_s", "hi");+ d.addField("foo_l", "12345678987654321L");+ d.addField("foo_b", false);+ d.addField("foo_f", 1.414f);+ d.addField("foo_d", -1.0E300);d.addField("foo_dt1", new Date(Instant.parse("2000-01-02T03:04:05Z").toEpochMilli()));d.addField("score", "2.718");d = new SolrDocument();SolrDocument d2 = d;- d.addField("id","2");- d.addField("v_ss","hi");- d.addField("v_ss","there");- d.addField("v2_ss","nice");- d.addField("v2_ss","output");+ d.addField("id", "2");+ d.addField("v_ss", "hi");+ d.addField("v_ss", "there");+ d.addField("v2_ss", "nice");+ d.addField("v2_ss", "output");d.addField("score", "89.83");- d.addField("shouldbeunstored","foo");+ d.addField("shouldbeunstored", "foo");SolrDocumentList sdl = new SolrDocumentList();sdl.add(d1);sdl.add(d2);-- SolrQueryRequest req = req("q","*:*");++ SolrQueryRequest req = req("q", "*:*");rsp = new SolrQueryResponse();rsp.addResponse(sdl);- rsp.setReturnFields( new SolrReturnFields("id,foo_s", req) );+ rsp.setReturnFields(new SolrReturnFields("id,foo_s", req));resultSheet = getWSResultForQuery(req, rsp);assertEquals("id,foo_s\n1,hi\n2,\n", getStringFromSheet(resultSheet));// try scores- rsp.setReturnFields( new SolrReturnFields("id,score,foo_s", req) );+ rsp.setReturnFields(new SolrReturnFields("id,score,foo_s", req));resultSheet = getWSResultForQuery(req, rsp);assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", getStringFromSheet(resultSheet));// get field values from docs... should be ordered and not include score unless requested- rsp.setReturnFields( new SolrReturnFields("*", req) );+ rsp.setReturnFields(new SolrReturnFields("*", req));resultSheet = getWSResultForQuery(req, rsp);- assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n" +- "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" +- "2,,,,,,,,hi; there,nice; output\n", getStringFromSheet(resultSheet));-- // get field values and scores - just check that the scores are there... we don't guarantee where- rsp.setReturnFields( new SolrReturnFields("*,score", req) );+ assertEquals(+ "id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n"+ + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n"+ + "2,,,,,,,,hi; there,nice; output\n",+ getStringFromSheet(resultSheet));++ // get field values and scores - just check that the scores are there... we don't guarantee+ // where+ rsp.setReturnFields(new SolrReturnFields("*,score", req));resultSheet = getWSResultForQuery(req, rsp);String s = getStringFromSheet(resultSheet);- assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 );-+ assertTrue(s.indexOf("score") >= 0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0);+// Test field globs- rsp.setReturnFields( new SolrReturnFields("id,foo*", req) );+ rsp.setReturnFields(new SolrReturnFields("id,foo*", req));resultSheet = getWSResultForQuery(req, rsp);- assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1\n" +- "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" +- "2,,,,,,,\n", getStringFromSheet(resultSheet));+ assertEquals(+ "id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1\n"+ + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n"+ + "2,,,,,,,\n",+ getStringFromSheet(resultSheet));- rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) );+ rsp.setReturnFields(new SolrReturnFields("id,*_d*", req));resultSheet = getWSResultForQuery(req, rsp);- assertEquals("id,foo_d,foo_dt1\n" +- "1,-1.0E300,2000-01-02T03:04:05Z\n" +- "2,,\n", getStringFromSheet(resultSheet));+ assertEquals(+ "id,foo_d,foo_dt1\n" + "1,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,\n",+ getStringFromSheet(resultSheet));// Test function queries- rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_s1),div(9,1),foo_f", req) );+ rsp.setReturnFields(new SolrReturnFields("sum(1,1),id,exists(foo_s1),div(9,1),foo_f", req));resultSheet = getWSResultForQuery(req, rsp);- assertEquals("sum(1,1),id,exists(foo_s1),div(9,1),foo_f\n" +- ",1,,,1.414\n" +- ",2,,,\n", getStringFromSheet(resultSheet));+ assertEquals(+ "sum(1,1),id,exists(foo_s1),div(9,1),foo_f\n" + ",1,,,1.414\n" + ",2,,,\n",+ getStringFromSheet(resultSheet));// Test transformers- rsp.setReturnFields( new SolrReturnFields("mydocid:[docid],[explain]", req) );+ rsp.setReturnFields(new SolrReturnFields("mydocid:[docid],[explain]", req));resultSheet = getWSResultForQuery(req, rsp);- assertEquals("mydocid,[explain]\n" +- ",\n" +- ",\n", getStringFromSheet(resultSheet));+ assertEquals("mydocid,[explain]\n" + ",\n" + ",\n", getStringFromSheet(resultSheet));req.close();}-@Testpublic void testPseudoFields() throws Exception {// Use Pseudo Field- SolrQueryRequest req = req("q","id:1", "wt","xlsx", "fl","XXX:id,foo_s");+ SolrQueryRequest req = req("q", "id:1", "wt", "xlsx", "fl", "XXX:id,foo_s");XSSFSheet resultSheet = getWSResultForQuery(req);assertEquals("XXX,foo_s\n1,hi\n", getStringFromSheet(resultSheet));-- String txt = getStringFromSheet(getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","XXX:id,YYY:[docid],FOO:foo_s")));++ String txt =+ getStringFromSheet(+ getWSResultForQuery(+ req("q", "id:1", "wt", "xlsx", "fl", "XXX:id,YYY:[docid],FOO:foo_s")));String[] lines = txt.split("\n");assertEquals(2, lines.length);- assertEquals("XXX,YYY,FOO", lines[0] );- assertEquals("1,0,hi", lines[1] );-- //assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423)- String funcText = getStringFromSheet(getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_s1)")));+ assertEquals("XXX,YYY,FOO", lines[0]);+ assertEquals("1,0,hi", lines[1]);++ // assertions specific to multiple pseudofields functions like abs, div, exists, etc..+ // (SOLR-5423)+ String funcText =+ getStringFromSheet(+ getWSResultForQuery(+ req("df", "text", "q", "*", "wt", "xlsx", "fl", "XXX:id,YYY:exists(foo_s1)")));String[] funcLines = funcText.split("\n");assertEquals(6, funcLines.length);- assertEquals("XXX,YYY", funcLines[0] );- assertEquals("1,false", funcLines[1] );- assertEquals("3,false", funcLines[3] );-- //assertions specific to single function without alias (SOLR-5423)- String singleFuncText = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","*", "wt","xlsx", "fl","exists(foo_s1),XXX:id")));+ assertEquals("XXX,YYY", funcLines[0]);+ assertEquals("1,false", funcLines[1]);+ assertEquals("3,false", funcLines[3]);++ // assertions specific to single function without alias (SOLR-5423)+ String singleFuncText =+ getStringFromSheet(+ getWSResultForQuery(+ req("df", "text", "q", "*", "wt", "xlsx", "fl", "exists(foo_s1),XXX:id")));String[] singleFuncLines = singleFuncText.split("\n");assertEquals(6, singleFuncLines.length);- assertEquals("exists(foo_s1),XXX", singleFuncLines[0] );- assertEquals("false,1", singleFuncLines[1] );- assertEquals("false,3", singleFuncLines[3] );+ assertEquals("exists(foo_s1),XXX", singleFuncLines[0]);+ assertEquals("false,1", singleFuncLines[1]);+ assertEquals("false,3", singleFuncLines[3]);// pseudo-fields with * in fl- txt = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","id:4", "wt","xlsx", "fl","*,YYY:[docid],FOO:foo_s1")));+ txt =+ getStringFromSheet(+ getWSResultForQuery(+ req("df", "text", "q", "id:4", "wt", "xlsx", "fl", "*,YYY:[docid],FOO:foo_s1")));lines = txt.split("\n");assertEquals(2, lines.length);- assertEquals(sortHeader("foo_i,foo_l,FOO,foo_s,pubyear_ii,store_iis," +- "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,YYY,foo_d,id,foo_f,v_ss,foo_s1,intDefault"), sortHeader(lines[0]));+ assertEquals(+ sortHeader(+ "foo_i,foo_l,FOO,foo_s,pubyear_ii,store_iis,"+ + "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,YYY,foo_d,id,foo_f,v_ss,foo_s1,intDefault"),+ sortHeader(lines[0]));}@Testpublic void testForDVEnabledFields() throws Exception {// for dv enabled and useDocValueAsStored=true// returns pubyear_ii, store_iis but not price_ff- String singleFuncText = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx")));- String sortedHeader = sortHeader("foo_i,foo_l,foo_s,pubyear_ii,store_iis," +- "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,foo_d,id,foo_f,v_ss,foo_s1,intDefault");+ String singleFuncText =+ getStringFromSheet(getWSResultForQuery(req("df", "text", "q", "id:5", "wt", "xlsx")));+ String sortedHeader =+ sortHeader(+ "foo_i,foo_l,foo_s,pubyear_ii,store_iis,"+ + "v2_ss,multiDefault,timestamp,foo_dt1,foo_b,foo_d,id,foo_f,v_ss,foo_s1,intDefault");String[] singleFuncLines = singleFuncText.split("\n");assertEquals(2, singleFuncLines.length);assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));- List<String> actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))- .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))- .collect(Collectors.toList());+ List<String> actualVal =+ Arrays.stream(singleFuncLines[1].trim().split(","))+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))+ .collect(Collectors.toList());assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));// explicit fl=*- singleFuncText = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "*")));+ singleFuncText =+ getStringFromSheet(+ getWSResultForQuery(req("df", "text", "q", "id:5", "wt", "xlsx", "fl", "*")));singleFuncLines = singleFuncText.split("\n");assertEquals(2, singleFuncLines.length);assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));- actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))- .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))- .collect(Collectors.toList());+ actualVal =+ Arrays.stream(singleFuncLines[1].trim().split(","))+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))+ .collect(Collectors.toList());assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12")));// explicit price_ff- singleFuncText = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "fl", "price_ff")));+ singleFuncText =+ getStringFromSheet(+ getWSResultForQuery(req("df", "text", "q", "id:5", "wt", "xlsx", "fl", "price_ff")));singleFuncLines = singleFuncText.split("\n");assertEquals(2, singleFuncLines.length);assertEquals("price_ff", singleFuncLines[0]);assertEquals("1.3", singleFuncLines[1]);// explicit price_ff with fl=*- singleFuncText = getStringFromSheet(- getWSResultForQuery(req("df", "text", "q","id:5", "wt","xlsx", "csv.header","true", "fl", "*,price_ff")));- sortedHeader = sortHeader("foo_i,foo_l,foo_b,foo_s,pubyear_ii,store_iis," +- "v2_ss,multiDefault,timestamp,foo_dt1,id,foo_d,foo_f,v_ss,foo_s1,intDefault,price_ff");+ singleFuncText =+ getStringFromSheet(+ getWSResultForQuery(+ req(+ "df",+ "text",+ "q",+ "id:5",+ "wt",+ "xlsx",+ "csv.header",+ "true",+ "fl",+ "*,price_ff")));+ sortedHeader =+ sortHeader(+ "foo_i,foo_l,foo_b,foo_s,pubyear_ii,store_iis,"+ + "v2_ss,multiDefault,timestamp,foo_dt1,id,foo_d,foo_f,v_ss,foo_s1,intDefault,price_ff");singleFuncLines = singleFuncText.split("\n");assertEquals(2, singleFuncLines.length);assertEquals(sortedHeader, sortHeader(singleFuncLines[0]));- actualVal = Arrays.stream(singleFuncLines[1].trim().split(","))- .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))- .collect(Collectors.toList());+ actualVal =+ Arrays.stream(singleFuncLines[1].trim().split(","))+ .filter(val -> !val.trim().isEmpty() && !val.trim().equals("\"\""))+ .collect(Collectors.toList());assertTrue(actualVal.containsAll(Arrays.asList("5", "123", "12", "1.3")));}@@ -300,7 +383,8 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {return getWSResultForQuery(req, rsp);}- private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {+ private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp)+ throws Exception {ByteArrayOutputStream xmlBout = new ByteArrayOutputStream();writerXlsx.write(xmlBout, req, rsp);XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray()));@@ -312,8 +396,8 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 {private String getStringFromSheet(XSSFSheet sheet) {StringBuilder output = new StringBuilder();- for (Row row: sheet) {- for (Cell cell: row) {+ for (Row row : sheet) {+ for (Cell cell : row) {output.append(cell.getStringCellValue());output.append(",");}