Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-16428: Add "permissive" mode to IgnoreLargeDocumentsProcessorFactory (#1040)

Jason Gerlowski708c46199241
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0c3e5f3a5d4..46e0429d905 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -36,7 +36,11 @@ New Features
Improvements
---------------------
-(No changes)
+
+* SOLR-16428: IgnoreLargeDocumentProcessorFactory now supports a "permissive" mode, where it logs and skips
+ offending documents but doesn't short-circuit the entire batch or return a 4xx error. This mode can be enabled
+ by setting the `permissiveMode` boolean option to `true` in your solrconfig.xml's
+ IgnoreLargeDocumentProcessorFactory declaration. (Jason Gerlowski)
Optimizations
---------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
index fea5fe6b225..16508dd864b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.java
@@ -21,16 +21,20 @@ import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.IdentityHashMap;
import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.AddUpdateCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Gives system administrators a way to ignore very large update from clients. When an update goes
@@ -41,14 +45,20 @@ import org.apache.solr.update.AddUpdateCommand;
*/
public class IgnoreLargeDocumentProcessorFactory extends UpdateRequestProcessorFactory {
public static final String LIMIT_SIZE_PARAM = "limit";
+ public static final String PERMISSIVE_MODE_PARAM = "permissiveMode";
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
// limit of a SolrInputDocument size (in kb)
- private long maxDocumentSize = 1024 * 1024;
+ private long maxDocumentSizeKb = 1024 * 1024;
+ private boolean permissiveModeEnabled = false;
@Override
public void init(NamedList<?> args) {
- maxDocumentSize = args.toSolrParams().required().getLong(LIMIT_SIZE_PARAM);
+ final SolrParams params = args.toSolrParams();
+ maxDocumentSizeKb = params.required().getLong(LIMIT_SIZE_PARAM);
args.remove(LIMIT_SIZE_PARAM);
+ permissiveModeEnabled = params.getBool(PERMISSIVE_MODE_PARAM, false);
+ args.remove(PERMISSIVE_MODE_PARAM);
if (args.size() > 0) {
throw new SolrException(SERVER_ERROR, "Unexpected init param(s): '" + args.getName(0) + "'");
@@ -59,15 +69,33 @@ public class IgnoreLargeDocumentProcessorFactory extends UpdateRequestProcessorF
public UpdateRequestProcessor getInstance(
SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
return new UpdateRequestProcessor(next) {
+
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
- long docSize = ObjectSizeEstimator.estimate(cmd.getSolrInputDocument());
- if (docSize / 1024 > maxDocumentSize) {
+ long docSizeBytes = ObjectSizeEstimator.estimate(cmd.getSolrInputDocument());
+ if (docSizeBytes > maxDocumentSizeKb * 1024) {
+ handleViolatingDoc(cmd, docSizeBytes);
+ } else {
+ super.processAdd(cmd);
+ }
+ }
+
+ private void handleViolatingDoc(AddUpdateCommand cmd, long estimatedSizeBytes) {
+ if (permissiveModeEnabled) {
+ log.warn(
+ "Skipping doc because estimated size exceeds limit. [docId={}, estimatedSize={} bytes, limitSize={}kb]",
+ cmd.getPrintableId(),
+ estimatedSizeBytes,
+ maxDocumentSizeKb);
+ } else {
throw new SolrException(
BAD_REQUEST,
- "Size of the document " + cmd.getPrintableId() + " is too large, around:" + docSize);
+ "Size of the document "
+ + cmd.getPrintableId()
+ + " is too large, around: "
+ + estimatedSizeBytes
+ + " bytes");
}
- super.processAdd(cmd);
}
};
}
diff --git a/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
index e99cb770f69..6ef37a62578 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactoryTest.java
@@ -18,6 +18,7 @@
package org.apache.solr.update.processor;
import static org.apache.solr.update.processor.IgnoreLargeDocumentProcessorFactory.ObjectSizeEstimator.estimate;
+import static org.hamcrest.Matchers.containsString;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -31,6 +32,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.util.LogListener;
import org.junit.Test;
public class IgnoreLargeDocumentProcessorFactoryTest extends SolrTestCase {
@@ -54,10 +56,30 @@ public class IgnoreLargeDocumentProcessorFactoryTest extends SolrTestCase {
requestProcessor.processAdd(getUpdate(1024));
}
- public AddUpdateCommand getUpdate(int size) {
+ @Test
+ public void testProcessorInPermissiveMode() throws IOException {
+ NamedList<Object> args = new NamedList<>();
+ args.add(IgnoreLargeDocumentProcessorFactory.LIMIT_SIZE_PARAM, 1);
+ args.add(IgnoreLargeDocumentProcessorFactory.PERMISSIVE_MODE_PARAM, true);
+
+ IgnoreLargeDocumentProcessorFactory factory = new IgnoreLargeDocumentProcessorFactory();
+ factory.init(args);
+
+ UpdateRequestProcessor processor = factory.getInstance(null, null, null);
+ try (LogListener listener = LogListener.warn(IgnoreLargeDocumentProcessorFactory.class)) {
+ processor.processAdd(getUpdate(1024));
+
+ assertThat(
+ listener.pollMessage(),
+ containsString("Skipping doc because estimated size exceeds limit"));
+ }
+ }
+
+ public AddUpdateCommand getUpdate(int sizeBytes) {
SolrInputDocument document = new SolrInputDocument();
- document.addField(new String(new byte[size], Charset.defaultCharset()), 1L);
- assertTrue(IgnoreLargeDocumentProcessorFactory.ObjectSizeEstimator.estimate(document) > size);
+ document.addField(new String(new byte[sizeBytes], Charset.defaultCharset()), 1L);
+ assertTrue(
+ IgnoreLargeDocumentProcessorFactory.ObjectSizeEstimator.estimate(document) > sizeBytes);
AddUpdateCommand cmd = new AddUpdateCommand(null);
cmd.solrDoc = document;
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
index 7faa3f86702..ee160af4136 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
@@ -331,6 +331,11 @@ See https://cwiki.apache.org/confluence/display/solr/SolrClassification for more
{solr-javadocs}/core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to prevent large documents with size more than `limit` (in KB) from getting indexed.
It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.
++
+By default, this processor will abort the update request and send an error back to users if it encounters a document that exceed its configured limit.
+Documents processed prior to the offender are indexed by Solr; documents following the offender are left unprocessed.
++
+Alternatively, the processor offers a "permissive" mode (`permissiveMode=true`) which skips the offending document and logs a warning, but doesn't abort the remainder of the batch or return an error to users.
{solr-javadocs}/core/org/apache/solr/update/processor/RegexpBoostProcessorFactory.html[RegexpBoostProcessorFactory]:: A processor which will match content of "inputField" against regular expressions found in "boostFilename", and if it matches will return the corresponding boost value from the file and output this to "boostField" as a double value.