Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-8127 Distributed Luke (#4149)

Luke Kot-Zaniewskicac69ae12dad
diff --git a/changelog/unreleased/SOLR-8127-distributed-luke.yml b/changelog/unreleased/SOLR-8127-distributed-luke.yml
new file mode 100644
index 00000000000..20e9c0aee8c
--- /dev/null
+++ b/changelog/unreleased/SOLR-8127-distributed-luke.yml
@@ -0,0 +1,8 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Luke handler now aggregates results across multiple shards and does this by default in SolrCloud mode.
+type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+ - name: Luke Kot-Zaniewski
+links:
+ name: SOLR-8127
+ url: https://issues.apache.org/jira/browse/SOLR-8127
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
index b5879024d10..eb58059457e 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
@@ -19,6 +19,8 @@ package org.apache.solr.handler.admin;
import static org.apache.lucene.index.IndexOptions.DOCS;
import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS;
import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+import static org.apache.solr.common.params.CommonParams.PATH;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -28,11 +30,14 @@ import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
+import java.util.Comparator;
import java.util.Date;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -67,15 +72,25 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.client.api.model.CoreStatusResponse;
+import org.apache.solr.client.api.util.SolrVersion;
+import org.apache.solr.client.solrj.response.LukeResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.api.V2ApiUtils;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.handler.component.ShardHandler;
+import org.apache.solr.handler.component.ShardHandlerFactory;
+import org.apache.solr.handler.component.ShardRequest;
+import org.apache.solr.handler.component.ShardResponse;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.CopyField;
@@ -84,7 +99,9 @@ import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.servlet.HttpSolrCall;
import org.apache.solr.update.SolrIndexWriter;
+import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,7 +114,7 @@ import org.slf4j.LoggerFactory;
* @see SegmentsInfoRequestHandler
* @since solr 1.2
*/
-public class LukeRequestHandler extends RequestHandlerBase {
+public class LukeRequestHandler extends RequestHandlerBase implements SolrCoreAware {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String NUMTERMS = "numTerms";
@@ -108,6 +125,35 @@ public class LukeRequestHandler extends RequestHandlerBase {
static final int HIST_ARRAY_SIZE = 33;
+ // Response section keys
+ static final String RSP_INDEX = "index";
+ static final String RSP_FIELDS = "fields";
+ static final String RSP_SCHEMA = "schema";
+ static final String RSP_INFO = "info";
+ static final String RSP_DOC = "doc";
+ static final String RSP_SHARDS = "shards";
+
+ // Field-level keys
+ static final String KEY_NUM_DOCS = "numDocs";
+ static final String KEY_MAX_DOC = "maxDoc";
+ static final String KEY_DELETED_DOCS = "deletedDocs";
+ static final String KEY_SEGMENT_COUNT = "segmentCount";
+ static final String KEY_TYPE = "type";
+ static final String KEY_SCHEMA_FLAGS = "schema";
+ static final String KEY_DOCS = "docs";
+ static final String KEY_DISTINCT = "distinct";
+ static final String KEY_TOP_TERMS = "topTerms";
+ static final String KEY_DYNAMIC_BASE = "dynamicBase";
+ static final String KEY_INDEX_FLAGS = "index";
+ static final String KEY_HISTOGRAM = "histogram";
+
+ private ShardHandlerFactory shardHandlerFactory;
+
+ @Override
+ public void inform(SolrCore core) {
+ this.shardHandlerFactory = core.getCoreContainer().getShardHandlerFactory();
+ }
+
@Override
public Name getPermissionName(AuthorizationContext request) {
return Name.READ_PERM;
@@ -123,7 +169,7 @@ public class LukeRequestHandler extends RequestHandlerBase {
if (v == null) return null;
if ("schema".equalsIgnoreCase(v)) return SCHEMA;
if ("index".equalsIgnoreCase(v)) return INDEX;
- if ("doc".equalsIgnoreCase(v)) return DOC;
+ if (RSP_DOC.equalsIgnoreCase(v)) return DOC;
if ("all".equalsIgnoreCase(v)) return ALL;
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown Show Style: " + v);
}
@@ -131,16 +177,26 @@ public class LukeRequestHandler extends RequestHandlerBase {
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ SolrParams params = req.getParams();
+
+ boolean isDistrib = params.getBool(DISTRIB, req.getCoreContainer().isZooKeeperAware());
+ if (!isDistrib) {
+ String shards = params.get(ShardParams.SHARDS);
+ isDistrib = shards != null && shards.indexOf('/') > 0;
+ }
+ if (isDistrib && handleDistributed(req, rsp)) {
+ return;
+ }
+
IndexSchema schema = req.getSchema();
SolrIndexSearcher searcher = req.getSearcher();
DirectoryReader reader = searcher.getIndexReader();
- SolrParams params = req.getParams();
ShowStyle style = ShowStyle.get(params.get("show"));
// If no doc is given, show all fields and top terms
final var indexVals = new SimpleOrderedMap<>();
V2ApiUtils.squashIntoNamedList(indexVals, getIndexInfo(reader));
- rsp.add("index", indexVals);
+ rsp.add(RSP_INDEX, indexVals);
if (ShowStyle.INDEX == style) {
return; // that's all we need
@@ -153,14 +209,10 @@ public class LukeRequestHandler extends RequestHandlerBase {
String v = uniqueKey.getType().toInternal(params.get(ID));
Term t = new Term(uniqueKey.getName(), v);
docId = searcher.getFirstMatch(t);
- if (docId < 0) {
- throw new SolrException(
- SolrException.ErrorCode.NOT_FOUND, "Can't find document: " + params.get(ID));
- }
}
// Read the document from the index
- if (docId != null) {
+ if (docId != null && docId > -1) {
if (style != null && style != ShowStyle.DOC) {
throw new SolrException(ErrorCode.BAD_REQUEST, "missing doc param for doc style");
}
@@ -179,11 +231,11 @@ public class LukeRequestHandler extends RequestHandlerBase {
docinfo.add("docId", docId);
docinfo.add("lucene", info);
docinfo.add("solr", doc);
- rsp.add("doc", docinfo);
+ rsp.add(RSP_DOC, docinfo);
} else if (ShowStyle.SCHEMA == style) {
- rsp.add("schema", getSchemaInfo(req.getSchema()));
+ rsp.add(RSP_SCHEMA, getSchemaInfo(req.getSchema()));
} else {
- rsp.add("fields", getIndexedFieldsInfo(req));
+ rsp.add(RSP_FIELDS, getIndexedFieldsInfo(req));
}
// Add some generally helpful information
@@ -192,8 +244,408 @@ public class LukeRequestHandler extends RequestHandlerBase {
info.add(
"NOTE",
"Document Frequency (df) is not updated when a document is marked for deletion. df values include deleted documents.");
- rsp.add("info", info);
+ rsp.add(RSP_INFO, info);
+ rsp.setHttpCaching(false);
+ }
+
+ /**
+ * Field-level response keys, declared in the order they appear in the local (non-distributed)
+ * response. EnumMap iteration follows declaration order, giving deterministic output.
+ */
+ enum FieldDataKey {
+ TYPE(KEY_TYPE),
+ SCHEMA(KEY_SCHEMA_FLAGS),
+ DYNAMIC_BASE(KEY_DYNAMIC_BASE),
+ INDEX(KEY_INDEX_FLAGS),
+ DOCS(KEY_DOCS);
+
+ final String responseKey;
+
+ FieldDataKey(String responseKey) {
+ this.responseKey = responseKey;
+ }
+ }
+
+ /** Per-field accumulation state across shards: aggregated response data and field validation. */
+ private static class AggregatedFieldData {
+ final EnumMap<FieldDataKey, Object> properties = new EnumMap<>(FieldDataKey.class);
+ final String originalShardAddr;
+ final LukeResponse.FieldInfo originalFieldInfo;
+ private String indexFlagsShardAddr;
+
+ AggregatedFieldData(String shardAddr, LukeResponse.FieldInfo fieldInfo) {
+ this.originalShardAddr = shardAddr;
+ this.originalFieldInfo = fieldInfo;
+ properties.put(FieldDataKey.TYPE, fieldInfo.getType());
+ properties.put(FieldDataKey.SCHEMA, fieldInfo.getSchema());
+ Object dynBase = fieldInfo.getExtras().get(KEY_DYNAMIC_BASE);
+ if (dynBase != null) {
+ properties.put(FieldDataKey.DYNAMIC_BASE, dynBase);
+ }
+ Object indexFlags = fieldInfo.getExtras().get(KEY_INDEX_FLAGS);
+ if (indexFlags != null) {
+ properties.put(FieldDataKey.INDEX, indexFlags);
+ this.indexFlagsShardAddr = shardAddr;
+ }
+ }
+
+ SimpleOrderedMap<Object> toResponse() {
+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+ for (Map.Entry<FieldDataKey, Object> entry : properties.entrySet()) {
+ result.add(entry.getKey().responseKey, entry.getValue());
+ }
+ return result;
+ }
+ }
+
+ private static class ShardData {
+ final String shardAddr; // key in "shards" response map
+ final Map<String, LukeResponse.FieldInfo> shardFieldInfo; // keyed by field name
+ private NamedList<Object> indexInfo; // value for "index" key in per-shard entry
+ private SimpleOrderedMap<Object> detailedFields; // keyed by field name
+
+ ShardData(String shardAddr, Map<String, LukeResponse.FieldInfo> shardFieldInfo) {
+ this.shardAddr = shardAddr;
+ this.shardFieldInfo = shardFieldInfo;
+ }
+
+ void setIndexInfo(NamedList<Object> indexInfo) {
+ this.indexInfo = indexInfo;
+ }
+
+ void addDetailedFieldInfo(String fieldName, SimpleOrderedMap<Object> fieldStats) {
+ if (detailedFields == null) {
+ detailedFields = new SimpleOrderedMap<>();
+ }
+ detailedFields.add(fieldName, fieldStats);
+ }
+
+ SimpleOrderedMap<Object> toResponseEntry() {
+ SimpleOrderedMap<Object> entry = new SimpleOrderedMap<>();
+ if (indexInfo != null) {
+ entry.add(RSP_INDEX, indexInfo);
+ }
+ if (detailedFields != null) {
+ entry.add(RSP_FIELDS, detailedFields);
+ }
+ return entry;
+ }
+ }
+
+ /**
+ * @return true if the request was handled in distributed mode, false if prepDistributed
+ * short-circuited (e.g. single-shard collection) and the caller should fall through to local
+ * logic.
+ */
+ private boolean handleDistributed(SolrQueryRequest req, SolrQueryResponse rsp) {
+ SolrParams reqParams = req.getParams();
+
+ // docId is a Lucene-internal integer, not meaningful across shards
+ if (reqParams.getInt(DOC_ID) != null) {
+ throw new SolrException(
+ ErrorCode.BAD_REQUEST,
+ "docId parameter is not supported in distributed mode."
+ + " Use the id parameter to look up documents by their Solr unique key.");
+ }
+
+ ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
+ ResponseBuilder rb = new ResponseBuilder(req, rsp, List.of());
+ shardHandler.prepDistributed(rb);
+
+ String[] shards = rb.shards;
+ if (shards == null || shards.length == 0) {
+ return false;
+ }
+
+ ShardRequest sreq = new ShardRequest();
+ sreq.shards = shards;
+ sreq.actualShards = shards;
+ sreq.responses = new ArrayList<>(shards.length);
+
+ String reqPath = (String) req.getContext().get(PATH);
+
+ for (String shard : shards) {
+ ModifiableSolrParams params = new ModifiableSolrParams(reqParams);
+ params.set(CommonParams.QT, reqPath);
+ ShardHandler.setShardAttributesToParams(params, sreq.purpose);
+ shardHandler.submit(sreq, shard, params);
+ }
+
+ ShardResponse lastSrsp = shardHandler.takeCompletedOrError();
+ if (lastSrsp == null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No responses received from shards");
+ }
+ List<ShardResponse> responses = sreq.responses;
+ for (ShardResponse srsp : responses) {
+ if (srsp.getException() != null) {
+ shardHandler.cancelAll();
+ if (srsp.getException() instanceof SolrException) {
+ throw (SolrException) srsp.getException();
+ }
+ throw new SolrException(ErrorCode.SERVER_ERROR, srsp.getException());
+ }
+ }
+
+ aggregateDistributedResponses(req, rsp, responses);
rsp.setHttpCaching(false);
+ return true;
+ }
+
+ private static String shardAddress(ShardResponse srsp) {
+ return srsp.getShardAddress() != null ? srsp.getShardAddress() : srsp.getShard();
+ }
+
+ private void aggregateDistributedResponses(
+ SolrQueryRequest req, SolrQueryResponse rsp, List<ShardResponse> responses) {
+
+ if (!responses.isEmpty()) {
+ ShardResponse firstRsp = responses.getFirst();
+ NamedList<Object> firstShardRsp = firstRsp.getSolrResponse().getResponse();
+ if (firstShardRsp == null) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Unexpected empty response from shard: " + shardAddress(firstRsp));
+ }
+ Object schema = firstShardRsp.get(RSP_SCHEMA);
+ if (schema != null) {
+ rsp.add(RSP_SCHEMA, schema);
+ }
+ }
+
+ long totalNumDocs = 0;
+ int totalMaxDoc = 0;
+ long totalDeletedDocs = 0;
+ int totalSegmentCount = 0;
+ Map<String, AggregatedFieldData> aggregatedFields = new TreeMap<>();
+ String firstDocShard = null;
+ Object firstDoc = null;
+ List<ShardData> shardDataList = new ArrayList<>();
+
+ for (ShardResponse srsp : responses) {
+ NamedList<Object> shardRsp = srsp.getSolrResponse().getResponse();
+ LukeResponse lukeRsp = new LukeResponse();
+ lukeRsp.setResponse(shardRsp);
+ // Only process field info if the shard explicitly included it in its response.
+ // LukeResponse.getFieldInfo() falls back to schema.fields which has incomplete data.
+ Map<String, LukeResponse.FieldInfo> fieldInfo =
+ shardRsp.get(RSP_FIELDS) != null ? lukeRsp.getFieldInfo() : null;
+ ShardData shardData = new ShardData(shardAddress(srsp), fieldInfo);
+
+ NamedList<Object> shardIndex = lukeRsp.getIndexInfo();
+ if (shardIndex != null) {
+ totalNumDocs += Optional.ofNullable(lukeRsp.getNumDocs()).orElse(0L);
+ totalMaxDoc = Math.max(totalMaxDoc, Optional.ofNullable(lukeRsp.getMaxDoc()).orElse(0));
+ totalDeletedDocs += Optional.ofNullable(lukeRsp.getDeletedDocs()).orElse(0L);
+ Number segCount = (Number) shardIndex.get(KEY_SEGMENT_COUNT);
+ totalSegmentCount += segCount != null ? segCount.intValue() : 0;
+
+ shardData.setIndexInfo(shardIndex);
+ }
+
+ processShardFields(shardData, aggregatedFields);
+ Object doc = shardRsp.get(RSP_DOC);
+ if (doc != null) {
+ if (firstDoc != null) {
+ throw new SolrException(
+ ErrorCode.SERVER_ERROR,
+ "Solr Id of document "
+ + firstDoc
+ + " found on multiple shards ("
+ + firstDocShard
+ + " and "
+ + shardAddress(srsp)
+ + "). The index is corrupt: unique key constraint violated.");
+ }
+ firstDoc = doc;
+ firstDocShard = shardAddress(srsp);
+ }
+ shardDataList.add(shardData);
+ }
+
+ shardDataList.sort(Comparator.comparing(sd -> sd.shardAddr));
+ SimpleOrderedMap<Object> shardsInfo = new SimpleOrderedMap<>();
+ for (ShardData sd : shardDataList) {
+ SimpleOrderedMap<Object> entry = sd.toResponseEntry();
+ if (!entry.isEmpty()) {
+ shardsInfo.add(sd.shardAddr, entry);
+ }
+ }
+
+ SimpleOrderedMap<Object> aggregatedIndex = new SimpleOrderedMap<>();
+ aggregatedIndex.add(KEY_NUM_DOCS, totalNumDocs);
+ aggregatedIndex.add(KEY_MAX_DOC, totalMaxDoc);
+ aggregatedIndex.add(KEY_DELETED_DOCS, totalDeletedDocs);
+ aggregatedIndex.add(KEY_SEGMENT_COUNT, totalSegmentCount);
+ rsp.add(RSP_INDEX, aggregatedIndex);
+
+ if (firstDoc != null) {
+ rsp.add(RSP_DOC, firstDoc);
+ }
+ boolean narrowLongs = shouldNarrowLongsForOldClient(req);
+ if (narrowLongs) {
+ narrowLongToInt(aggregatedIndex, KEY_NUM_DOCS);
+ narrowLongToInt(aggregatedIndex, KEY_DELETED_DOCS);
+ }
+ if (!aggregatedFields.isEmpty()) {
+ SimpleOrderedMap<Object> aggregatedFieldsNL = new SimpleOrderedMap<>();
+ for (Map.Entry<String, AggregatedFieldData> entry : aggregatedFields.entrySet()) {
+ SimpleOrderedMap<Object> fieldResponse = entry.getValue().toResponse();
+ if (narrowLongs) {
+ narrowLongToInt(fieldResponse, KEY_DOCS);
+ }
+ aggregatedFieldsNL.add(entry.getKey(), fieldResponse);
+ }
+ rsp.add(RSP_FIELDS, aggregatedFieldsNL);
+ }
+
+ // Add info section last (before shards), matching the local-mode key order.
+ if (!responses.isEmpty()) {
+ NamedList<Object> firstShardRsp = responses.getFirst().getSolrResponse().getResponse();
+ Object info = firstShardRsp == null ? null : firstShardRsp.get(RSP_INFO);
+ if (info != null) {
+ rsp.add(RSP_INFO, info);
+ }
+ }
+
+ if (req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
+ rsp.add(RSP_SHARDS, shardsInfo);
+ }
+ }
+
+ private void processShardFields(
+ ShardData shardData, Map<String, AggregatedFieldData> aggregatedFields) {
+ if (shardData.shardFieldInfo == null) {
+ return;
+ }
+ for (Map.Entry<String, LukeResponse.FieldInfo> entry : shardData.shardFieldInfo.entrySet()) {
+ String fieldName = entry.getKey();
+ LukeResponse.FieldInfo fi = entry.getValue();
+
+ aggregateShardField(shardData.shardAddr, fi, aggregatedFields);
+
+ // Detailed stats — kept per-shard, not aggregated
+ NamedList<Integer> topTerms = fi.getTopTerms();
+ if (topTerms != null) {
+ SimpleOrderedMap<Object> detailedFieldInfo = new SimpleOrderedMap<>();
+ detailedFieldInfo.add(KEY_TOP_TERMS, topTerms);
+ detailedFieldInfo.add(KEY_HISTOGRAM, fi.getExtras().get(KEY_HISTOGRAM));
+ detailedFieldInfo.add(KEY_DISTINCT, fi.getDistinct());
+ shardData.addDetailedFieldInfo(fieldName, detailedFieldInfo);
+ }
+ }
+ }
+
+ private void aggregateShardField(
+ String shardAddr,
+ LukeResponse.FieldInfo fi,
+ Map<String, AggregatedFieldData> aggregatedFields) {
+
+ String fieldName = fi.getName();
+
+ AggregatedFieldData fieldData = aggregatedFields.get(fieldName);
+ if (fieldData == null) {
+ fieldData = new AggregatedFieldData(shardAddr, fi);
+ aggregatedFields.put(fieldName, fieldData);
+ } else {
+ // Subsequent shards: validate that type, schema, and dynamicBase match
+ validateFieldAttr(
+ fieldName,
+ KEY_TYPE,
+ fi.getType(),
+ fieldData.originalFieldInfo.getType(),
+ shardAddr,
+ fieldData.originalShardAddr);
+ validateFieldAttr(
+ fieldName,
+ KEY_SCHEMA_FLAGS,
+ fi.getSchema(),
+ fieldData.originalFieldInfo.getSchema(),
+ shardAddr,
+ fieldData.originalShardAddr);
+ validateFieldAttr(
+ fieldName,
+ KEY_DYNAMIC_BASE,
+ fi.getExtras().get(KEY_DYNAMIC_BASE),
+ fieldData.originalFieldInfo.getExtras().get(KEY_DYNAMIC_BASE),
+ shardAddr,
+ fieldData.originalShardAddr);
+
+ Object indexFlags = fi.getExtras().get(KEY_INDEX_FLAGS);
+ if (indexFlags != null) {
+ Object existing = fieldData.properties.get(FieldDataKey.INDEX);
+ if (existing == null) {
+ fieldData.properties.put(FieldDataKey.INDEX, indexFlags);
+ fieldData.indexFlagsShardAddr = shardAddr;
+ } else {
+ validateFieldAttr(
+ fieldName,
+ KEY_INDEX_FLAGS,
+ indexFlags,
+ existing,
+ shardAddr,
+ fieldData.indexFlagsShardAddr);
+ }
+ }
+ }
+
+ // Sum per-shard doc counts
+ fieldData.properties.merge(FieldDataKey.DOCS, fi.getDocs(), (a, b) -> (long) a + (long) b);
+ }
+
+ /**
+ * Minimum client version that understands Long values in distributed Luke responses. Distributed
+ * Luke aggregates counts across shards, which can overflow Integer. Older clients cast these
+ * values to Integer and would fail with a ClassCastException.
+ */
+ private static final SolrVersion DISTRIB_LONG_COUNTS_MIN_VERSION =
+ SolrVersion.forIntegers(10, 1, 0);
+
+ private static boolean shouldNarrowLongsForOldClient(SolrQueryRequest req) {
+ HttpSolrCall call = req.getHttpSolrCall();
+ if (call == null) return false;
+ SolrVersion clientVersion = call.getUserAgentSolrVersion();
+ return clientVersion != null && clientVersion.lessThan(DISTRIB_LONG_COUNTS_MIN_VERSION);
+ }
+
+ /** Narrows a Long value to Integer if it fits, for javabin backward compatibility. */
+ private static void narrowLongToInt(NamedList<Object> nl, String key) {
+ int idx = nl.indexOf(key, 0);
+ if (idx >= 0) {
+ Object val = nl.getVal(idx);
+ if (val instanceof Long l && l >= Integer.MIN_VALUE && l <= Integer.MAX_VALUE) {
+ nl.setVal(idx, l.intValue());
+ }
+ }
+ }
+
+ /** Validates that a field attribute value is identical across shards. */
+ private void validateFieldAttr(
+ String fieldName,
+ String attrName,
+ Object currentVal,
+ Object expectedVal,
+ String currentShardAddr,
+ String expectedShardAddr) {
+ String currentStr = currentVal != null ? currentVal.toString() : null;
+ String expectedStr = expectedVal != null ? expectedVal.toString() : null;
+ if (!Objects.equals(currentStr, expectedStr)) {
+ String error =
+ "FIELD CONFIGURATION MISMATCH! Field '"
+ + fieldName
+ + "' has inconsistent '"
+ + attrName
+ + "' across shards: '"
+ + expectedStr
+ + "' (from "
+ + expectedShardAddr
+ + ") vs '"
+ + currentStr
+ + "' (from "
+ + currentShardAddr
+ + "). Use distrib=false to query individual shards and compare field configurations.";
+ log.error(error);
+ throw new SolrException(ErrorCode.SERVER_ERROR, error);
+ }
}
/**
@@ -329,8 +781,8 @@ public class LukeRequestHandler extends RequestHandlerBase {
SchemaField sfield = schema.getFieldOrNull(field.name());
FieldType ftype = (sfield == null) ? null : sfield.getType();
- f.add("type", (ftype == null) ? null : ftype.getTypeName());
- f.add("schema", getFieldFlags(sfield));
+ f.add(KEY_TYPE, (ftype == null) ? null : ftype.getTypeName());
+ f.add(KEY_SCHEMA_FLAGS, getFieldFlags(sfield));
f.add("flags", getFieldFlags(field));
f.add("value", (ftype == null) ? null : ftype.toExternal(field));
@@ -417,17 +869,18 @@ public class LukeRequestHandler extends RequestHandlerBase {
SchemaField sfield = schema.getFieldOrNull(fieldName);
FieldType ftype = (sfield == null) ? null : sfield.getType();
- fieldMap.add("type", (ftype == null) ? null : ftype.getTypeName());
- fieldMap.add("schema", getFieldFlags(sfield));
+ fieldMap.add(KEY_TYPE, (ftype == null) ? null : ftype.getTypeName());
+ fieldMap.add(KEY_SCHEMA_FLAGS, getFieldFlags(sfield));
if (sfield != null
&& schema.isDynamicField(sfield.getName())
&& schema.getDynamicPattern(sfield.getName()) != null) {
- fieldMap.add("dynamicBase", schema.getDynamicPattern(sfield.getName()));
+ fieldMap.add(KEY_DYNAMIC_BASE, schema.getDynamicPattern(sfield.getName()));
}
Terms terms = reader.terms(fieldName);
// Not indexed, so we need to report what we can (it made it through the fl param if
// specified)
if (terms == null) {
+ fieldMap.add(KEY_DOCS, 0);
finfo.add(fieldName, fieldMap);
continue;
}
@@ -441,17 +894,16 @@ public class LukeRequestHandler extends RequestHandlerBase {
try {
IndexableField fld = doc.getField(fieldName);
if (fld != null) {
- fieldMap.add("index", getFieldFlags(fld));
+ fieldMap.add(KEY_INDEX_FLAGS, getFieldFlags(fld));
} else {
- // it is a non-stored field...
- fieldMap.add("index", "(unstored field)");
+ fieldMap.add(KEY_INDEX_FLAGS, "(unstored field)");
}
} catch (Exception ex) {
log.warn("error reading field: {}", fieldName);
}
}
}
- fieldMap.add("docs", terms.getDocCount());
+ fieldMap.add(KEY_DOCS, terms.getDocCount());
}
if (fields != null && (fields.contains(fieldName) || fields.contains("*"))) {
getDetailedFieldInfo(req, fieldName, fieldMap);
@@ -728,13 +1180,13 @@ public class LukeRequestHandler extends RequestHandlerBase {
}
}
tiq.histogram.add(buckets);
- fieldMap.add("distinct", tiq.distinctTerms);
+ fieldMap.add(KEY_DISTINCT, tiq.distinctTerms);
// Include top terms
- fieldMap.add("topTerms", tiq.toNamedList(req.getSearcher().getSchema()));
+ fieldMap.add(KEY_TOP_TERMS, tiq.toNamedList(req.getSearcher().getSchema()));
// Add a histogram
- fieldMap.add("histogram", tiq.histogram.toNamedList());
+ fieldMap.add(KEY_HISTOGRAM, tiq.histogram.toNamedList());
}
private static List<String> toListOfStrings(SchemaField[] raw) {
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java
new file mode 100644
index 00000000000..f49ac3ccca0
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.LukeRequest;
+import org.apache.solr.client.solrj.request.SolrQuery;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.SolrParams;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/** Cloud-specific Luke tests that require SolrCloud features like managed schema and Schema API. */
+public class LukeHandlerCloudTest extends SolrCloudTestCase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(2).addConfig("managed", configset("cloud-managed")).configure();
+ }
+
+ private void requestLuke(String collection, SolrParams extra) throws Exception {
+ LukeRequest req = new LukeRequest(extra);
+ req.setNumTerms(0);
+ req.process(cluster.getSolrClient(), collection);
+ }
+
+ /**
+ * Verifies that distributed Luke detects inconsistent index flags across shards. Uses Schema API
+ * to change a field's {@code stored} property between indexing on different shards, producing
+ * different Lucene FieldInfo (and thus different index flags strings) on each shard.
+ */
+ @Test
+ public void testInconsistentIndexFlagsAcrossShards() throws Exception {
+ String collection = "lukeInconsistentFlags";
+ System.setProperty("managed.schema.mutable", "true");
+ CollectionAdminRequest.createCollection(collection, "managed", 2, 1)
+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+
+ cluster.waitForActiveCollection(collection, 2, 2);
+
+ try {
+ // Add a field with stored=true, indexed=true
+ Map<String, Object> fieldAttrs = new LinkedHashMap<>();
+ fieldAttrs.put("name", "test_flag_s");
+ fieldAttrs.put("type", "string");
+ fieldAttrs.put("stored", true);
+ fieldAttrs.put("indexed", true);
+ new SchemaRequest.AddField(fieldAttrs).process(cluster.getSolrClient(), collection);
+
+ // Index a target doc WITH the field, plus seed docs without it
+ SolrInputDocument targetDoc = new SolrInputDocument();
+ targetDoc.addField("id", "target");
+ targetDoc.addField("test_flag_s", "has_indexed");
+ cluster.getSolrClient().add(collection, targetDoc);
+
+ List<SolrInputDocument> seedDocs = new ArrayList<>();
+ for (int i = 0; i < 20; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "seed_" + i);
+ seedDocs.add(doc);
+ }
+ cluster.getSolrClient().add(collection, seedDocs);
+ cluster.getSolrClient().commit(collection);
+
+ // Find which shard has the target doc by querying each replica directly.
+ // Must use distrib=false — SolrCloud defaults distrib to true even on direct replica queries.
+ DocCollection docColl = getCollectionState(collection);
+ String targetSliceName = null;
+ for (Slice slice : docColl.getSlices()) {
+ Replica leader = slice.getLeader();
+ try (SolrClient client = getHttpSolrClient(leader)) {
+ SolrQuery q = new SolrQuery("id:target");
+ q.set(DISTRIB, "false");
+ QueryResponse qr = client.query(q);
+ if (qr.getResults().getNumFound() > 0) {
+ targetSliceName = slice.getName();
+ }
+ }
+ }
+ assertNotNull("target doc should exist on a shard", targetSliceName);
+
+ // Find a seed doc on the other shard
+ String otherDocId = null;
+ for (Slice slice : docColl.getSlices()) {
+ if (!slice.getName().equals(targetSliceName)) {
+ Replica leader = slice.getLeader();
+ try (SolrClient client = getHttpSolrClient(leader)) {
+ SolrQuery q = new SolrQuery("*:*");
+ q.setRows(1);
+ q.set(DISTRIB, "false");
+ QueryResponse qr = client.query(q);
+ assertTrue("other shard should have seed docs", qr.getResults().getNumFound() > 0);
+ otherDocId = (String) qr.getResults().getFirst().getFieldValue("id");
+ }
+ break;
+ }
+ }
+ assertNotNull("should find a seed doc on the other shard", otherDocId);
+
+ // Change the field to stored=false via Schema API
+ fieldAttrs.put("stored", false);
+ new SchemaRequest.ReplaceField(fieldAttrs).process(cluster.getSolrClient(), collection);
+
+ // Reload collection to pick up schema change
+ CollectionAdminRequest.reloadCollection(collection).process(cluster.getSolrClient());
+
+ // Update the other-shard doc to include the field (now unstored in the new segment)
+ SolrInputDocument updateDoc = new SolrInputDocument();
+ updateDoc.addField("id", otherDocId);
+ updateDoc.addField("test_flag_s", "not_indexed");
+ cluster.getSolrClient().add(collection, updateDoc);
+ cluster.getSolrClient().commit(collection);
+
+ // Distributed Luke should detect inconsistent index flags between the two shards.
+ // One shard has stored=true segments, the other has stored=false segments for test_flag_s.
+ // No need to set distrib=true — ZK-aware nodes default to distributed mode.
+ Exception ex =
+ expectThrows(Exception.class, () -> requestLuke(collection, params("fl", "test_flag_s")));
+ String fullMessage = SolrException.getRootCause(ex).getMessage();
+ assertTrue(
+ "exception chain should mention inconsistent index flags: " + fullMessage,
+ fullMessage.contains("inconsistent"));
+ } finally {
+ CollectionAdminRequest.deleteCollection(collection)
+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java
new file mode 100644
index 00000000000..15296027756
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java
@@ -0,0 +1,508 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import java.util.Map;
+import org.apache.solr.BaseDistributedSearchTestCase;
+import org.apache.solr.client.solrj.request.LukeRequest;
+import org.apache.solr.client.solrj.response.InputStreamResponseParser;
+import org.apache.solr.client.solrj.response.LukeResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.update.CommitUpdateCommand;
+import org.apache.solr.util.BaseTestHarness;
+import org.junit.Test;
+
+public class LukeRequestHandlerDistribTest extends BaseDistributedSearchTestCase {
+
+ private static final Long NUM_DOCS = 20L;
+
+ public LukeRequestHandlerDistribTest() {
+ fixShardCount(2);
+ }
+
+ private LukeResponse requestLuke() throws Exception {
+ return requestLuke(new ModifiableSolrParams());
+ }
+
+ private LukeResponse requestLuke(ModifiableSolrParams extra) throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("qt", "/admin/luke");
+ params.set("numTerms", "0");
+ params.set("shards.info", "true");
+ params.add(extra);
+
+ // query() sends to control and a random shard with shards param, compares responses
+ handle.put("QTime", SKIPVAL);
+ handle.put(LukeRequestHandler.RSP_INDEX, SKIP);
+ handle.put(LukeRequestHandler.RSP_SHARDS, SKIP);
+ // Detailed per-field stats (distinct, topTerms, histogram) are kept per-shard in
+ // distributed mode and intentionally excluded from the aggregated top-level fields.
+ // Local mode includes them inline, so skip them in the comparison.
+ handle.put(LukeRequestHandler.KEY_DISTINCT, SKIP);
+ handle.put(LukeRequestHandler.KEY_TOP_TERMS, SKIP);
+ handle.put(LukeRequestHandler.KEY_HISTOGRAM, SKIP);
+ QueryResponse qr = query(params);
+ LukeResponse rsp = new LukeResponse();
+ rsp.setResponse(qr.getResponse());
+
+ return rsp;
+ }
+
+ private void assertLukeXPath(ModifiableSolrParams extra, String... xpaths) throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("shards", shards);
+ params.set("shards.info", "true");
+ params.add(extra);
+ LukeRequest req = new LukeRequest(params);
+ req.setNumTerms(0);
+ req.setResponseParser(new InputStreamResponseParser("xml"));
+ NamedList<Object> raw = controlClient.request(req);
+ String xml = InputStreamResponseParser.consumeResponseToString(raw);
+ String failedXpath = BaseTestHarness.validateXPath(xml, xpaths);
+ assertNull("XPath validation failed: " + failedXpath + "\nResponse:\n" + xml, failedXpath);
+ }
+
+ private void indexTestData() throws Exception {
+ for (int i = 0; i < NUM_DOCS; i++) {
+ index("id", String.valueOf(i), "name", "name_" + i, "subject", "subject value " + (i % 5));
+ }
+ commit();
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedAggregate() throws Exception {
+ indexTestData();
+
+ LukeResponse rsp = requestLuke();
+
+ assertEquals("aggregated numDocs should equal total docs", NUM_DOCS, rsp.getNumDocs());
+ assertTrue("aggregated maxDoc should be > 0", rsp.getMaxDoc() > 0);
+ assertNotNull("deletedDocs should be present", rsp.getDeletedDocs());
+
+ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();
+ assertNotNull("shards section should be present", shardResponses);
+ assertEquals("should have 2 shard entries", 2, shardResponses.size());
+
+ Long sumShardDocs = 0L;
+ for (Map.Entry<String, LukeResponse> entry : shardResponses.entrySet()) {
+ LukeResponse shardLuke = entry.getValue();
+ assertNotNull("each shard should have numDocs", shardLuke.getNumDocs());
+ assertNotNull("each shard should have maxDoc", shardLuke.getMaxDoc());
+ sumShardDocs += shardLuke.getNumDocs();
+ }
+ assertEquals(
+ "sum of per-shard numDocs should equal aggregated numDocs", rsp.getNumDocs(), sumShardDocs);
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedFieldsAggregate() throws Exception {
+ indexTestData();
+
+ LukeResponse rsp = requestLuke();
+
+ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();
+ assertNotNull("fields should be present", fields);
+
+ LukeResponse.FieldInfo nameField = fields.get("name");
+ assertNotNull("'name' field should be present", nameField);
+ assertNotNull("field type should be present", nameField.getType());
+ assertNotNull("schema flags should be present", nameField.getSchema());
+ assertEquals(
+ "aggregated docs count for 'name' should equal total docs",
+ (long) NUM_DOCS,
+ nameField.getDocs());
+
+ LukeResponse.FieldInfo idField = fields.get("id");
+ assertNotNull("'id' field should be present", idField);
+ assertEquals("id field type should be string", "string", idField.getType());
+
+ assertLukeXPath(
+ new ModifiableSolrParams(),
+ "//lst[@name='index']/long[@name='numDocs'][.='20']",
+ "count(//lst[@name='shards']/lst)=2",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='schema']",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='index']",
+ "//lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']",
+ "//lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']",
+ "//lst[@name='fields']/lst[@name='id']/long[@name='docs'][.='20']");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDetailedFieldStatsPerShard() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("fl", "name");
+ params.set("numTerms", "5");
+
+ LukeResponse rsp = requestLuke(params);
+
+ // Top-level fields should NOT have topTerms, distinct, histogram
+ LukeResponse.FieldInfo nameField = rsp.getFieldInfo().get("name");
+ assertNotNull("'name' field should be present", nameField);
+ assertNull("topTerms should NOT be in top-level fields", nameField.getTopTerms());
+ assertEquals("distinct should NOT be in top-level fields", 0, nameField.getDistinct());
+
+ // Per-shard entries should have detailed stats
+ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();
+ assertNotNull("shards section should be present", shardResponses);
+
+ ModifiableSolrParams detailedParams = new ModifiableSolrParams();
+ detailedParams.set("fl", "name");
+ detailedParams.set("numTerms", "5");
+ assertLukeXPath(
+ detailedParams,
+ "/response/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",
+ "/response/lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']",
+ "not(/response/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms'])",
+ "not(/response/lst[@name='fields']/lst[@name='name']/lst[@name='histogram'])",
+ "not(/response/lst[@name='fields']/lst[@name='name']/int[@name='distinct'])",
+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms']",
+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='histogram']/int[@name='1']",
+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/int[@name='distinct']");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testLocalModeDefault() throws Exception {
+ indexTestData();
+
+ // Query a single client without the shards param — local mode
+ LukeRequest req = new LukeRequest();
+ req.setNumTerms(0);
+ LukeResponse rsp = req.process(controlClient);
+
+ assertNotNull("index info should be present", rsp.getIndexInfo());
+ assertNull("shards should NOT be present in local mode", rsp.getShardResponses());
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testExplicitDistribFalse() throws Exception {
+ indexTestData();
+
+ // Query a single client with distrib=false — no shards param
+ LukeRequest req = new LukeRequest(params("distrib", "false"));
+ req.setNumTerms(0);
+ LukeResponse rsp = req.process(controlClient);
+
+ assertNotNull("index info should be present", rsp.getIndexInfo());
+ assertNull("shards should NOT be present with distrib=false", rsp.getShardResponses());
+ }
+
+ @Test
+ @ShardsFixed(num = 12)
+ public void testSparseShards() throws Exception {
+ // Index a single doc on shard 0
+ index_specific(
+ 0, "id", "100", "name", "sparse test", "subject", "subject value", "cat_s", "category");
+ commit();
+
+ LukeResponse rsp = requestLuke();
+
+ // Index-level stats
+ assertEquals("numDocs should be 1", 1, (long) rsp.getNumDocs());
+ assertTrue("maxDoc should be > 0", rsp.getMaxDoc() > 0);
+ assertEquals("deletedDocs should be 0", 0L, (long) rsp.getDeletedDocs());
+
+ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();
+ assertNotNull("shards section should be present", shardResponses);
+ assertEquals("should have 12 shard entries", 12, shardResponses.size());
+
+ long sumShardDocs = 0;
+ for (Map.Entry<String, LukeResponse> entry : shardResponses.entrySet()) {
+ LukeResponse shardLuke = entry.getValue();
+ assertNotNull("each shard should have numDocs", shardLuke.getNumDocs());
+ sumShardDocs += shardLuke.getNumDocs();
+ }
+ assertEquals("sum of per-shard numDocs should be 1", 1, sumShardDocs);
+
+ // Field-level checks
+ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();
+ assertNotNull("fields should be present", fields);
+
+ LukeResponse.FieldInfo idField = fields.get("id");
+ assertNotNull("'id' field should be present", idField);
+ assertEquals("id type", "string", idField.getType());
+ assertNotNull("id schema flags", idField.getSchema());
+
+ LukeResponse.FieldInfo nameField = fields.get("name");
+ assertNotNull("'name' field should be present", nameField);
+ assertNotNull("name type", nameField.getType());
+ assertNotNull("name schema flags", nameField.getSchema());
+ assertEquals("name docs should be 1", 1, nameField.getDocs());
+
+ // Dynamic field — should have dynamicBase in extras
+ LukeResponse.FieldInfo catField = fields.get("cat_s");
+ assertNotNull("'cat_s' field should be present", catField);
+ assertNotNull("cat_s type", catField.getType());
+ assertNotNull("cat_s dynamicBase", catField.getExtras().get("dynamicBase"));
+
+ assertLukeXPath(
+ new ModifiableSolrParams(),
+ "//lst[@name='index']/long[@name='numDocs'][.='1']",
+ "//lst[@name='index']/long[@name='deletedDocs'][.='0']",
+ "count(//lst[@name='shards']/lst)=12",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='schema']",
+ "//lst[@name='fields']/lst[@name='name']/str[@name='index']",
+ "//lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='1']",
+ "//lst[@name='fields']/lst[@name='cat_s']/str[@name='type'][.='string']",
+ "//lst[@name='fields']/lst[@name='cat_s']/str[@name='dynamicBase'][.='*_s']",
+ "//lst[@name='fields']/lst[@name='cat_s']/long[@name='docs'][.='1']");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistribShowSchema() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("show", "schema");
+
+ assertLukeXPath(
+ params,
+ "//lst[@name='schema']/lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']",
+ "//lst[@name='schema']/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",
+ "//lst[@name='schema']/lst[@name='dynamicFields']/lst[@name='*_s']",
+ "//lst[@name='schema']/str[@name='uniqueKeyField'][.='id']",
+ "//lst[@name='schema']/lst[@name='types']/lst[@name='string']",
+ "//lst[@name='schema']/lst[@name='types']/lst[@name='nametext']",
+ "//lst[@name='schema']/lst[@name='similarity']",
+ "not(/response/lst[@name='fields'])",
+ "count(//lst[@name='shards']/lst)=2");
+ }
+
+ @Test
+ @ShardsFixed(num = 16)
+ public void testDeferredIndexFlags() throws Exception {
+ // Index docs with the target field across shards, plus anchor docs without it.
+ // Use numeric IDs (the default test schema copies id to integer fields).
+ // Target docs get even IDs starting at 1000, anchor docs get odd IDs.
+ for (int i = 0; i < 16 * 4; i++) {
+ index("id", String.valueOf(1000 + i * 2), "flag_target_s", "value_" + i);
+ index("id", String.valueOf(1001 + i * 2), "name", "anchor");
+ }
+ commit();
+
+ // Delete all target docs except the first one, using per-shard deletes.
+ // Then optimize to force segment merge — expunges soft-deleted docs so
+ // Terms.getDocCount() (which backs docs) reflects only live docs.
+ for (int i = 0; i < clients.size(); i++) {
+ clients.get(i).deleteByQuery("flag_target_s:* AND -id:1000");
+ clients.get(i).optimize();
+ }
+ controlClient.deleteByQuery("flag_target_s:* AND -id:1000");
+ controlClient.optimize();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("fl", "flag_target_s");
+
+ LukeResponse rsp = requestLuke(params);
+
+ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();
+ assertNotNull("fields should be present", fields);
+ LukeResponse.FieldInfo targetField = fields.get("flag_target_s");
+ assertNotNull("'flag_target_s' field should be present", targetField);
+
+ ModifiableSolrParams xpathParams = new ModifiableSolrParams();
+ xpathParams.set("fl", "flag_target_s");
+ assertLukeXPath(
+ xpathParams,
+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='type'][.='string']",
+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='dynamicBase'][.='*_s']",
+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='index']",
+ "//lst[@name='fields']/lst[@name='flag_target_s']/long[@name='docs'][.='1']");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedShardError() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("id", "0");
+ params.set("show", "schema");
+
+ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));
+ String fullMessage = SolrException.getRootCause(ex).getMessage();
+ assertTrue(
+ "exception should mention doc style mismatch: " + fullMessage,
+ fullMessage.contains("missing doc param for doc style"));
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedDocIdRejected() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("docId", "0");
+
+ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));
+ String fullMessage = SolrException.getRootCause(ex).getMessage();
+ assertTrue(
+ "exception should mention docId not supported: " + fullMessage,
+ fullMessage.contains("docId parameter is not supported in distributed mode"));
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedDocLookupFound() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("id", "0");
+
+ assertLukeXPath(
+ params,
+ "//lst[@name='doc']/int[@name='docId']",
+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='type'][.='string']",
+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='value'][.='0']",
+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='type'][.='nametext']",
+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='value'][.='name_0']",
+ "//lst[@name='doc']/arr[@name='solr']/str[.='0']",
+ "//lst[@name='doc']/arr[@name='solr']/str[.='name_0']",
+ "//lst[@name='index']",
+ "//lst[@name='info']");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedDocLookupNotFound() throws Exception {
+ indexTestData();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("id", "999888777");
+
+ LukeResponse rsp = requestLuke(params);
+
+ NamedList<Object> raw = rsp.getResponse();
+ assertNull("doc section should NOT be present for missing ID", raw.get("doc"));
+
+ assertLukeXPath(params, "not(//lst[@name='doc'])");
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testDistributedDocLookupDuplicateId() throws Exception {
+ String dupId = "99999";
+
+ // Write the same document directly to two shard cores via UpdateHandler,
+ // completely bypassing the distributed update processor chain.
+ for (int i = 0; i < 2; i++) {
+ try (SolrCore core = jettys.get(i).getCoreContainer().getCore("collection1")) {
+ SolrInputDocument solrDoc = new SolrInputDocument();
+ solrDoc.addField("id", dupId);
+ solrDoc.addField("name", "dup_copy_" + i);
+
+ AddUpdateCommand addCmd =
+ new AddUpdateCommand(new SolrQueryRequestBase(core, new ModifiableSolrParams()) {});
+ addCmd.solrDoc = solrDoc;
+ core.getUpdateHandler().addDoc(addCmd);
+
+ CommitUpdateCommand commitCmd =
+ new CommitUpdateCommand(
+ new SolrQueryRequestBase(core, new ModifiableSolrParams()) {}, false);
+ commitCmd.waitSearcher = true;
+ core.getUpdateHandler().commit(commitCmd);
+ }
+ }
+
+ // Distributed Luke doc lookup should detect the corruption
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("id", dupId);
+
+ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));
+ String fullMessage = SolrException.getRootCause(ex).getMessage();
+ assertTrue(
+ "exception should mention duplicate/corrupt index: " + fullMessage,
+ fullMessage.contains("found on multiple shards"));
+ }
+
+ @Test
+ @ShardsFixed(num = 2)
+ public void testShardsParamRoutesToSpecificShard() throws Exception {
+ // Index a doc with a dynamic field only to shard 0
+ index_specific(0, "id", "700", "name", "shard0_only", "only_on_shard0_s", "present");
+ // Index a plain doc to shard 1 (no dynamic field)
+ index_specific(1, "id", "701", "name", "shard1_only");
+ commit();
+
+ // Query with shards= pointing only at shard 1 — the dynamic field should NOT appear.
+ // This also tests that a single remote shard is correctly fanned out to rather than
+ // falling through to local-mode on the coordinating node.
+ LukeRequest req = new LukeRequest(params("shards", shardsArr[1]));
+ req.setNumTerms(0);
+ LukeResponse rsp = req.process(controlClient);
+
+ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();
+ assertNotNull("fields should be present", fields);
+ assertNull(
+ "only_on_shard0_s should NOT be present when querying only shard 1",
+ fields.get("only_on_shard0_s"));
+ assertNotNull("'name' field should still be present", fields.get("name"));
+
+ // Now query with shards= pointing only at shard 0 — the dynamic field SHOULD appear
+ req = new LukeRequest(params("shards", shardsArr[0]));
+ req.setNumTerms(0);
+ rsp = req.process(controlClient);
+
+ fields = rsp.getFieldInfo();
+ assertNotNull("fields should be present", fields);
+ assertNotNull(
+ "only_on_shard0_s SHOULD be present when querying shard 0", fields.get("only_on_shard0_s"));
+ }
+
+ @Test
+ @ShardsFixed(num = 1)
+ public void testSingleShardViaParamStillDistributes() throws Exception {
+ index("id", "500", "name", "test_name");
+ commit();
+
+ // Pass the shards param with a single shard — should still fan out to it
+ // rather than incorrectly falling through to local mode
+ LukeRequest req = new LukeRequest(params("shards", shards, "shards.info", "true"));
+ req.setNumTerms(0);
+ LukeResponse rsp = req.process(controlClient);
+
+ assertNotNull("index info should be present", rsp.getIndexInfo());
+ assertEquals("should see the 1 doc we indexed", 1, (long) rsp.getNumDocs());
+ assertNotNull(
+ "shards section should be present when shards.info=true", rsp.getShardResponses());
+ assertEquals("should have 1 shard entry", 1, rsp.getShardResponses().size());
+
+ // Without shards.info, shards section should be absent
+ req = new LukeRequest(params("shards", shards));
+ req.setNumTerms(0);
+ rsp = req.process(controlClient);
+ assertNotNull("index info should be present", rsp.getIndexInfo());
+ assertEquals("should see the 1 doc we indexed", 1, (long) rsp.getNumDocs());
+ assertNull("shards section should be absent without shards.info", rsp.getShardResponses());
+ }
+}
diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc
index fb795f62cc1..42180716544 100644
--- a/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc
+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc
@@ -83,6 +83,18 @@ The number of top terms for each field.
Choose whether `/luke` should return the index-flags for each field.
Fetching and returning the index-flags for each field in the index has non-zero cost, and can slow down requests to `/luke`.
+`distrib`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: `false`
+|===
++
+When set to `true` in SolrCloud mode, the handler aggregates results from all shards in the collection.
+Additive index metrics (`numDocs`, `deletedDocs`, `segmentCount`) are summed across shards; `maxDoc` is the maximum across shards.
+Field types and schema flags are validated for consistency across shards.
+Per-shard index details and per-field detailed statistics are returned under a `shards` key.
+
== LukeRequestHandler Examples
All of the examples in this section assume you are running the "techproducts" Solr example:
@@ -118,3 +130,43 @@ Alternatively, to work through the Lucene native id:
http://localhost:8983/solr/techproducts/admin/luke?fl=manu&docId=0
From SolrJ, you can access /luke using the {solr-javadocs}/solrj/org/apache/solr/client/solrj/request/LukeRequest.html[`LukeRequest`] object.
+
+== Distributed Mode (multiple shards)
+
+When running in SolrCloud, the Luke handler automatically distributes requests across all shards in the collection, the same as search requests.
+To inspect only the receiving shard's index set `distrib=false`.
+In user-managed clusters, you can distribute across shards by passing the `shards` parameter with explicit shard URLs.
+
+To get a collection-wide view:
+
+[source,text]
+http://localhost:8983/solr/techproducts/admin/luke
+
+To get detailed field statistics across all shards for a specific field:
+
+[source,text]
+http://localhost:8983/solr/techproducts/admin/luke?fl=manu
+
+=== Response Structure
+
+In distributed mode, the response contains:
+
+* `index` -- Aggregated metrics across all shards: `numDocs`, `deletedDocs`, `segmentCount` are summed; `maxDoc` is the maximum across shards.
+* `fields` -- Aggregated field metadata. For each field: `type`, `schema` flags, and `dynamicBase` are validated to be consistent across shards; `index` flags use the first non-null value. The `docs` count is summed. Per-field detailed statistics (`topTerms`, `distinct`, `histogram`) are _not_ included at this level.
+* `doc` -- Present when `id` is specified. Contains the document from whichever shard owns it, including a `lucene` section (per-field analysis with shard-local `docFreq` values) and a `solr` section (stored fields). Only `id` is supported for distributed doc lookup; `docId` is rejected because Lucene document IDs are shard-local.
+* `schema` -- Schema information from the first responding shard (identical across shards sharing the same configset).
+* `info` -- Static info from the first responding shard.
+* `shards` -- Only present when `shards.info=true`. Contains per-shard details, with each entry keyed by shard address:
+** `index` -- Full index info for that shard (including `directory`, `segmentsFile`, `version`, `current`, `hasDeletions`, `lastModified`, `userData`).
+** `fields` -- Only present when `fl` triggers detailed statistics. Contains per-field `topTerms`, `distinct`, and `histogram` from that shard.
+
+=== Aggregation Semantics
+
+Field `type`, `schema` flags, and `dynamicBase` are validated for consistency across shards.
+If a mismatch is detected, the handler returns an error identifying the field, the conflicting values, and the shard addresses involved.
+You can use `distrib=false` to query individual shards and compare their field configurations when troubleshooting mismatches.
+The `index` flags are index-derived (not schema-derived) and may be absent on shards where the field has no indexed data; the first non-null value is used, and any subsequent non-null values are validated for consistency.
+
+Per-field detailed statistics (`topTerms`, `distinct`, `histogram`) are not aggregated across shards.
+These statistics are shard-local and appear in each shard's entry under the `shards` key (requires `shards.info=true`).
+For collection-wide term frequencies or cardinality estimates, Solr's xref:query-guide:faceting.adoc[faceting API] may cover some of these use cases.
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java
index 243d811e5d7..2680992bb7e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java
@@ -33,12 +33,18 @@ public class LukeRequest extends CollectionRequiringSolrRequest<LukeResponse> {
private int numTerms = -1;
private boolean showSchema = false;
private Boolean includeIndexFieldFlags = null;
+ private SolrParams extraParams;
public LukeRequest() {
// this request is not processed as an ADMIN request
super(METHOD.GET, "/admin/luke", SolrRequestType.ADMIN);
}
+ public LukeRequest(SolrParams params) {
+ this();
+ this.extraParams = params;
+ }
+
public LukeRequest(String path) {
super(METHOD.GET, path, SolrRequestType.ADMIN);
}
@@ -122,6 +128,9 @@ public class LukeRequest extends CollectionRequiringSolrRequest<LukeResponse> {
if (includeIndexFieldFlags != null) {
params.add("includeIndexFieldFlags", includeIndexFieldFlags.toString());
}
+ if (extraParams != null) {
+ params.add(extraParams);
+ }
return params;
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java
index 265a40dfb3e..f23bb29cffa 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java
@@ -20,10 +20,13 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
/**
* This is an incomplete representation of the data returned from Luke
@@ -113,11 +116,12 @@ public class LukeResponse extends SolrResponseBase {
String name;
String type;
String schema;
- int docs;
+ long docs;
int distinct;
EnumSet<FieldFlag> flags;
boolean cacheableFaceting;
NamedList<Integer> topTerms;
+ Map<String, Object> extras = new TreeMap<>();
public FieldInfo(String n) {
name = n;
@@ -128,19 +132,20 @@ public class LukeResponse extends SolrResponseBase {
for (Map.Entry<String, Object> entry : nl) {
if ("type".equals(entry.getKey())) {
type = (String) entry.getValue();
- }
- if ("flags".equals(entry.getKey())) {
+ } else if ("flags".equals(entry.getKey())) {
flags = parseFlags((String) entry.getValue());
} else if ("schema".equals(entry.getKey())) {
schema = (String) entry.getValue();
} else if ("docs".equals(entry.getKey())) {
- docs = (Integer) entry.getValue();
+ docs = ((Number) entry.getValue()).longValue();
} else if ("distinct".equals(entry.getKey())) {
distinct = (Integer) entry.getValue();
} else if ("cacheableFaceting".equals(entry.getKey())) {
cacheableFaceting = (Boolean) entry.getValue();
} else if ("topTerms".equals(entry.getKey())) {
topTerms = (NamedList<Integer>) entry.getValue();
+ } else {
+ extras.put(entry.getKey(), entry.getValue());
}
}
}
@@ -173,7 +178,7 @@ public class LukeResponse extends SolrResponseBase {
return distinct;
}
- public int getDocs() {
+ public long getDocs() {
return docs;
}
@@ -192,12 +197,17 @@ public class LukeResponse extends SolrResponseBase {
public NamedList<Integer> getTopTerms() {
return topTerms;
}
+
+ public Map<String, Object> getExtras() {
+ return extras;
+ }
}
private NamedList<Object> indexInfo;
private Map<String, FieldInfo> fieldInfo;
private Map<String, FieldInfo> dynamicFieldInfo;
private Map<String, FieldTypeInfo> fieldTypeInfo;
+ private Map<String, LukeResponse> shardResponses;
@Override
@SuppressWarnings("unchecked")
@@ -246,6 +256,18 @@ public class LukeResponse extends SolrResponseBase {
}
}
}
+
+ // Parse shards section (present in distributed responses)
+ SimpleOrderedMap<NamedList<Object>> shardsNL =
+ (SimpleOrderedMap<NamedList<Object>>) res.get("shards");
+ if (shardsNL != null) {
+ shardResponses = new LinkedHashMap<>();
+ for (Map.Entry<String, NamedList<Object>> entry : shardsNL) {
+ LukeResponse shardRsp = new LukeResponse();
+ shardRsp.setResponse(entry.getValue());
+ shardResponses.put(entry.getKey(), shardRsp);
+ }
+ }
}
// ----------------------------------------------------------------
@@ -256,9 +278,14 @@ public class LukeResponse extends SolrResponseBase {
return (String) indexInfo.get("directory");
}
- public Integer getNumDocs() {
+ private Long getLong(String key) {
if (indexInfo == null) return null;
- return (Integer) indexInfo.get("numDocs");
+ Number n = (Number) indexInfo.get(key);
+ return n != null ? n.longValue() : null;
+ }
+
+ public Long getNumDocs() {
+ return getLong("numDocs");
}
public Integer getMaxDoc() {
@@ -266,6 +293,10 @@ public class LukeResponse extends SolrResponseBase {
return (Integer) indexInfo.get("maxDoc");
}
+ public Long getDeletedDocs() {
+ return getLong("deletedDocs");
+ }
+
public Integer getNumTerms() {
if (indexInfo == null) return null;
return (Integer) indexInfo.get("numTerms");
@@ -299,5 +330,9 @@ public class LukeResponse extends SolrResponseBase {
return dynamicFieldInfo.get(f);
}
+ public Map<String, LukeResponse> getShardResponses() {
+ return shardResponses;
+ }
+
// ----------------------------------------------------------------
}