Apache Solr open-source search software
SOLR-8127 Distributed Luke (#4149)
Luke Kot-Zaniewskicac69ae12dad
diff --git a/changelog/unreleased/SOLR-8127-distributed-luke.yml b/changelog/unreleased/SOLR-8127-distributed-luke.ymlnew file mode 100644index 00000000000..20e9c0aee8c--- /dev/null+++ b/changelog/unreleased/SOLR-8127-distributed-luke.yml@@ -0,0 +1,8 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Luke handler now aggregates results across multiple shards and does this by default in SolrCloud mode.+type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Luke Kot-Zaniewski+links:+ name: SOLR-8127+ url: https://issues.apache.org/jira/browse/SOLR-8127diff --git a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.javaindex b5879024d10..eb58059457e 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java@@ -19,6 +19,8 @@ package org.apache.solr.handler.admin;import static org.apache.lucene.index.IndexOptions.DOCS;import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS;import static org.apache.lucene.index.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;+import static org.apache.solr.common.params.CommonParams.DISTRIB;+import static org.apache.solr.common.params.CommonParams.PATH;import java.io.IOException;import java.lang.invoke.MethodHandles;@@ -28,11 +30,14 @@ import java.nio.file.NoSuchFileException;import java.util.ArrayList;import java.util.Arrays;import java.util.Base64;+import java.util.Comparator;import java.util.Date;+import java.util.EnumMap;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Objects;+import java.util.Optional;import java.util.Set;import java.util.TreeMap;import java.util.TreeSet;@@ -67,15 +72,25 @@ import org.apache.lucene.util.CharsRefBuilder;import org.apache.lucene.util.PriorityQueue;import org.apache.solr.analysis.TokenizerChain;import org.apache.solr.client.api.model.CoreStatusResponse;+import org.apache.solr.client.api.util.SolrVersion;+import org.apache.solr.client.solrj.response.LukeResponse;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.luke.FieldFlag;import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.ShardParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;+import org.apache.solr.core.SolrCore;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.handler.api.V2ApiUtils;+import org.apache.solr.handler.component.ResponseBuilder;+import org.apache.solr.handler.component.ShardHandler;+import org.apache.solr.handler.component.ShardHandlerFactory;+import org.apache.solr.handler.component.ShardRequest;+import org.apache.solr.handler.component.ShardResponse;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.schema.CopyField;@@ -84,7 +99,9 @@ import org.apache.solr.schema.IndexSchema;import org.apache.solr.schema.SchemaField;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.security.AuthorizationContext;+import org.apache.solr.servlet.HttpSolrCall;import org.apache.solr.update.SolrIndexWriter;+import org.apache.solr.util.plugin.SolrCoreAware;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -97,7 +114,7 @@ import org.slf4j.LoggerFactory;* @see SegmentsInfoRequestHandler* @since solr 1.2*/-public class LukeRequestHandler extends RequestHandlerBase {+public class LukeRequestHandler extends RequestHandlerBase implements SolrCoreAware {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());public static final String NUMTERMS = "numTerms";@@ -108,6 +125,35 @@ public class LukeRequestHandler extends RequestHandlerBase {static final int HIST_ARRAY_SIZE = 33;+ // Response section keys+ static final String RSP_INDEX = "index";+ static final String RSP_FIELDS = "fields";+ static final String RSP_SCHEMA = "schema";+ static final String RSP_INFO = "info";+ static final String RSP_DOC = "doc";+ static final String RSP_SHARDS = "shards";++ // Field-level keys+ static final String KEY_NUM_DOCS = "numDocs";+ static final String KEY_MAX_DOC = "maxDoc";+ static final String KEY_DELETED_DOCS = "deletedDocs";+ static final String KEY_SEGMENT_COUNT = "segmentCount";+ static final String KEY_TYPE = "type";+ static final String KEY_SCHEMA_FLAGS = "schema";+ static final String KEY_DOCS = "docs";+ static final String KEY_DISTINCT = "distinct";+ static final String KEY_TOP_TERMS = "topTerms";+ static final String KEY_DYNAMIC_BASE = "dynamicBase";+ static final String KEY_INDEX_FLAGS = "index";+ static final String KEY_HISTOGRAM = "histogram";++ private ShardHandlerFactory shardHandlerFactory;++ @Override+ public void inform(SolrCore core) {+ this.shardHandlerFactory = core.getCoreContainer().getShardHandlerFactory();+ }+@Overridepublic Name getPermissionName(AuthorizationContext request) {return Name.READ_PERM;@@ -123,7 +169,7 @@ public class LukeRequestHandler extends RequestHandlerBase {if (v == null) return null;if ("schema".equalsIgnoreCase(v)) return SCHEMA;if ("index".equalsIgnoreCase(v)) return INDEX;- if ("doc".equalsIgnoreCase(v)) return DOC;+ if (RSP_DOC.equalsIgnoreCase(v)) return DOC;if ("all".equalsIgnoreCase(v)) return ALL;throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown Show Style: " + v);}@@ -131,16 +177,26 @@ public class LukeRequestHandler extends RequestHandlerBase {@Overridepublic void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {+ SolrParams params = req.getParams();++ boolean isDistrib = params.getBool(DISTRIB, req.getCoreContainer().isZooKeeperAware());+ if (!isDistrib) {+ String shards = params.get(ShardParams.SHARDS);+ isDistrib = shards != null && shards.indexOf('/') > 0;+ }+ if (isDistrib && handleDistributed(req, rsp)) {+ return;+ }+IndexSchema schema = req.getSchema();SolrIndexSearcher searcher = req.getSearcher();DirectoryReader reader = searcher.getIndexReader();- SolrParams params = req.getParams();ShowStyle style = ShowStyle.get(params.get("show"));// If no doc is given, show all fields and top termsfinal var indexVals = new SimpleOrderedMap<>();V2ApiUtils.squashIntoNamedList(indexVals, getIndexInfo(reader));- rsp.add("index", indexVals);+ rsp.add(RSP_INDEX, indexVals);if (ShowStyle.INDEX == style) {return; // that's all we need@@ -153,14 +209,10 @@ public class LukeRequestHandler extends RequestHandlerBase {String v = uniqueKey.getType().toInternal(params.get(ID));Term t = new Term(uniqueKey.getName(), v);docId = searcher.getFirstMatch(t);- if (docId < 0) {- throw new SolrException(- SolrException.ErrorCode.NOT_FOUND, "Can't find document: " + params.get(ID));- }}// Read the document from the index- if (docId != null) {+ if (docId != null && docId > -1) {if (style != null && style != ShowStyle.DOC) {throw new SolrException(ErrorCode.BAD_REQUEST, "missing doc param for doc style");}@@ -179,11 +231,11 @@ public class LukeRequestHandler extends RequestHandlerBase {docinfo.add("docId", docId);docinfo.add("lucene", info);docinfo.add("solr", doc);- rsp.add("doc", docinfo);+ rsp.add(RSP_DOC, docinfo);} else if (ShowStyle.SCHEMA == style) {- rsp.add("schema", getSchemaInfo(req.getSchema()));+ rsp.add(RSP_SCHEMA, getSchemaInfo(req.getSchema()));} else {- rsp.add("fields", getIndexedFieldsInfo(req));+ rsp.add(RSP_FIELDS, getIndexedFieldsInfo(req));}// Add some generally helpful information@@ -192,8 +244,408 @@ public class LukeRequestHandler extends RequestHandlerBase {info.add("NOTE","Document Frequency (df) is not updated when a document is marked for deletion. df values include deleted documents.");- rsp.add("info", info);+ rsp.add(RSP_INFO, info);+ rsp.setHttpCaching(false);+ }++ /**+ * Field-level response keys, declared in the order they appear in the local (non-distributed)+ * response. EnumMap iteration follows declaration order, giving deterministic output.+ */+ enum FieldDataKey {+ TYPE(KEY_TYPE),+ SCHEMA(KEY_SCHEMA_FLAGS),+ DYNAMIC_BASE(KEY_DYNAMIC_BASE),+ INDEX(KEY_INDEX_FLAGS),+ DOCS(KEY_DOCS);++ final String responseKey;++ FieldDataKey(String responseKey) {+ this.responseKey = responseKey;+ }+ }++ /** Per-field accumulation state across shards: aggregated response data and field validation. */+ private static class AggregatedFieldData {+ final EnumMap<FieldDataKey, Object> properties = new EnumMap<>(FieldDataKey.class);+ final String originalShardAddr;+ final LukeResponse.FieldInfo originalFieldInfo;+ private String indexFlagsShardAddr;++ AggregatedFieldData(String shardAddr, LukeResponse.FieldInfo fieldInfo) {+ this.originalShardAddr = shardAddr;+ this.originalFieldInfo = fieldInfo;+ properties.put(FieldDataKey.TYPE, fieldInfo.getType());+ properties.put(FieldDataKey.SCHEMA, fieldInfo.getSchema());+ Object dynBase = fieldInfo.getExtras().get(KEY_DYNAMIC_BASE);+ if (dynBase != null) {+ properties.put(FieldDataKey.DYNAMIC_BASE, dynBase);+ }+ Object indexFlags = fieldInfo.getExtras().get(KEY_INDEX_FLAGS);+ if (indexFlags != null) {+ properties.put(FieldDataKey.INDEX, indexFlags);+ this.indexFlagsShardAddr = shardAddr;+ }+ }++ SimpleOrderedMap<Object> toResponse() {+ SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();+ for (Map.Entry<FieldDataKey, Object> entry : properties.entrySet()) {+ result.add(entry.getKey().responseKey, entry.getValue());+ }+ return result;+ }+ }++ private static class ShardData {+ final String shardAddr; // key in "shards" response map+ final Map<String, LukeResponse.FieldInfo> shardFieldInfo; // keyed by field name+ private NamedList<Object> indexInfo; // value for "index" key in per-shard entry+ private SimpleOrderedMap<Object> detailedFields; // keyed by field name++ ShardData(String shardAddr, Map<String, LukeResponse.FieldInfo> shardFieldInfo) {+ this.shardAddr = shardAddr;+ this.shardFieldInfo = shardFieldInfo;+ }++ void setIndexInfo(NamedList<Object> indexInfo) {+ this.indexInfo = indexInfo;+ }++ void addDetailedFieldInfo(String fieldName, SimpleOrderedMap<Object> fieldStats) {+ if (detailedFields == null) {+ detailedFields = new SimpleOrderedMap<>();+ }+ detailedFields.add(fieldName, fieldStats);+ }++ SimpleOrderedMap<Object> toResponseEntry() {+ SimpleOrderedMap<Object> entry = new SimpleOrderedMap<>();+ if (indexInfo != null) {+ entry.add(RSP_INDEX, indexInfo);+ }+ if (detailedFields != null) {+ entry.add(RSP_FIELDS, detailedFields);+ }+ return entry;+ }+ }++ /**+ * @return true if the request was handled in distributed mode, false if prepDistributed+ * short-circuited (e.g. single-shard collection) and the caller should fall through to local+ * logic.+ */+ private boolean handleDistributed(SolrQueryRequest req, SolrQueryResponse rsp) {+ SolrParams reqParams = req.getParams();++ // docId is a Lucene-internal integer, not meaningful across shards+ if (reqParams.getInt(DOC_ID) != null) {+ throw new SolrException(+ ErrorCode.BAD_REQUEST,+ "docId parameter is not supported in distributed mode."+ + " Use the id parameter to look up documents by their Solr unique key.");+ }++ ShardHandler shardHandler = shardHandlerFactory.getShardHandler();+ ResponseBuilder rb = new ResponseBuilder(req, rsp, List.of());+ shardHandler.prepDistributed(rb);++ String[] shards = rb.shards;+ if (shards == null || shards.length == 0) {+ return false;+ }++ ShardRequest sreq = new ShardRequest();+ sreq.shards = shards;+ sreq.actualShards = shards;+ sreq.responses = new ArrayList<>(shards.length);++ String reqPath = (String) req.getContext().get(PATH);++ for (String shard : shards) {+ ModifiableSolrParams params = new ModifiableSolrParams(reqParams);+ params.set(CommonParams.QT, reqPath);+ ShardHandler.setShardAttributesToParams(params, sreq.purpose);+ shardHandler.submit(sreq, shard, params);+ }++ ShardResponse lastSrsp = shardHandler.takeCompletedOrError();+ if (lastSrsp == null) {+ throw new SolrException(ErrorCode.SERVER_ERROR, "No responses received from shards");+ }+ List<ShardResponse> responses = sreq.responses;+ for (ShardResponse srsp : responses) {+ if (srsp.getException() != null) {+ shardHandler.cancelAll();+ if (srsp.getException() instanceof SolrException) {+ throw (SolrException) srsp.getException();+ }+ throw new SolrException(ErrorCode.SERVER_ERROR, srsp.getException());+ }+ }++ aggregateDistributedResponses(req, rsp, responses);rsp.setHttpCaching(false);+ return true;+ }++ private static String shardAddress(ShardResponse srsp) {+ return srsp.getShardAddress() != null ? srsp.getShardAddress() : srsp.getShard();+ }++ private void aggregateDistributedResponses(+ SolrQueryRequest req, SolrQueryResponse rsp, List<ShardResponse> responses) {++ if (!responses.isEmpty()) {+ ShardResponse firstRsp = responses.getFirst();+ NamedList<Object> firstShardRsp = firstRsp.getSolrResponse().getResponse();+ if (firstShardRsp == null) {+ throw new SolrException(+ ErrorCode.SERVER_ERROR,+ "Unexpected empty response from shard: " + shardAddress(firstRsp));+ }+ Object schema = firstShardRsp.get(RSP_SCHEMA);+ if (schema != null) {+ rsp.add(RSP_SCHEMA, schema);+ }+ }++ long totalNumDocs = 0;+ int totalMaxDoc = 0;+ long totalDeletedDocs = 0;+ int totalSegmentCount = 0;+ Map<String, AggregatedFieldData> aggregatedFields = new TreeMap<>();+ String firstDocShard = null;+ Object firstDoc = null;+ List<ShardData> shardDataList = new ArrayList<>();++ for (ShardResponse srsp : responses) {+ NamedList<Object> shardRsp = srsp.getSolrResponse().getResponse();+ LukeResponse lukeRsp = new LukeResponse();+ lukeRsp.setResponse(shardRsp);+ // Only process field info if the shard explicitly included it in its response.+ // LukeResponse.getFieldInfo() falls back to schema.fields which has incomplete data.+ Map<String, LukeResponse.FieldInfo> fieldInfo =+ shardRsp.get(RSP_FIELDS) != null ? lukeRsp.getFieldInfo() : null;+ ShardData shardData = new ShardData(shardAddress(srsp), fieldInfo);++ NamedList<Object> shardIndex = lukeRsp.getIndexInfo();+ if (shardIndex != null) {+ totalNumDocs += Optional.ofNullable(lukeRsp.getNumDocs()).orElse(0L);+ totalMaxDoc = Math.max(totalMaxDoc, Optional.ofNullable(lukeRsp.getMaxDoc()).orElse(0));+ totalDeletedDocs += Optional.ofNullable(lukeRsp.getDeletedDocs()).orElse(0L);+ Number segCount = (Number) shardIndex.get(KEY_SEGMENT_COUNT);+ totalSegmentCount += segCount != null ? segCount.intValue() : 0;++ shardData.setIndexInfo(shardIndex);+ }++ processShardFields(shardData, aggregatedFields);+ Object doc = shardRsp.get(RSP_DOC);+ if (doc != null) {+ if (firstDoc != null) {+ throw new SolrException(+ ErrorCode.SERVER_ERROR,+ "Solr Id of document "+ + firstDoc+ + " found on multiple shards ("+ + firstDocShard+ + " and "+ + shardAddress(srsp)+ + "). The index is corrupt: unique key constraint violated.");+ }+ firstDoc = doc;+ firstDocShard = shardAddress(srsp);+ }+ shardDataList.add(shardData);+ }++ shardDataList.sort(Comparator.comparing(sd -> sd.shardAddr));+ SimpleOrderedMap<Object> shardsInfo = new SimpleOrderedMap<>();+ for (ShardData sd : shardDataList) {+ SimpleOrderedMap<Object> entry = sd.toResponseEntry();+ if (!entry.isEmpty()) {+ shardsInfo.add(sd.shardAddr, entry);+ }+ }++ SimpleOrderedMap<Object> aggregatedIndex = new SimpleOrderedMap<>();+ aggregatedIndex.add(KEY_NUM_DOCS, totalNumDocs);+ aggregatedIndex.add(KEY_MAX_DOC, totalMaxDoc);+ aggregatedIndex.add(KEY_DELETED_DOCS, totalDeletedDocs);+ aggregatedIndex.add(KEY_SEGMENT_COUNT, totalSegmentCount);+ rsp.add(RSP_INDEX, aggregatedIndex);++ if (firstDoc != null) {+ rsp.add(RSP_DOC, firstDoc);+ }+ boolean narrowLongs = shouldNarrowLongsForOldClient(req);+ if (narrowLongs) {+ narrowLongToInt(aggregatedIndex, KEY_NUM_DOCS);+ narrowLongToInt(aggregatedIndex, KEY_DELETED_DOCS);+ }+ if (!aggregatedFields.isEmpty()) {+ SimpleOrderedMap<Object> aggregatedFieldsNL = new SimpleOrderedMap<>();+ for (Map.Entry<String, AggregatedFieldData> entry : aggregatedFields.entrySet()) {+ SimpleOrderedMap<Object> fieldResponse = entry.getValue().toResponse();+ if (narrowLongs) {+ narrowLongToInt(fieldResponse, KEY_DOCS);+ }+ aggregatedFieldsNL.add(entry.getKey(), fieldResponse);+ }+ rsp.add(RSP_FIELDS, aggregatedFieldsNL);+ }++ // Add info section last (before shards), matching the local-mode key order.+ if (!responses.isEmpty()) {+ NamedList<Object> firstShardRsp = responses.getFirst().getSolrResponse().getResponse();+ Object info = firstShardRsp == null ? null : firstShardRsp.get(RSP_INFO);+ if (info != null) {+ rsp.add(RSP_INFO, info);+ }+ }++ if (req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {+ rsp.add(RSP_SHARDS, shardsInfo);+ }+ }++ private void processShardFields(+ ShardData shardData, Map<String, AggregatedFieldData> aggregatedFields) {+ if (shardData.shardFieldInfo == null) {+ return;+ }+ for (Map.Entry<String, LukeResponse.FieldInfo> entry : shardData.shardFieldInfo.entrySet()) {+ String fieldName = entry.getKey();+ LukeResponse.FieldInfo fi = entry.getValue();++ aggregateShardField(shardData.shardAddr, fi, aggregatedFields);++ // Detailed stats — kept per-shard, not aggregated+ NamedList<Integer> topTerms = fi.getTopTerms();+ if (topTerms != null) {+ SimpleOrderedMap<Object> detailedFieldInfo = new SimpleOrderedMap<>();+ detailedFieldInfo.add(KEY_TOP_TERMS, topTerms);+ detailedFieldInfo.add(KEY_HISTOGRAM, fi.getExtras().get(KEY_HISTOGRAM));+ detailedFieldInfo.add(KEY_DISTINCT, fi.getDistinct());+ shardData.addDetailedFieldInfo(fieldName, detailedFieldInfo);+ }+ }+ }++ private void aggregateShardField(+ String shardAddr,+ LukeResponse.FieldInfo fi,+ Map<String, AggregatedFieldData> aggregatedFields) {++ String fieldName = fi.getName();++ AggregatedFieldData fieldData = aggregatedFields.get(fieldName);+ if (fieldData == null) {+ fieldData = new AggregatedFieldData(shardAddr, fi);+ aggregatedFields.put(fieldName, fieldData);+ } else {+ // Subsequent shards: validate that type, schema, and dynamicBase match+ validateFieldAttr(+ fieldName,+ KEY_TYPE,+ fi.getType(),+ fieldData.originalFieldInfo.getType(),+ shardAddr,+ fieldData.originalShardAddr);+ validateFieldAttr(+ fieldName,+ KEY_SCHEMA_FLAGS,+ fi.getSchema(),+ fieldData.originalFieldInfo.getSchema(),+ shardAddr,+ fieldData.originalShardAddr);+ validateFieldAttr(+ fieldName,+ KEY_DYNAMIC_BASE,+ fi.getExtras().get(KEY_DYNAMIC_BASE),+ fieldData.originalFieldInfo.getExtras().get(KEY_DYNAMIC_BASE),+ shardAddr,+ fieldData.originalShardAddr);++ Object indexFlags = fi.getExtras().get(KEY_INDEX_FLAGS);+ if (indexFlags != null) {+ Object existing = fieldData.properties.get(FieldDataKey.INDEX);+ if (existing == null) {+ fieldData.properties.put(FieldDataKey.INDEX, indexFlags);+ fieldData.indexFlagsShardAddr = shardAddr;+ } else {+ validateFieldAttr(+ fieldName,+ KEY_INDEX_FLAGS,+ indexFlags,+ existing,+ shardAddr,+ fieldData.indexFlagsShardAddr);+ }+ }+ }++ // Sum per-shard doc counts+ fieldData.properties.merge(FieldDataKey.DOCS, fi.getDocs(), (a, b) -> (long) a + (long) b);+ }++ /**+ * Minimum client version that understands Long values in distributed Luke responses. Distributed+ * Luke aggregates counts across shards, which can overflow Integer. Older clients cast these+ * values to Integer and would fail with a ClassCastException.+ */+ private static final SolrVersion DISTRIB_LONG_COUNTS_MIN_VERSION =+ SolrVersion.forIntegers(10, 1, 0);++ private static boolean shouldNarrowLongsForOldClient(SolrQueryRequest req) {+ HttpSolrCall call = req.getHttpSolrCall();+ if (call == null) return false;+ SolrVersion clientVersion = call.getUserAgentSolrVersion();+ return clientVersion != null && clientVersion.lessThan(DISTRIB_LONG_COUNTS_MIN_VERSION);+ }++ /** Narrows a Long value to Integer if it fits, for javabin backward compatibility. */+ private static void narrowLongToInt(NamedList<Object> nl, String key) {+ int idx = nl.indexOf(key, 0);+ if (idx >= 0) {+ Object val = nl.getVal(idx);+ if (val instanceof Long l && l >= Integer.MIN_VALUE && l <= Integer.MAX_VALUE) {+ nl.setVal(idx, l.intValue());+ }+ }+ }++ /** Validates that a field attribute value is identical across shards. */+ private void validateFieldAttr(+ String fieldName,+ String attrName,+ Object currentVal,+ Object expectedVal,+ String currentShardAddr,+ String expectedShardAddr) {+ String currentStr = currentVal != null ? currentVal.toString() : null;+ String expectedStr = expectedVal != null ? expectedVal.toString() : null;+ if (!Objects.equals(currentStr, expectedStr)) {+ String error =+ "FIELD CONFIGURATION MISMATCH! Field '"+ + fieldName+ + "' has inconsistent '"+ + attrName+ + "' across shards: '"+ + expectedStr+ + "' (from "+ + expectedShardAddr+ + ") vs '"+ + currentStr+ + "' (from "+ + currentShardAddr+ + "). Use distrib=false to query individual shards and compare field configurations.";+ log.error(error);+ throw new SolrException(ErrorCode.SERVER_ERROR, error);+ }}/**@@ -329,8 +781,8 @@ public class LukeRequestHandler extends RequestHandlerBase {SchemaField sfield = schema.getFieldOrNull(field.name());FieldType ftype = (sfield == null) ? null : sfield.getType();- f.add("type", (ftype == null) ? null : ftype.getTypeName());- f.add("schema", getFieldFlags(sfield));+ f.add(KEY_TYPE, (ftype == null) ? null : ftype.getTypeName());+ f.add(KEY_SCHEMA_FLAGS, getFieldFlags(sfield));f.add("flags", getFieldFlags(field));f.add("value", (ftype == null) ? null : ftype.toExternal(field));@@ -417,17 +869,18 @@ public class LukeRequestHandler extends RequestHandlerBase {SchemaField sfield = schema.getFieldOrNull(fieldName);FieldType ftype = (sfield == null) ? null : sfield.getType();- fieldMap.add("type", (ftype == null) ? null : ftype.getTypeName());- fieldMap.add("schema", getFieldFlags(sfield));+ fieldMap.add(KEY_TYPE, (ftype == null) ? null : ftype.getTypeName());+ fieldMap.add(KEY_SCHEMA_FLAGS, getFieldFlags(sfield));if (sfield != null&& schema.isDynamicField(sfield.getName())&& schema.getDynamicPattern(sfield.getName()) != null) {- fieldMap.add("dynamicBase", schema.getDynamicPattern(sfield.getName()));+ fieldMap.add(KEY_DYNAMIC_BASE, schema.getDynamicPattern(sfield.getName()));}Terms terms = reader.terms(fieldName);// Not indexed, so we need to report what we can (it made it through the fl param if// specified)if (terms == null) {+ fieldMap.add(KEY_DOCS, 0);finfo.add(fieldName, fieldMap);continue;}@@ -441,17 +894,16 @@ public class LukeRequestHandler extends RequestHandlerBase {try {IndexableField fld = doc.getField(fieldName);if (fld != null) {- fieldMap.add("index", getFieldFlags(fld));+ fieldMap.add(KEY_INDEX_FLAGS, getFieldFlags(fld));} else {- // it is a non-stored field...- fieldMap.add("index", "(unstored field)");+ fieldMap.add(KEY_INDEX_FLAGS, "(unstored field)");}} catch (Exception ex) {log.warn("error reading field: {}", fieldName);}}}- fieldMap.add("docs", terms.getDocCount());+ fieldMap.add(KEY_DOCS, terms.getDocCount());}if (fields != null && (fields.contains(fieldName) || fields.contains("*"))) {getDetailedFieldInfo(req, fieldName, fieldMap);@@ -728,13 +1180,13 @@ public class LukeRequestHandler extends RequestHandlerBase {}}tiq.histogram.add(buckets);- fieldMap.add("distinct", tiq.distinctTerms);+ fieldMap.add(KEY_DISTINCT, tiq.distinctTerms);// Include top terms- fieldMap.add("topTerms", tiq.toNamedList(req.getSearcher().getSchema()));+ fieldMap.add(KEY_TOP_TERMS, tiq.toNamedList(req.getSearcher().getSchema()));// Add a histogram- fieldMap.add("histogram", tiq.histogram.toNamedList());+ fieldMap.add(KEY_HISTOGRAM, tiq.histogram.toNamedList());}private static List<String> toListOfStrings(SchemaField[] raw) {diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.javanew file mode 100644index 00000000000..f49ac3ccca0--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeHandlerCloudTest.java@@ -0,0 +1,156 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.admin;++import static org.apache.solr.common.params.CommonParams.DISTRIB;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.LukeRequest;+import org.apache.solr.client.solrj.request.SolrQuery;+import org.apache.solr.client.solrj.request.schema.SchemaRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.params.SolrParams;+import org.junit.BeforeClass;+import org.junit.Test;++/** Cloud-specific Luke tests that require SolrCloud features like managed schema and Schema API. */+public class LukeHandlerCloudTest extends SolrCloudTestCase {++ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(2).addConfig("managed", configset("cloud-managed")).configure();+ }++ private void requestLuke(String collection, SolrParams extra) throws Exception {+ LukeRequest req = new LukeRequest(extra);+ req.setNumTerms(0);+ req.process(cluster.getSolrClient(), collection);+ }++ /**+ * Verifies that distributed Luke detects inconsistent index flags across shards. Uses Schema API+ * to change a field's {@code stored} property between indexing on different shards, producing+ * different Lucene FieldInfo (and thus different index flags strings) on each shard.+ */+ @Test+ public void testInconsistentIndexFlagsAcrossShards() throws Exception {+ String collection = "lukeInconsistentFlags";+ System.setProperty("managed.schema.mutable", "true");+ CollectionAdminRequest.createCollection(collection, "managed", 2, 1)+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);++ cluster.waitForActiveCollection(collection, 2, 2);++ try {+ // Add a field with stored=true, indexed=true+ Map<String, Object> fieldAttrs = new LinkedHashMap<>();+ fieldAttrs.put("name", "test_flag_s");+ fieldAttrs.put("type", "string");+ fieldAttrs.put("stored", true);+ fieldAttrs.put("indexed", true);+ new SchemaRequest.AddField(fieldAttrs).process(cluster.getSolrClient(), collection);++ // Index a target doc WITH the field, plus seed docs without it+ SolrInputDocument targetDoc = new SolrInputDocument();+ targetDoc.addField("id", "target");+ targetDoc.addField("test_flag_s", "has_indexed");+ cluster.getSolrClient().add(collection, targetDoc);++ List<SolrInputDocument> seedDocs = new ArrayList<>();+ for (int i = 0; i < 20; i++) {+ SolrInputDocument doc = new SolrInputDocument();+ doc.addField("id", "seed_" + i);+ seedDocs.add(doc);+ }+ cluster.getSolrClient().add(collection, seedDocs);+ cluster.getSolrClient().commit(collection);++ // Find which shard has the target doc by querying each replica directly.+ // Must use distrib=false — SolrCloud defaults distrib to true even on direct replica queries.+ DocCollection docColl = getCollectionState(collection);+ String targetSliceName = null;+ for (Slice slice : docColl.getSlices()) {+ Replica leader = slice.getLeader();+ try (SolrClient client = getHttpSolrClient(leader)) {+ SolrQuery q = new SolrQuery("id:target");+ q.set(DISTRIB, "false");+ QueryResponse qr = client.query(q);+ if (qr.getResults().getNumFound() > 0) {+ targetSliceName = slice.getName();+ }+ }+ }+ assertNotNull("target doc should exist on a shard", targetSliceName);++ // Find a seed doc on the other shard+ String otherDocId = null;+ for (Slice slice : docColl.getSlices()) {+ if (!slice.getName().equals(targetSliceName)) {+ Replica leader = slice.getLeader();+ try (SolrClient client = getHttpSolrClient(leader)) {+ SolrQuery q = new SolrQuery("*:*");+ q.setRows(1);+ q.set(DISTRIB, "false");+ QueryResponse qr = client.query(q);+ assertTrue("other shard should have seed docs", qr.getResults().getNumFound() > 0);+ otherDocId = (String) qr.getResults().getFirst().getFieldValue("id");+ }+ break;+ }+ }+ assertNotNull("should find a seed doc on the other shard", otherDocId);++ // Change the field to stored=false via Schema API+ fieldAttrs.put("stored", false);+ new SchemaRequest.ReplaceField(fieldAttrs).process(cluster.getSolrClient(), collection);++ // Reload collection to pick up schema change+ CollectionAdminRequest.reloadCollection(collection).process(cluster.getSolrClient());++ // Update the other-shard doc to include the field (now unstored in the new segment)+ SolrInputDocument updateDoc = new SolrInputDocument();+ updateDoc.addField("id", otherDocId);+ updateDoc.addField("test_flag_s", "not_indexed");+ cluster.getSolrClient().add(collection, updateDoc);+ cluster.getSolrClient().commit(collection);++ // Distributed Luke should detect inconsistent index flags between the two shards.+ // One shard has stored=true segments, the other has stored=false segments for test_flag_s.+ // No need to set distrib=true — ZK-aware nodes default to distributed mode.+ Exception ex =+ expectThrows(Exception.class, () -> requestLuke(collection, params("fl", "test_flag_s")));+ String fullMessage = SolrException.getRootCause(ex).getMessage();+ assertTrue(+ "exception chain should mention inconsistent index flags: " + fullMessage,+ fullMessage.contains("inconsistent"));+ } finally {+ CollectionAdminRequest.deleteCollection(collection)+ .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);+ }+ }+}diff --git a/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.javanew file mode 100644index 00000000000..15296027756--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerDistribTest.java@@ -0,0 +1,508 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.admin;++import java.util.Map;+import org.apache.solr.BaseDistributedSearchTestCase;+import org.apache.solr.client.solrj.request.LukeRequest;+import org.apache.solr.client.solrj.response.InputStreamResponseParser;+import org.apache.solr.client.solrj.response.LukeResponse;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.SolrCore;+import org.apache.solr.request.SolrQueryRequestBase;+import org.apache.solr.update.AddUpdateCommand;+import org.apache.solr.update.CommitUpdateCommand;+import org.apache.solr.util.BaseTestHarness;+import org.junit.Test;++public class LukeRequestHandlerDistribTest extends BaseDistributedSearchTestCase {++ private static final Long NUM_DOCS = 20L;++ public LukeRequestHandlerDistribTest() {+ fixShardCount(2);+ }++ private LukeResponse requestLuke() throws Exception {+ return requestLuke(new ModifiableSolrParams());+ }++ private LukeResponse requestLuke(ModifiableSolrParams extra) throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("qt", "/admin/luke");+ params.set("numTerms", "0");+ params.set("shards.info", "true");+ params.add(extra);++ // query() sends to control and a random shard with shards param, compares responses+ handle.put("QTime", SKIPVAL);+ handle.put(LukeRequestHandler.RSP_INDEX, SKIP);+ handle.put(LukeRequestHandler.RSP_SHARDS, SKIP);+ // Detailed per-field stats (distinct, topTerms, histogram) are kept per-shard in+ // distributed mode and intentionally excluded from the aggregated top-level fields.+ // Local mode includes them inline, so skip them in the comparison.+ handle.put(LukeRequestHandler.KEY_DISTINCT, SKIP);+ handle.put(LukeRequestHandler.KEY_TOP_TERMS, SKIP);+ handle.put(LukeRequestHandler.KEY_HISTOGRAM, SKIP);+ QueryResponse qr = query(params);+ LukeResponse rsp = new LukeResponse();+ rsp.setResponse(qr.getResponse());++ return rsp;+ }++ private void assertLukeXPath(ModifiableSolrParams extra, String... xpaths) throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("shards", shards);+ params.set("shards.info", "true");+ params.add(extra);+ LukeRequest req = new LukeRequest(params);+ req.setNumTerms(0);+ req.setResponseParser(new InputStreamResponseParser("xml"));+ NamedList<Object> raw = controlClient.request(req);+ String xml = InputStreamResponseParser.consumeResponseToString(raw);+ String failedXpath = BaseTestHarness.validateXPath(xml, xpaths);+ assertNull("XPath validation failed: " + failedXpath + "\nResponse:\n" + xml, failedXpath);+ }++ private void indexTestData() throws Exception {+ for (int i = 0; i < NUM_DOCS; i++) {+ index("id", String.valueOf(i), "name", "name_" + i, "subject", "subject value " + (i % 5));+ }+ commit();+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedAggregate() throws Exception {+ indexTestData();++ LukeResponse rsp = requestLuke();++ assertEquals("aggregated numDocs should equal total docs", NUM_DOCS, rsp.getNumDocs());+ assertTrue("aggregated maxDoc should be > 0", rsp.getMaxDoc() > 0);+ assertNotNull("deletedDocs should be present", rsp.getDeletedDocs());++ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();+ assertNotNull("shards section should be present", shardResponses);+ assertEquals("should have 2 shard entries", 2, shardResponses.size());++ Long sumShardDocs = 0L;+ for (Map.Entry<String, LukeResponse> entry : shardResponses.entrySet()) {+ LukeResponse shardLuke = entry.getValue();+ assertNotNull("each shard should have numDocs", shardLuke.getNumDocs());+ assertNotNull("each shard should have maxDoc", shardLuke.getMaxDoc());+ sumShardDocs += shardLuke.getNumDocs();+ }+ assertEquals(+ "sum of per-shard numDocs should equal aggregated numDocs", rsp.getNumDocs(), sumShardDocs);+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedFieldsAggregate() throws Exception {+ indexTestData();++ LukeResponse rsp = requestLuke();++ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();+ assertNotNull("fields should be present", fields);++ LukeResponse.FieldInfo nameField = fields.get("name");+ assertNotNull("'name' field should be present", nameField);+ assertNotNull("field type should be present", nameField.getType());+ assertNotNull("schema flags should be present", nameField.getSchema());+ assertEquals(+ "aggregated docs count for 'name' should equal total docs",+ (long) NUM_DOCS,+ nameField.getDocs());++ LukeResponse.FieldInfo idField = fields.get("id");+ assertNotNull("'id' field should be present", idField);+ assertEquals("id field type should be string", "string", idField.getType());++ assertLukeXPath(+ new ModifiableSolrParams(),+ "//lst[@name='index']/long[@name='numDocs'][.='20']",+ "count(//lst[@name='shards']/lst)=2",+ "//lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",+ "//lst[@name='fields']/lst[@name='name']/str[@name='schema']",+ "//lst[@name='fields']/lst[@name='name']/str[@name='index']",+ "//lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']",+ "//lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']",+ "//lst[@name='fields']/lst[@name='id']/long[@name='docs'][.='20']");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDetailedFieldStatsPerShard() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("fl", "name");+ params.set("numTerms", "5");++ LukeResponse rsp = requestLuke(params);++ // Top-level fields should NOT have topTerms, distinct, histogram+ LukeResponse.FieldInfo nameField = rsp.getFieldInfo().get("name");+ assertNotNull("'name' field should be present", nameField);+ assertNull("topTerms should NOT be in top-level fields", nameField.getTopTerms());+ assertEquals("distinct should NOT be in top-level fields", 0, nameField.getDistinct());++ // Per-shard entries should have detailed stats+ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();+ assertNotNull("shards section should be present", shardResponses);++ ModifiableSolrParams detailedParams = new ModifiableSolrParams();+ detailedParams.set("fl", "name");+ detailedParams.set("numTerms", "5");+ assertLukeXPath(+ detailedParams,+ "/response/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",+ "/response/lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='20']",+ "not(/response/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms'])",+ "not(/response/lst[@name='fields']/lst[@name='name']/lst[@name='histogram'])",+ "not(/response/lst[@name='fields']/lst[@name='name']/int[@name='distinct'])",+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='topTerms']",+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/lst[@name='histogram']/int[@name='1']",+ "//lst[@name='shards']/lst/lst[@name='fields']/lst[@name='name']/int[@name='distinct']");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testLocalModeDefault() throws Exception {+ indexTestData();++ // Query a single client without the shards param — local mode+ LukeRequest req = new LukeRequest();+ req.setNumTerms(0);+ LukeResponse rsp = req.process(controlClient);++ assertNotNull("index info should be present", rsp.getIndexInfo());+ assertNull("shards should NOT be present in local mode", rsp.getShardResponses());+ }++ @Test+ @ShardsFixed(num = 2)+ public void testExplicitDistribFalse() throws Exception {+ indexTestData();++ // Query a single client with distrib=false — no shards param+ LukeRequest req = new LukeRequest(params("distrib", "false"));+ req.setNumTerms(0);+ LukeResponse rsp = req.process(controlClient);++ assertNotNull("index info should be present", rsp.getIndexInfo());+ assertNull("shards should NOT be present with distrib=false", rsp.getShardResponses());+ }++ @Test+ @ShardsFixed(num = 12)+ public void testSparseShards() throws Exception {+ // Index a single doc on shard 0+ index_specific(+ 0, "id", "100", "name", "sparse test", "subject", "subject value", "cat_s", "category");+ commit();++ LukeResponse rsp = requestLuke();++ // Index-level stats+ assertEquals("numDocs should be 1", 1, (long) rsp.getNumDocs());+ assertTrue("maxDoc should be > 0", rsp.getMaxDoc() > 0);+ assertEquals("deletedDocs should be 0", 0L, (long) rsp.getDeletedDocs());++ Map<String, LukeResponse> shardResponses = rsp.getShardResponses();+ assertNotNull("shards section should be present", shardResponses);+ assertEquals("should have 12 shard entries", 12, shardResponses.size());++ long sumShardDocs = 0;+ for (Map.Entry<String, LukeResponse> entry : shardResponses.entrySet()) {+ LukeResponse shardLuke = entry.getValue();+ assertNotNull("each shard should have numDocs", shardLuke.getNumDocs());+ sumShardDocs += shardLuke.getNumDocs();+ }+ assertEquals("sum of per-shard numDocs should be 1", 1, sumShardDocs);++ // Field-level checks+ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();+ assertNotNull("fields should be present", fields);++ LukeResponse.FieldInfo idField = fields.get("id");+ assertNotNull("'id' field should be present", idField);+ assertEquals("id type", "string", idField.getType());+ assertNotNull("id schema flags", idField.getSchema());++ LukeResponse.FieldInfo nameField = fields.get("name");+ assertNotNull("'name' field should be present", nameField);+ assertNotNull("name type", nameField.getType());+ assertNotNull("name schema flags", nameField.getSchema());+ assertEquals("name docs should be 1", 1, nameField.getDocs());++ // Dynamic field — should have dynamicBase in extras+ LukeResponse.FieldInfo catField = fields.get("cat_s");+ assertNotNull("'cat_s' field should be present", catField);+ assertNotNull("cat_s type", catField.getType());+ assertNotNull("cat_s dynamicBase", catField.getExtras().get("dynamicBase"));++ assertLukeXPath(+ new ModifiableSolrParams(),+ "//lst[@name='index']/long[@name='numDocs'][.='1']",+ "//lst[@name='index']/long[@name='deletedDocs'][.='0']",+ "count(//lst[@name='shards']/lst)=12",+ "//lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",+ "//lst[@name='fields']/lst[@name='name']/str[@name='schema']",+ "//lst[@name='fields']/lst[@name='name']/str[@name='index']",+ "//lst[@name='fields']/lst[@name='name']/long[@name='docs'][.='1']",+ "//lst[@name='fields']/lst[@name='cat_s']/str[@name='type'][.='string']",+ "//lst[@name='fields']/lst[@name='cat_s']/str[@name='dynamicBase'][.='*_s']",+ "//lst[@name='fields']/lst[@name='cat_s']/long[@name='docs'][.='1']");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistribShowSchema() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("show", "schema");++ assertLukeXPath(+ params,+ "//lst[@name='schema']/lst[@name='fields']/lst[@name='id']/str[@name='type'][.='string']",+ "//lst[@name='schema']/lst[@name='fields']/lst[@name='name']/str[@name='type'][.='nametext']",+ "//lst[@name='schema']/lst[@name='dynamicFields']/lst[@name='*_s']",+ "//lst[@name='schema']/str[@name='uniqueKeyField'][.='id']",+ "//lst[@name='schema']/lst[@name='types']/lst[@name='string']",+ "//lst[@name='schema']/lst[@name='types']/lst[@name='nametext']",+ "//lst[@name='schema']/lst[@name='similarity']",+ "not(/response/lst[@name='fields'])",+ "count(//lst[@name='shards']/lst)=2");+ }++ @Test+ @ShardsFixed(num = 16)+ public void testDeferredIndexFlags() throws Exception {+ // Index docs with the target field across shards, plus anchor docs without it.+ // Use numeric IDs (the default test schema copies id to integer fields).+ // Target docs get even IDs starting at 1000, anchor docs get odd IDs.+ for (int i = 0; i < 16 * 4; i++) {+ index("id", String.valueOf(1000 + i * 2), "flag_target_s", "value_" + i);+ index("id", String.valueOf(1001 + i * 2), "name", "anchor");+ }+ commit();++ // Delete all target docs except the first one, using per-shard deletes.+ // Then optimize to force segment merge — expunges soft-deleted docs so+ // Terms.getDocCount() (which backs docs) reflects only live docs.+ for (int i = 0; i < clients.size(); i++) {+ clients.get(i).deleteByQuery("flag_target_s:* AND -id:1000");+ clients.get(i).optimize();+ }+ controlClient.deleteByQuery("flag_target_s:* AND -id:1000");+ controlClient.optimize();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("fl", "flag_target_s");++ LukeResponse rsp = requestLuke(params);++ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();+ assertNotNull("fields should be present", fields);+ LukeResponse.FieldInfo targetField = fields.get("flag_target_s");+ assertNotNull("'flag_target_s' field should be present", targetField);++ ModifiableSolrParams xpathParams = new ModifiableSolrParams();+ xpathParams.set("fl", "flag_target_s");+ assertLukeXPath(+ xpathParams,+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='type'][.='string']",+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='dynamicBase'][.='*_s']",+ "//lst[@name='fields']/lst[@name='flag_target_s']/str[@name='index']",+ "//lst[@name='fields']/lst[@name='flag_target_s']/long[@name='docs'][.='1']");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedShardError() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("id", "0");+ params.set("show", "schema");++ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));+ String fullMessage = SolrException.getRootCause(ex).getMessage();+ assertTrue(+ "exception should mention doc style mismatch: " + fullMessage,+ fullMessage.contains("missing doc param for doc style"));+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedDocIdRejected() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("docId", "0");++ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));+ String fullMessage = SolrException.getRootCause(ex).getMessage();+ assertTrue(+ "exception should mention docId not supported: " + fullMessage,+ fullMessage.contains("docId parameter is not supported in distributed mode"));+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedDocLookupFound() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("id", "0");++ assertLukeXPath(+ params,+ "//lst[@name='doc']/int[@name='docId']",+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='type'][.='string']",+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='id']/str[@name='value'][.='0']",+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='type'][.='nametext']",+ "//lst[@name='doc']/lst[@name='lucene']/lst[@name='name']/str[@name='value'][.='name_0']",+ "//lst[@name='doc']/arr[@name='solr']/str[.='0']",+ "//lst[@name='doc']/arr[@name='solr']/str[.='name_0']",+ "//lst[@name='index']",+ "//lst[@name='info']");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedDocLookupNotFound() throws Exception {+ indexTestData();++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("id", "999888777");++ LukeResponse rsp = requestLuke(params);++ NamedList<Object> raw = rsp.getResponse();+ assertNull("doc section should NOT be present for missing ID", raw.get("doc"));++ assertLukeXPath(params, "not(//lst[@name='doc'])");+ }++ @Test+ @ShardsFixed(num = 2)+ public void testDistributedDocLookupDuplicateId() throws Exception {+ String dupId = "99999";++ // Write the same document directly to two shard cores via UpdateHandler,+ // completely bypassing the distributed update processor chain.+ for (int i = 0; i < 2; i++) {+ try (SolrCore core = jettys.get(i).getCoreContainer().getCore("collection1")) {+ SolrInputDocument solrDoc = new SolrInputDocument();+ solrDoc.addField("id", dupId);+ solrDoc.addField("name", "dup_copy_" + i);++ AddUpdateCommand addCmd =+ new AddUpdateCommand(new SolrQueryRequestBase(core, new ModifiableSolrParams()) {});+ addCmd.solrDoc = solrDoc;+ core.getUpdateHandler().addDoc(addCmd);++ CommitUpdateCommand commitCmd =+ new CommitUpdateCommand(+ new SolrQueryRequestBase(core, new ModifiableSolrParams()) {}, false);+ commitCmd.waitSearcher = true;+ core.getUpdateHandler().commit(commitCmd);+ }+ }++ // Distributed Luke doc lookup should detect the corruption+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("id", dupId);++ Exception ex = expectThrows(Exception.class, () -> requestLuke(params));+ String fullMessage = SolrException.getRootCause(ex).getMessage();+ assertTrue(+ "exception should mention duplicate/corrupt index: " + fullMessage,+ fullMessage.contains("found on multiple shards"));+ }++ @Test+ @ShardsFixed(num = 2)+ public void testShardsParamRoutesToSpecificShard() throws Exception {+ // Index a doc with a dynamic field only to shard 0+ index_specific(0, "id", "700", "name", "shard0_only", "only_on_shard0_s", "present");+ // Index a plain doc to shard 1 (no dynamic field)+ index_specific(1, "id", "701", "name", "shard1_only");+ commit();++ // Query with shards= pointing only at shard 1 — the dynamic field should NOT appear.+ // This also tests that a single remote shard is correctly fanned out to rather than+ // falling through to local-mode on the coordinating node.+ LukeRequest req = new LukeRequest(params("shards", shardsArr[1]));+ req.setNumTerms(0);+ LukeResponse rsp = req.process(controlClient);++ Map<String, LukeResponse.FieldInfo> fields = rsp.getFieldInfo();+ assertNotNull("fields should be present", fields);+ assertNull(+ "only_on_shard0_s should NOT be present when querying only shard 1",+ fields.get("only_on_shard0_s"));+ assertNotNull("'name' field should still be present", fields.get("name"));++ // Now query with shards= pointing only at shard 0 — the dynamic field SHOULD appear+ req = new LukeRequest(params("shards", shardsArr[0]));+ req.setNumTerms(0);+ rsp = req.process(controlClient);++ fields = rsp.getFieldInfo();+ assertNotNull("fields should be present", fields);+ assertNotNull(+ "only_on_shard0_s SHOULD be present when querying shard 0", fields.get("only_on_shard0_s"));+ }++ @Test+ @ShardsFixed(num = 1)+ public void testSingleShardViaParamStillDistributes() throws Exception {+ index("id", "500", "name", "test_name");+ commit();++ // Pass the shards param with a single shard — should still fan out to it+ // rather than incorrectly falling through to local mode+ LukeRequest req = new LukeRequest(params("shards", shards, "shards.info", "true"));+ req.setNumTerms(0);+ LukeResponse rsp = req.process(controlClient);++ assertNotNull("index info should be present", rsp.getIndexInfo());+ assertEquals("should see the 1 doc we indexed", 1, (long) rsp.getNumDocs());+ assertNotNull(+ "shards section should be present when shards.info=true", rsp.getShardResponses());+ assertEquals("should have 1 shard entry", 1, rsp.getShardResponses().size());++ // Without shards.info, shards section should be absent+ req = new LukeRequest(params("shards", shards));+ req.setNumTerms(0);+ rsp = req.process(controlClient);+ assertNotNull("index info should be present", rsp.getIndexInfo());+ assertEquals("should see the 1 doc we indexed", 1, (long) rsp.getNumDocs());+ assertNull("shards section should be absent without shards.info", rsp.getShardResponses());+ }+}diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adocindex fb795f62cc1..42180716544 100644--- a/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/luke-request-handler.adoc@@ -83,6 +83,18 @@ The number of top terms for each field.Choose whether `/luke` should return the index-flags for each field.Fetching and returning the index-flags for each field in the index has non-zero cost, and can slow down requests to `/luke`.+`distrib`::+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++When set to `true` in SolrCloud mode, the handler aggregates results from all shards in the collection.+Additive index metrics (`numDocs`, `deletedDocs`, `segmentCount`) are summed across shards; `maxDoc` is the maximum across shards.+Field types and schema flags are validated for consistency across shards.+Per-shard index details and per-field detailed statistics are returned under a `shards` key.+== LukeRequestHandler ExamplesAll of the examples in this section assume you are running the "techproducts" Solr example:@@ -118,3 +130,43 @@ Alternatively, to work through the Lucene native id:http://localhost:8983/solr/techproducts/admin/luke?fl=manu&docId=0From SolrJ, you can access /luke using the {solr-javadocs}/solrj/org/apache/solr/client/solrj/request/LukeRequest.html[`LukeRequest`] object.++== Distributed Mode (multiple shards)++When running in SolrCloud, the Luke handler automatically distributes requests across all shards in the collection, the same as search requests.+To inspect only the receiving shard's index set `distrib=false`.+In user-managed clusters, you can distribute across shards by passing the `shards` parameter with explicit shard URLs.++To get a collection-wide view:++[source,text]+http://localhost:8983/solr/techproducts/admin/luke++To get detailed field statistics across all shards for a specific field:++[source,text]+http://localhost:8983/solr/techproducts/admin/luke?fl=manu++=== Response Structure++In distributed mode, the response contains:++* `index` -- Aggregated metrics across all shards: `numDocs`, `deletedDocs`, `segmentCount` are summed; `maxDoc` is the maximum across shards.+* `fields` -- Aggregated field metadata. For each field: `type`, `schema` flags, and `dynamicBase` are validated to be consistent across shards; `index` flags use the first non-null value. The `docs` count is summed. Per-field detailed statistics (`topTerms`, `distinct`, `histogram`) are _not_ included at this level.+* `doc` -- Present when `id` is specified. Contains the document from whichever shard owns it, including a `lucene` section (per-field analysis with shard-local `docFreq` values) and a `solr` section (stored fields). Only `id` is supported for distributed doc lookup; `docId` is rejected because Lucene document IDs are shard-local.+* `schema` -- Schema information from the first responding shard (identical across shards sharing the same configset).+* `info` -- Static info from the first responding shard.+* `shards` -- Only present when `shards.info=true`. Contains per-shard details, with each entry keyed by shard address:+** `index` -- Full index info for that shard (including `directory`, `segmentsFile`, `version`, `current`, `hasDeletions`, `lastModified`, `userData`).+** `fields` -- Only present when `fl` triggers detailed statistics. Contains per-field `topTerms`, `distinct`, and `histogram` from that shard.++=== Aggregation Semantics++Field `type`, `schema` flags, and `dynamicBase` are validated for consistency across shards.+If a mismatch is detected, the handler returns an error identifying the field, the conflicting values, and the shard addresses involved.+You can use `distrib=false` to query individual shards and compare their field configurations when troubleshooting mismatches.+The `index` flags are index-derived (not schema-derived) and may be absent on shards where the field has no indexed data; the first non-null value is used, and any subsequent non-null values are validated for consistency.++Per-field detailed statistics (`topTerms`, `distinct`, `histogram`) are not aggregated across shards.+These statistics are shard-local and appear in each shard's entry under the `shards` key (requires `shards.info=true`).+For collection-wide term frequencies or cardinality estimates, Solr's xref:query-guide:faceting.adoc[faceting API] may cover some of these use cases.diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.javaindex 243d811e5d7..2680992bb7e 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/LukeRequest.java@@ -33,12 +33,18 @@ public class LukeRequest extends CollectionRequiringSolrRequest<LukeResponse> {private int numTerms = -1;private boolean showSchema = false;private Boolean includeIndexFieldFlags = null;+ private SolrParams extraParams;public LukeRequest() {// this request is not processed as an ADMIN requestsuper(METHOD.GET, "/admin/luke", SolrRequestType.ADMIN);}+ public LukeRequest(SolrParams params) {+ this();+ this.extraParams = params;+ }+public LukeRequest(String path) {super(METHOD.GET, path, SolrRequestType.ADMIN);}@@ -122,6 +128,9 @@ public class LukeRequest extends CollectionRequiringSolrRequest<LukeResponse> {if (includeIndexFieldFlags != null) {params.add("includeIndexFieldFlags", includeIndexFieldFlags.toString());}+ if (extraParams != null) {+ params.add(extraParams);+ }return params;}diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.javaindex 265a40dfb3e..f23bb29cffa 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/LukeResponse.java@@ -20,10 +20,13 @@ import java.io.Serializable;import java.util.ArrayList;import java.util.EnumSet;import java.util.HashMap;+import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.TreeMap;import org.apache.solr.common.luke.FieldFlag;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SimpleOrderedMap;/*** This is an incomplete representation of the data returned from Luke@@ -113,11 +116,12 @@ public class LukeResponse extends SolrResponseBase {String name;String type;String schema;- int docs;+ long docs;int distinct;EnumSet<FieldFlag> flags;boolean cacheableFaceting;NamedList<Integer> topTerms;+ Map<String, Object> extras = new TreeMap<>();public FieldInfo(String n) {name = n;@@ -128,19 +132,20 @@ public class LukeResponse extends SolrResponseBase {for (Map.Entry<String, Object> entry : nl) {if ("type".equals(entry.getKey())) {type = (String) entry.getValue();- }- if ("flags".equals(entry.getKey())) {+ } else if ("flags".equals(entry.getKey())) {flags = parseFlags((String) entry.getValue());} else if ("schema".equals(entry.getKey())) {schema = (String) entry.getValue();} else if ("docs".equals(entry.getKey())) {- docs = (Integer) entry.getValue();+ docs = ((Number) entry.getValue()).longValue();} else if ("distinct".equals(entry.getKey())) {distinct = (Integer) entry.getValue();} else if ("cacheableFaceting".equals(entry.getKey())) {cacheableFaceting = (Boolean) entry.getValue();} else if ("topTerms".equals(entry.getKey())) {topTerms = (NamedList<Integer>) entry.getValue();+ } else {+ extras.put(entry.getKey(), entry.getValue());}}}@@ -173,7 +178,7 @@ public class LukeResponse extends SolrResponseBase {return distinct;}- public int getDocs() {+ public long getDocs() {return docs;}@@ -192,12 +197,17 @@ public class LukeResponse extends SolrResponseBase {public NamedList<Integer> getTopTerms() {return topTerms;}++ public Map<String, Object> getExtras() {+ return extras;+ }}private NamedList<Object> indexInfo;private Map<String, FieldInfo> fieldInfo;private Map<String, FieldInfo> dynamicFieldInfo;private Map<String, FieldTypeInfo> fieldTypeInfo;+ private Map<String, LukeResponse> shardResponses;@Override@SuppressWarnings("unchecked")@@ -246,6 +256,18 @@ public class LukeResponse extends SolrResponseBase {}}}++ // Parse shards section (present in distributed responses)+ SimpleOrderedMap<NamedList<Object>> shardsNL =+ (SimpleOrderedMap<NamedList<Object>>) res.get("shards");+ if (shardsNL != null) {+ shardResponses = new LinkedHashMap<>();+ for (Map.Entry<String, NamedList<Object>> entry : shardsNL) {+ LukeResponse shardRsp = new LukeResponse();+ shardRsp.setResponse(entry.getValue());+ shardResponses.put(entry.getKey(), shardRsp);+ }+ }}// ----------------------------------------------------------------@@ -256,9 +278,14 @@ public class LukeResponse extends SolrResponseBase {return (String) indexInfo.get("directory");}- public Integer getNumDocs() {+ private Long getLong(String key) {if (indexInfo == null) return null;- return (Integer) indexInfo.get("numDocs");+ Number n = (Number) indexInfo.get(key);+ return n != null ? n.longValue() : null;+ }++ public Long getNumDocs() {+ return getLong("numDocs");}public Integer getMaxDoc() {@@ -266,6 +293,10 @@ public class LukeResponse extends SolrResponseBase {return (Integer) indexInfo.get("maxDoc");}+ public Long getDeletedDocs() {+ return getLong("deletedDocs");+ }+public Integer getNumTerms() {if (indexInfo == null) return null;return (Integer) indexInfo.get("numTerms");@@ -299,5 +330,9 @@ public class LukeResponse extends SolrResponseBase {return dynamicFieldInfo.get(f);}+ public Map<String, LukeResponse> getShardResponses() {+ return shardResponses;+ }+// ----------------------------------------------------------------}