Apache Solr open-source search software
DocRouter: strengthen abstraction (#1215)
David Smileyd2c4788065bc
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.javaindex 70c45027ae4..4ee59d13ca9 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MigrateCmd.java@@ -53,7 +53,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;import org.apache.solr.handler.component.ShardHandler;-import org.apache.solr.update.SolrIndexSplitter;import org.apache.solr.util.TimeOut;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -253,7 +252,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {SHARD_ID_PROP,sourceSlice.getName(),"routeKey",- SolrIndexSplitter.getRouteKey(splitKey) + "!",+ sourceRouter.getRouteKeyNoSuffix(splitKey) + "!","range",splitRange.toString(),"targetCollection",@@ -283,7 +282,7 @@ public class MigrateCmd implements CollApiCmds.CollectionApiCommand {sourceSlice = sourceCollection.getSlice(sourceSlice.getName());Map<String, RoutingRule> rules = sourceSlice.getRoutingRules();if (rules != null) {- RoutingRule rule = rules.get(SolrIndexSplitter.getRouteKey(splitKey) + "!");+ RoutingRule rule = rules.get(sourceRouter.getRouteKeyNoSuffix(splitKey) + "!");if (rule != null && rule.getRouteRanges().contains(splitRange)) {added = true;break;diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.javaindex 646b19e2dc2..3b66794c5d5 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java@@ -263,8 +263,9 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {DocCollection collection = clusterState.getCollection(collectionName);String sliceName = parentCore.getCoreDescriptor().getCloudDescriptor().getShardId();Slice slice = collection.getSlice(sliceName);- DocRouter router =- collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;+ CompositeIdRouter router =+ (CompositeIdRouter)+ (collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT);DocRouter.Range currentRange = slice.getRange();Object routerObj =@@ -354,7 +355,10 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {* Returns a list of range counts sorted by the range lower bound*/static Collection<RangeCount> getHashHistogram(- SolrIndexSearcher searcher, String prefixField, DocRouter router, DocCollection collection)+ SolrIndexSearcher searcher,+ String prefixField,+ CompositeIdRouter router,+ DocCollection collection)throws IOException {RTimer timer = new RTimer();TreeMap<DocRouter.Range, RangeCount> counts = new TreeMap<>();@@ -374,14 +378,18 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {while ((term = termsEnum.next()) != null) {numPrefixes++;- String termStr = term.utf8ToString();- int firstSep = termStr.indexOf(CompositeIdRouter.SEPARATOR);// truncate to first separator since we don't support multiple levels currently// NOTE: this does not currently work for tri-level composite ids since the number of bits// allocated to the first ID is 16 for a 2 part id and 8 for a 3 part id!- if (firstSep != termStr.length() - 1 && firstSep > 0) {- numTriLevel++;- termStr = termStr.substring(0, firstSep + 1);+ String termStr;+ int routeKeyLen = router.getRouteKeyWithSeparator(term.bytes, term.offset, term.length);+ if (routeKeyLen == 0 || routeKeyLen == term.length) {+ termStr = term.utf8ToString();+ } else {+ int prevLen = term.length;+ term.length = routeKeyLen;+ termStr = term.utf8ToString();+ term.length = prevLen; // restore (Question: must we do this?)}DocRouter.Range range = router.getSearchRangeSingle(termStr, null, collection);@@ -418,7 +426,10 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {* (i.e. the terms are full IDs, not just prefixes)*/static Collection<RangeCount> getHashHistogramFromId(- SolrIndexSearcher searcher, String idField, DocRouter router, DocCollection collection)+ SolrIndexSearcher searcher,+ String idField,+ CompositeIdRouter router,+ DocCollection collection)throws IOException {RTimer timer = new RTimer();@@ -433,9 +444,8 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {int numCollisions = 0;long sumBuckets = 0;- byte sep = (byte) CompositeIdRouter.SEPARATOR.charAt(0);TermsEnum termsEnum = terms.iterator();- BytesRef currPrefix = new BytesRef(); // prefix of the previous "id" term+ BytesRef currPrefix = new BytesRef(); // prefix of the previous "id" term WITH SEPARATORint bucketCount = 0; // count of the number of docs in the current bucket// We're going to iterate over all terms, so do the minimum amount of work per term.@@ -445,7 +455,9 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {BytesRef term = termsEnum.next();// compare to current prefix bucket and see if this new term shares the same prefix- if (term != null && term.length >= currPrefix.length && currPrefix.length > 0) {+ if (term != null && currPrefix.length > 0) {+ // since currPrefix includes the trailing separator, we can assume startsWith is a+ // sufficient testif (StringHelper.startsWith(term, currPrefix)) {bucketCount++; // use 1 since we are dealing with unique idscontinue;@@ -474,25 +486,16 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {// if the current term is null, we ran out of valuesif (term == null) break;- // find the new prefix (if any)-- // resize if needed- if (currPrefix.length < term.length) {- currPrefix.bytes = new byte[term.length + 10];- }-- // Copy the bytes up to and including the separator, and set the length if the separator is- // found. If there was no separator, then length remains 0 and it's the indicator that we have- // no prefix bucket- currPrefix.length = 0;- for (int i = 0; i < term.length; i++) {- byte b = term.bytes[i + term.offset];- currPrefix.bytes[i] = b;- if (b == sep) {- currPrefix.length = i + 1;- bucketCount++;- break;+ // find the new prefix (if any), with trailing separator+ currPrefix.length = router.getRouteKeyWithSeparator(term.bytes, term.offset, term.length);+ if (currPrefix.length > 0) {+ // resize if needed+ if (currPrefix.length > currPrefix.bytes.length) {+ currPrefix.bytes = new byte[currPrefix.length + 10];}+ System.arraycopy(term.bytes, term.offset, currPrefix.bytes, 0, currPrefix.length);++ bucketCount++;}}diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.javaindex a285f79e73f..61ba505ddd5 100644--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java@@ -133,7 +133,8 @@ public class SolrIndexSplitter {if (cmd.splitKey == null) {splitKey = null;} else {- splitKey = getRouteKey(cmd.splitKey);+ checkRouterSupportsSplitKey(hashRouter, cmd.splitKey);+ splitKey = ((CompositeIdRouter) hashRouter).getRouteKeyNoSuffix(cmd.splitKey);}if (cmd.cores == null) {this.splitMethod = SplitMethod.REWRITE;@@ -647,6 +648,7 @@ public class SolrIndexSplitter {AtomicInteger currentPartition,boolean delete)throws IOException {+ checkRouterSupportsSplitKey(hashRouter, splitKey);LeafReader reader = readerContext.reader();FixedBitSet[] docSets = new FixedBitSet[numPieces];for (int i = 0; i < docSets.length; i++) {@@ -689,8 +691,7 @@ public class SolrIndexSplitter {String idString = idRef.toString();if (splitKey != null) {- // todo have composite routers support these kind of things instead- String part1 = getRouteKey(idString);+ String part1 = ((CompositeIdRouter) hashRouter).getRouteKeyNoSuffix(idString);if (part1 == null) continue;if (!splitKey.equals(part1)) {continue;@@ -765,18 +766,11 @@ public class SolrIndexSplitter {return docSets;}- public static String getRouteKey(String idString) {- int idx = idString.indexOf(CompositeIdRouter.SEPARATOR);- if (idx <= 0) return null;- String part1 = idString.substring(0, idx);- int commaIdx = part1.indexOf(CompositeIdRouter.bitsSeparator);- if (commaIdx > 0 && commaIdx + 1 < part1.length()) {- char ch = part1.charAt(commaIdx + 1);- if (ch >= '0' && ch <= '9') {- part1 = part1.substring(0, commaIdx);- }+ private static void checkRouterSupportsSplitKey(HashBasedRouter hashRouter, String splitKey) {+ if (splitKey != null && !(hashRouter instanceof CompositeIdRouter)) {+ throw new IllegalStateException(+ "splitKey isn't supported for router " + hashRouter.getClass());}- return part1;}// change livedocs on the reader to delete those docs we don't wantdiff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.javaindex fbe89388192..c1eee8572c2 100644--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java@@ -67,7 +67,6 @@ import org.apache.solr.update.DeleteUpdateCommand;import org.apache.solr.update.MergeIndexesCommand;import org.apache.solr.update.RollbackUpdateCommand;import org.apache.solr.update.SolrCmdDistributor;-import org.apache.solr.update.SolrIndexSplitter;import org.apache.solr.update.UpdateCommand;import org.apache.solr.util.TestInjection;import org.apache.zookeeper.KeeperException;@@ -1067,7 +1066,7 @@ public class DistributedZkUpdateProcessor extends DistributedUpdateProcessor {return nodes;}- String routeKey = SolrIndexSplitter.getRouteKey(id);+ String routeKey = compositeIdRouter.getRouteKeyNoSuffix(id);if (routeKey != null) {RoutingRule rule = routingRules.get(routeKey + "!");if (rule != null) {diff --git a/solr/core/src/test/org/apache/solr/handler/admin/SplitHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/SplitHandlerTest.javaindex f3f1f69322f..011f5fe3885 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/SplitHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/SplitHandlerTest.java@@ -239,7 +239,7 @@ public class SplitHandlerTest extends SolrTestCaseJ4 {String prefixField = "id_prefix_s";String idField = "id";- DocRouter router = new CompositeIdRouter();+ CompositeIdRouter router = new CompositeIdRouter();for (int i = 0; i < 100; i++) {SolrQueryRequest req = req("myquery");diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java b/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.javaindex dd0b6669195..9094ddd84e8 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/CompositeIdRouter.java@@ -81,12 +81,57 @@ import org.apache.solr.common.util.Hash;public class CompositeIdRouter extends HashBasedRouter {public static final String NAME = "compositeId";- public static final String SEPARATOR = "!";+ // TODO standardize naming: routeKey (probably) or shardKey or key; pick one.++ /**+ * This character separates a composite ID into a leading route key and the rest.+ *+ * <p>Importantly, it's also used at the end of a provided route key parameter (which appears in+ * many places) to designate a hash range which translates to a list of slices. If a route key+ * does not end with this character, then semantically the key points to a single slice that holds+ * a doc with that ID.+ */+ public static final char SEPARATOR = '!';// separator used to optionally specify number of bits to allocate toward first part.public static final int bitsSeparator = '/';private int bits = 16;+ /**+ * Parse out the route key from {@code id} up to and including the {@link #SEPARATOR}, returning+ * it's length. If no route key is detected then 0 is returned.+ */+ public int getRouteKeyWithSeparator(byte[] id, int idOffset, int idLength) {+ final byte SEPARATOR_BYTE = (byte) CompositeIdRouter.SEPARATOR;+ for (int i = 0; i < idLength; i++) {+ byte b = id[idOffset + i];+ if (b == SEPARATOR_BYTE) {+ return i + 1;+ }+ }+ return 0;+ }++ /**+ * Parse out the route key from {@code id}. It will not have the "bits" suffix or separator, if+ * present. If there is no route key found then null is returned.+ */+ public String getRouteKeyNoSuffix(String id) {+ int idx = id.indexOf(SEPARATOR);+ if (idx <= 0) {+ return null;+ }+ String part1 = id.substring(0, idx);+ int commaIdx = part1.indexOf(bitsSeparator);+ if (commaIdx > 0 && commaIdx + 1 < part1.length()) {+ char ch = part1.charAt(commaIdx + 1);+ if (ch >= '0' && ch <= '9') {+ part1 = part1.substring(0, commaIdx);+ }+ }+ return part1;+ }+@Overridepublic int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) {@@ -132,6 +177,8 @@ public class CompositeIdRouter extends HashBasedRouter {* @return Range for given routeKey*/public Range keyHashRange(String routeKey) {+ routeKey = preprocessRouteKey(routeKey);+if (routeKey.indexOf(SEPARATOR) < 0) {int hash = sliceHash(routeKey, null, null, null);return new Range(hash, hash);@@ -147,6 +194,8 @@ public class CompositeIdRouter extends HashBasedRouter {return fullRange();}+ shardKey = preprocessRouteKey(shardKey);+if (shardKey.indexOf(SEPARATOR) < 0) {// shardKey is a simple id, so don't do a rangeint hash = Hash.murmurhash3_x86_32(shardKey, 0, shardKey.length(), 0);@@ -164,7 +213,8 @@ public class CompositeIdRouter extends HashBasedRouter {// TODO: this may need modification in the future when shard splitting could cause an overlapreturn collection.getActiveSlices();}- String id = shardKey;++ String id = preprocessRouteKey(shardKey);if (shardKey.indexOf(SEPARATOR) < 0) {// shardKey is a simple id, so don't do a range@@ -185,6 +235,14 @@ public class CompositeIdRouter extends HashBasedRouter {return targetSlices;}+ /**+ * Methods accepting a route key (shard key) can have this input preprocessed by a subclass before+ * further analysis.+ */+ protected String preprocessRouteKey(String shardKey) {+ return shardKey;+ }+@Overridepublic String getName() {return NAME;@@ -266,7 +324,7 @@ public class CompositeIdRouter extends HashBasedRouter {}/** Helper class to calculate parts, masks etc for an id. */- static class KeyParser {+ protected static class KeyParser {String key;int[] numBits;int[] hashes;@@ -333,7 +391,7 @@ public class CompositeIdRouter extends HashBasedRouter {masks = getMasks();}- Range getRange() {+ public Range getRange() {int lowerBound;int upperBound;@@ -395,7 +453,7 @@ public class CompositeIdRouter extends HashBasedRouter {return masks;}- int getHash() {+ public int getHash() {int result = hashes[0] & masks[0];for (int i = 1; i < pieces; i++) result = result | (hashes[i] & masks[i]);diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.javaindex af0ebc66f12..490a0a21eda 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java@@ -39,7 +39,7 @@ import org.noggit.JSONWriter;*/public abstract class DocRouter {public static final String DEFAULT_NAME = CompositeIdRouter.NAME;- public static final DocRouter DEFAULT = new CompositeIdRouter();+ public static final DocRouter DEFAULT;public static DocRouter getDocRouter(String routerName) {DocRouter router = routerMap.get(routerName);@@ -79,11 +79,11 @@ public abstract class DocRouter {// to "plain" if it doesn't have any properties.routerMap.put(null, plain); // back compat with 4.0routerMap.put(PlainIdRouter.NAME, plain);- routerMap.put(- CompositeIdRouter.NAME,- DEFAULT_NAME.equals(CompositeIdRouter.NAME) ? DEFAULT : new CompositeIdRouter());+ routerMap.put(CompositeIdRouter.NAME, new CompositeIdRouter());routerMap.put(ImplicitDocRouter.NAME, new ImplicitDocRouter());// NOTE: careful that the map keys (the static .NAME members) are filled in by making them final++ DEFAULT = routerMap.get(DEFAULT_NAME);}// Hash ranges can't currently "wrap" - i.e. max must be greater or equal to min.