Apache Solr open-source search software
SOLR-15986: CommitUpdateCommand and SplitIndexCommand can write user commit metadata.
Bruno Roustant7561f5ff99f1
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 87b94dc7b79..b519bad345e 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -35,7 +35,7 @@ New FeaturesImprovements----------------------(No changes)+* SOLR-15986: CommitUpdateCommand and SplitIndexCommand can write user commit metadata. (Bruno Roustant)Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.javaindex 8edc6d2dcd3..6ba14ecc179 100644--- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java+++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java@@ -18,8 +18,10 @@ package org.apache.solr.update;import org.apache.solr.request.SolrQueryRequest;+import java.util.Map;+/**- *+ * A commit index command encapsulated in an object.*/public class CommitUpdateCommand extends UpdateCommand {public boolean optimize;@@ -28,6 +30,8 @@ public class CommitUpdateCommand extends UpdateCommand {public boolean expungeDeletes = false;public boolean softCommit = false;public boolean prepareCommit = false;+ /** User provided commit data. Can be let to null if there is none. */+ public Map<String, String> commitData;/*** During optimize, optimize down to <= this many segments. Must be >= 1@@ -48,12 +52,16 @@ public class CommitUpdateCommand extends UpdateCommand {@Overridepublic String toString() {- return super.toString() + ",optimize="+optimize- +",openSearcher="+openSearcher- +",waitSearcher="+waitSearcher- +",expungeDeletes="+expungeDeletes- +",softCommit="+softCommit- +",prepareCommit="+prepareCommit- +'}';+ StringBuilder sb = new StringBuilder(super.toString())+ .append(",optimize=").append(optimize)+ .append(",openSearcher=").append(openSearcher)+ .append(",expungeDeletes=").append(expungeDeletes)+ .append(",softCommit=").append(softCommit)+ .append(",prepareCommit=").append(prepareCommit);+ if (commitData != null) {+ sb.append(",commitData=").append(commitData);+ }+ sb.append('}');+ return sb.toString();}}diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.javaindex 7b1dd1724fd..b68b6263488 100644--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java@@ -580,7 +580,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreStatelog.debug("start {}", cmd);RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);try {- SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion());+ SolrIndexWriter.setCommitData(iw.get(), cmd.getVersion(), cmd.commitData);iw.get().prepareCommit();} finally {iw.decref();@@ -657,7 +657,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState// SolrCore.verbose("writer.commit() start writer=",writer);if (writer.hasUncommittedChanges()) {- SolrIndexWriter.setCommitData(writer, cmd.getVersion());+ SolrIndexWriter.setCommitData(writer, cmd.getVersion(), cmd.commitData);writer.commit();} else {log.debug("No uncommitted changes. Skipping IW.commit.");@@ -859,7 +859,7 @@ public class DirectUpdateHandler2 extends UpdateHandler implements SolrCoreState}// todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't be used)- SolrIndexWriter.setCommitData(writer, cmd.getVersion());+ SolrIndexWriter.setCommitData(writer, cmd.getVersion(), null);writer.commit();synchronized (solrCoreState.getUpdateLock()) {diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.javaindex ea8bc1ce34c..e1ed501a27b 100644--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java@@ -103,42 +103,39 @@ public class SolrIndexSplitter {}}- SolrIndexSearcher searcher;- SchemaField field;- List<DocRouter.Range> ranges;- DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra speed in inner loops- List<String> paths;- List<SolrCore> cores;- HashBasedRouter hashRouter;- int numPieces;- String routeFieldName;- String splitKey;+ final SplitIndexCommand cmd;+ final SolrIndexSearcher searcher;+ final SchemaField field;+ final DocRouter.Range[] rangesArr; // same as ranges list, but an array for extra speed in inner loops+ final HashBasedRouter hashRouter;+ final int numPieces;+ final String splitKey;SplitMethod splitMethod;- RTimerTree timings = new RTimerTree();+ final RTimerTree timings = new RTimerTree();public SolrIndexSplitter(SplitIndexCommand cmd) {+ this.cmd = cmd;searcher = cmd.getReq().getSearcher();- ranges = cmd.ranges;- paths = cmd.paths;- cores = cmd.cores;hashRouter = cmd.router instanceof HashBasedRouter ? (HashBasedRouter)cmd.router : null;- if (ranges == null) {- numPieces = paths != null ? paths.size() : cores.size();+ if (cmd.ranges == null) {+ numPieces = cmd.paths != null ? cmd.paths.size() : cmd.cores.size();+ rangesArr = null;} else {- numPieces = ranges.size();- rangesArr = ranges.toArray(new DocRouter.Range[ranges.size()]);+ numPieces = cmd.ranges.size();+ rangesArr = cmd.ranges.toArray(new DocRouter.Range[0]);}- routeFieldName = cmd.routeFieldName;- if (routeFieldName == null) {+ if (cmd.routeFieldName == null) {field = searcher.getSchema().getUniqueKeyField();} else {- field = searcher.getSchema().getField(routeFieldName);+ field = searcher.getSchema().getField(cmd.routeFieldName);}- if (cmd.splitKey != null) {+ if (cmd.splitKey == null) {+ splitKey = null;+ } else {splitKey = getRouteKey(cmd.splitKey);}- if (cores == null) {+ if (cmd.cores == null) {this.splitMethod = SplitMethod.REWRITE;} else {this.splitMethod = cmd.splitMethod;@@ -239,20 +236,20 @@ public class SolrIndexSplitter {// - need to worry about if IW.addIndexes does a sync or not...// - would be more efficient on the read side, but prob less efficient mergingfor (int partitionNumber=0; partitionNumber<numPieces; partitionNumber++) {- String partitionName = "SolrIndexSplitter:partition=" + partitionNumber + ",partitionCount=" + numPieces + (ranges != null ? ",range=" + ranges.get(partitionNumber) : "");+ String partitionName = "SolrIndexSplitter:partition=" + partitionNumber + ",partitionCount=" + numPieces + (cmd.ranges != null ? ",range=" + cmd.ranges.get(partitionNumber) : "");log.info(partitionName);boolean success = false;RefCounted<IndexWriter> iwRef = null;IndexWriter iw;- if (cores != null && splitMethod != SplitMethod.LINK) {- SolrCore subCore = cores.get(partitionNumber);+ if (cmd.cores != null && splitMethod != SplitMethod.LINK) {+ SolrCore subCore = cmd.cores.get(partitionNumber);iwRef = subCore.getUpdateHandler().getSolrCoreState().getIndexWriter(subCore);iw = iwRef.get();} else {if (splitMethod == SplitMethod.LINK) {- SolrCore subCore = cores.get(partitionNumber);+ SolrCore subCore = cmd.cores.get(partitionNumber);String path = subCore.getDataDir() + INDEX_PREFIX + timestamp;t = timings.sub("hardLinkCopy");t.resume();@@ -288,7 +285,7 @@ public class SolrIndexSplitter {t.pause();} else {SolrCore core = searcher.getCore();- String path = paths.get(partitionNumber);+ String path = cmd.paths.get(partitionNumber);t = timings.sub("createSubIW");t.resume();iw = SolrIndexWriter.create(core, partitionName, path,@@ -313,7 +310,7 @@ public class SolrIndexSplitter {for (int segmentNumber = 0; segmentNumber<leaves.size(); segmentNumber++) {if (log.isInfoEnabled()) {log.info("SolrIndexSplitter: partition # {} partitionCount={} {} segment #={} segmentCount={}", partitionNumber, numPieces- , (ranges != null ? " range=" + ranges.get(partitionNumber) : ""), segmentNumber, leaves.size()); // nowarn+ , (cmd.ranges != null ? " range=" + cmd.ranges.get(partitionNumber) : ""), segmentNumber, leaves.size()); // nowarn}CodecReader subReader = SlowCodecReaderWrapper.wrap(leaves.get(segmentNumber).reader());iw.addIndexes(new LiveDocsReader(subReader, segmentDocSets.get(segmentNumber)[partitionNumber]));@@ -324,7 +321,7 @@ public class SolrIndexSplitter {// because the sub-shard cores will just ignore such a commit because the update log is not// in active state at this time.//TODO no commitUpdateCommand- SolrIndexWriter.setCommitData(iw, -1);+ SolrIndexWriter.setCommitData(iw, -1, cmd.commitData);t = timings.sub("subIWCommit");t.resume();iw.commit();@@ -343,7 +340,7 @@ public class SolrIndexSplitter {IOUtils.closeWhileHandlingException(iw);}if (splitMethod == SplitMethod.LINK) {- SolrCore subCore = cores.get(partitionNumber);+ SolrCore subCore = cmd.cores.get(partitionNumber);subCore.getDirectoryFactory().release(iw.getDirectory());}}@@ -351,11 +348,11 @@ public class SolrIndexSplitter {}// all sub-indexes created ok// when using hard-linking switch directories & refresh cores- if (splitMethod == SplitMethod.LINK && cores != null) {+ if (splitMethod == SplitMethod.LINK && cmd.cores != null) {boolean switchOk = true;t = timings.sub("switchSubIndexes");for (int partitionNumber = 0; partitionNumber < numPieces; partitionNumber++) {- SolrCore subCore = cores.get(partitionNumber);+ SolrCore subCore = cmd.cores.get(partitionNumber);String indexDirPath = subCore.getIndexDir();log.debug("Switching directories");@@ -375,7 +372,7 @@ public class SolrIndexSplitter {t = timings.sub("rollbackSubIndexes");// rollback the switchfor (int partitionNumber = 0; partitionNumber < numPieces; partitionNumber++) {- SolrCore subCore = cores.get(partitionNumber);+ SolrCore subCore = cmd.cores.get(partitionNumber);Directory dir = null;try {dir = subCore.getDirectoryFactory().get(subCore.getDataDir(), DirectoryFactory.DirContext.META_DATA,@@ -411,7 +408,7 @@ public class SolrIndexSplitter {// complete the switch - remove original indext = timings.sub("cleanSubIndex");for (int partitionNumber = 0; partitionNumber < numPieces; partitionNumber++) {- SolrCore subCore = cores.get(partitionNumber);+ SolrCore subCore = cmd.cores.get(partitionNumber);String oldIndexPath = subCore.getDataDir() + "index";Directory indexDir = null;try {diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.javaindex aa841f30e0a..b7bd35e5b29 100644--- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java@@ -178,12 +178,12 @@ public class SolrIndexWriter extends IndexWriter {@SuppressForbidden(reason = "Need currentTimeMillis, commit time should be used only for debugging purposes, " +" but currently suspiciously used for replication as well")- public static void setCommitData(IndexWriter iw, long commitCommandVersion) {- log.debug("Calling setCommitData with IW:{} commitCommandVersion:{}", iw, commitCommandVersion);- final Map<String,String> commitData = new HashMap<>();- commitData.put(COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));- commitData.put(COMMIT_COMMAND_VERSION, String.valueOf(commitCommandVersion));- iw.setLiveCommitData(commitData.entrySet());+ public static void setCommitData(IndexWriter iw, long commitCommandVersion, Map<String, String> commitData) {+ log.debug("Calling setCommitData with IW:{} commitCommandVersion:{} commitData:{}", iw, commitCommandVersion, commitData);+ Map<String,String> finalCommitData = commitData == null ? new HashMap<>(4) : new HashMap<>(commitData);+ finalCommitData.put(COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));+ finalCommitData.put(COMMIT_COMMAND_VERSION, String.valueOf(commitCommandVersion));+ iw.setLiveCommitData(finalCommitData.entrySet());}private void setDirectoryFactory(DirectoryFactory factory) {diff --git a/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java b/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.javaindex 56208a8e400..a18b12b3796 100644--- a/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java+++ b/solr/core/src/java/org/apache/solr/update/SplitIndexCommand.java@@ -22,6 +22,7 @@ import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import java.util.List;+import java.util.Map;/*** A split index command encapsulated in an object.@@ -38,6 +39,11 @@ public class SplitIndexCommand extends UpdateCommand {public final String routeFieldName;public final String splitKey;public final SolrIndexSplitter.SplitMethod splitMethod;+ /**+ * User provided commit data. Can be let to null if there is none.+ * This commit data is used when the split command commits.+ */+ public Map<String, String> commitData;public SplitIndexCommand(SolrQueryRequest req, SolrQueryResponse rsp, List<String> paths, List<SolrCore> cores, List<DocRouter.Range> ranges,DocRouter router, String routeFieldName, String splitKey, SolrIndexSplitter.SplitMethod splitMethod) {@@ -71,6 +77,9 @@ public class SplitIndexCommand extends UpdateCommand {sb.append(",split.key=").append(splitKey);}sb.append(",method=").append(splitMethod.toLower());+ if (commitData != null) {+ sb.append(",commitData=").append(commitData);+ }sb.append('}');return sb.toString();}