Apache Solr open-source search software
SOLR-14444: Ref Guide re-organization (#190)
Cassandra Targett73bb5654c525
diff --git a/solr/solr-ref-guide/build.gradle b/solr/solr-ref-guide/build.gradleindex 00b0e983553..67f19c67c19 100644--- a/solr/solr-ref-guide/build.gradle+++ b/solr/solr-ref-guide/build.gradle@@ -22,7 +22,7 @@ buildscript {}dependencies {- classpath "org.asciidoctor:asciidoctorj:1.6.2"+ classpath "org.asciidoctor:asciidoctorj:2.5.1"}}@@ -68,14 +68,14 @@ dependencies {depVer('org.apache.zookeeper:zookeeper')// jekyll dependencies- gems 'rubygems:jekyll:3.5.2'+ gems 'rubygems:jekyll:4.2.0'gems 'rubygems:jekyll-asciidoc:3.0.0'// don't know why we have to explicitly add these deps but it doesn't resolve them// automatically.gems 'rubygems:tilt:2.0.10'- gems 'rubygems:slim:4.0.1'- gems 'rubygems:concurrent-ruby:1.0.5'+ gems 'rubygems:slim:4.1.0'+ gems 'rubygems:concurrent-ruby:1.1.9'}sourceSets {@@ -156,6 +156,7 @@ ext {def dependencyProps = [["ivyCommonsCodec", "commons-codec", "commons-codec"],["ivyDropwizardMetrics", "io.dropwizard.metrics", "metrics-core"],+ ["ivyHadoop", "org.apache.hadoop", "hadoop-auth"],["ivyLog4j", "org.apache.logging.log4j", "log4j-core"],["ivyOpennlpTools", "org.apache.opennlp", "opennlp-tools"],["ivyTika", "org.apache.tika", "tika-core"],diff --git a/solr/solr-ref-guide/src/_config.yml.template b/solr/solr-ref-guide/src/_config.yml.templateindex 88016fe140d..45c80d22003 100755--- a/solr/solr-ref-guide/src/_config.yml.template+++ b/solr/solr-ref-guide/src/_config.yml.template@@ -79,6 +79,7 @@ solr-attributes: &solr-attributes-refbuild-year: '${buildYear}'ivy-commons-codec-version: '${ivyCommonsCodec}'ivy-dropwizard-version: '${ivyDropwizardMetrics}'+ ivy-hadoop-version: '${ivyHadoop}'ivy-log4j-version: '${ivyLog4j}'ivy-opennlp-version: '${ivyOpennlpTools}'ivy-tika-version: '${ivyTika}'@@ -92,5 +93,7 @@ asciidoctor:attribute-missing: "warn"icons: "font"source-highlighter: "rouge"- rouge-theme: "thankful-eyes"+ rouge-css: "style"+ rouge-style: "github"stem:+ sectanchors: "true"diff --git a/solr/solr-ref-guide/src/_includes/head.html b/solr/solr-ref-guide/src/_includes/head.htmlindex 023328b27a3..c92db6dbb3f 100755--- a/solr/solr-ref-guide/src/_includes/head.html+++ b/solr/solr-ref-guide/src/_includes/head.html@@ -22,8 +22,6 @@<script src="{{ "js/jquery.navgoco.min.js" }}"></script>-<script src="https://cdnjs.cloudflare.com/ajax/libs/anchor-js/4.2.0/anchor.min.js" crossorigin="anonymous"></script>-<link rel="shortcut icon" href="{{ "images/icons/favicon.ico" }}"><!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->diff --git a/solr/solr-ref-guide/src/_includes/topnav.html b/solr/solr-ref-guide/src/_includes/topnav.htmlindex fa868a0748f..d1147288d5a 100755--- a/solr/solr-ref-guide/src/_includes/topnav.html+++ b/solr/solr-ref-guide/src/_includes/topnav.html@@ -11,22 +11,24 @@<div class="collapse navbar-collapse justify-content-end" id="bs-example-navbar-collapse-1"><ul class="nav navbar-nav navbar-right"><!-- Link to Solr website -->- <li><a href="https://solr.apache.org/" target="_blank">Solr Website</a></li>+ <li><a href="https://solr.apache.org/" target="_blank" rel="noreferrer nopener">Solr Website</a></li>+ <li><a href="https://solr.apache.org/downloads.html" target="_blank" rel="noreferrer nopener">Download Solr</a></li><!-- Other Guide Formats dropdown --><li class="dropdown">- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Other Formats<b class="caret"></b></a>+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Other Versions<b class="caret"></b></a><ul class="dropdown-menu">- <li><a href="https://archive.apache.org/dist/lucene/solr/ref-guide/" target="_blank">Archived PDFs</a></li>- <li><a href="https://solr.apache.org/guide/" target="_blank">Other Versions Online</a></li>+ <li><a href="https://archive.apache.org/dist/lucene/solr/ref-guide/" target="_blank" rel="noreferrer nopener">Archived PDFs</a></li>+ <li><a href="https://solr.apache.org/guide/" target="_blank" rel="noreferrer nopener">Other Versions Online</a></li></ul></li><!-- Solr Resources dropdown --><li class="dropdown"><a href="#" class="dropdown-toggle" data-toggle="dropdown">Solr Resources<b class="caret"></b></a><ul class="dropdown-menu">- <li><a href="{{site.solr-attributes.solr-javadocs}}/solr-core/index.html" target="_blank">Solr Javadocs</a></li>- <li><a href="https://solr.apache.org/community.html#version-control" target="_blank">Lucene/Solr Source Code</a></li>- <li><a href="https://solr.apache.org/community.html" target="_blank">Solr Community Links</a></li>+ <li><a href="{{site.solr-attributes.solr-javadocs}}/solr-core/index.html" target="_blank" rel="noreferrer nopener">Javadocs</a></li>+ <li><a href="https://solr.apache.org/community.html#version-control" target="_blank" rel="noreferrer nopener">Source Code</a></li>+ <li><a href="https://solr.apache.org/community.html" target="_blank" rel="noreferrer nopener">Community Links</a></li>+ <li><a href="https://github.com/apache/solr/tree/main/dev-docs/ref-guide" target="_blank" rel="noreferrer nopener">Contribute</a></li></ul></li>diff --git a/solr/solr-ref-guide/src/_templates/example.html.slim b/solr/solr-ref-guide/src/_templates/example.html.slimdeleted file mode 100644index 1514e63a479..00000000000--- a/solr/solr-ref-guide/src/_templates/example.html.slim+++ /dev/null@@ -1,3 +0,0 @@-= block_with_caption :top, :class=>'exampleblock'- .example- =contentdiff --git a/solr/solr-ref-guide/src/_templates/helpers.rb b/solr/solr-ref-guide/src/_templates/helpers.rbindex 07163db687c..0f4a7c49b18 100644--- a/solr/solr-ref-guide/src/_templates/helpers.rb+++ b/solr/solr-ref-guide/src/_templates/helpers.rb@@ -472,9 +472,9 @@ is book and it's a child of a book part. Excluding block content."option? :autowidthend- def spread?+ def stretch?if !autowidth? || local_attr?('width')- 'spread' if attr? :tablepcwidth, 100+ 'stretch' if attr? :tablepcwidth, 100endenddiff --git a/solr/solr-ref-guide/src/_templates/pass.html.slim b/solr/solr-ref-guide/src/_templates/pass.html.slimdeleted file mode 100644index 7125c22c78d..00000000000--- a/solr/solr-ref-guide/src/_templates/pass.html.slim+++ /dev/null@@ -1 +0,0 @@-=contentdiff --git a/solr/solr-ref-guide/src/_templates/section.html.slim b/solr/solr-ref-guide/src/_templates/section.html.slimdeleted file mode 100644index b8776f5fbff..00000000000--- a/solr/solr-ref-guide/src/_templates/section.html.slim+++ /dev/null@@ -1,13 +0,0 @@-- sect0 = section_level == 0-= html_tag_if !sect0, :section, class: [%(sect#{section_level}), role]- *{tag: %(h#{section_level + 1}), id: id, class: ('sect0' if sect0)}- - if id- - if document.attr? :sectanchors- a.anchor href="##{id}" aria-hidden='true'- - if document.attr? :sectlinks- a.link href="##{id}" =section_title- - else- =section_title- - else- =section_title- =contentdiff --git a/solr/solr-ref-guide/src/_templates/table.html.slim b/solr/solr-ref-guide/src/_templates/table.html.slimindex b66bf5bdb79..7f0f8e46275 100644--- a/solr/solr-ref-guide/src/_templates/table.html.slim+++ b/solr/solr-ref-guide/src/_templates/table.html.slim@@ -1,7 +1,7 @@= block_with_caption :top, :class=>'tableblock'table [- class=["tableblock", "frame-#{attr :frame, 'all'}", "grid-#{attr :grid, 'all'}", spread?]- style=style_value(width: ("#{attr :tablepcwidth}%" if !autowidth? && !spread? || (local_attr :width)),+ class=["tableblock", "frame-#{attr :frame, 'all'}", "grid-#{attr :grid, 'all'}", "stripes-#{attr :stripes, 'even'}", stretch?]+ style=style_value(width: ("#{attr :tablepcwidth}%" if !autowidth? && !stretch? || (local_attr :width)),float: (attr :float)) ]- unless (attr :rowcount).zero?colgroupdiff --git a/solr/solr-ref-guide/src/_templates/thematic_break.html.slim b/solr/solr-ref-guide/src/_templates/thematic_break.html.slimdeleted file mode 100644index fcb5e9c6542..00000000000--- a/solr/solr-ref-guide/src/_templates/thematic_break.html.slim+++ /dev/null@@ -1 +0,0 @@-hrdiff --git a/solr/solr-ref-guide/src/a-quick-overview.adoc b/solr/solr-ref-guide/src/a-quick-overview.adocdeleted file mode 100644index 0542cb36aa0..00000000000--- a/solr/solr-ref-guide/src/a-quick-overview.adoc+++ /dev/null@@ -1,45 +0,0 @@-= A Quick Overview-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr is a search server built on top of Apache Lucene, an open source, Java-based, information retrieval library. It is designed to drive powerful document retrieval applications - wherever you need to serve data to users based on their queries, Solr can work for you.--Here is a example of how Solr could integrate with an application:--.Solr integration with applications-image::images/a-quick-overview/sample-client-app-arch.png[image,width=500,height=379]--In the scenario above, Solr runs alongside other server applications. For example, an online store application would provide a user interface, a shopping cart, and a way to make purchases for end users; while an inventory management application would allow store employees to edit product information. The product metadata would be kept in some kind of database, as well as in Solr.--Solr makes it easy to add the capability to search through the online store through the following steps:--. Define a _schema_. The schema tells Solr about the contents of documents it will be indexing. In the online store example, the schema would define fields for the product name, description, price, manufacturer, and so on. Solr's schema is powerful and flexible and allows you to tailor Solr's behavior to your application. See <<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>> for all the details.-. Feed Solr documents for which your users will search.-. Expose search functionality in your application.--Because Solr is based on open standards, it is highly extensible. Solr queries are simple HTTP request URLs and the response is a structured document: mainly JSON, but it could also be XML, CSV, or other formats. This means that a wide variety of clients will be able to use Solr, from other web applications to browser clients, rich client applications, and mobile devices. Any platform capable of HTTP can talk to Solr. See <<client-apis.adoc#,Client APIs>> for details on client APIs.--Solr offers support for the simplest keyword searching through to complex queries on multiple fields and faceted search results. <<searching.adoc#,Searching>> has more information about searching and queries.--If Solr's capabilities are not impressive enough, its ability to handle very high-volume applications should do the trick.--A relatively common scenario is that you have so much data, or so many queries, that a single Solr server is unable to handle your entire workload. In this case, you can scale up the capabilities of your application using <<solrcloud.adoc#,SolrCloud>> to better distribute the data, and the processing of requests, across many servers. Multiple options can be mixed and matched depending on the scalability you need.--For example: "Sharding" is a scaling technique in which a collection is split into multiple logical pieces called "shards" in order to scale up the number of documents in a collection beyond what could physically fit on a single server. Incoming queries are distributed to every shard in the collection, which respond with merged results. Another technique available is to increase the "Replication Factor" of your collection, which allows you to add servers with additional copies of your collection to handle higher concurrent query load by spreading the requests around to multiple machines. Sharding and replication are not mutually exclusive, and together make Solr an extremely powerful and scalable platform.--Best of all, this talk about high-volume applications is not just hypothetical: some of the famous Internet sites that use Solr today are Macy's, EBay, and Zappo's.-For more examples, take a look at https://cwiki.apache.org/confluence/display/solr/PublicServers.diff --git a/solr/solr-ref-guide/src/about-filters.adoc b/solr/solr-ref-guide/src/about-filters.adocdeleted file mode 100644index 900707e840e..00000000000--- a/solr/solr-ref-guide/src/about-filters.adoc+++ /dev/null@@ -1,63 +0,0 @@-= About Filters-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Like <<tokenizers.adoc#,tokenizers>>, <<filter-descriptions.adoc#,filters>> consume input and produce a stream of tokens. Filters also derive from `org.apache.lucene.analysis.TokenStream`. Unlike tokenizers, a filter's input is another TokenStream. The job of a filter is usually easier than that of a tokenizer since in most cases a filter looks at each token in the stream sequentially and decides whether to pass it along, replace it or discard it.--A filter may also do more complex analysis by looking ahead to consider multiple tokens at once, although this is less common. One hypothetical use for such a filter might be to normalize state names that would be tokenized as two words. For example, the single token "california" would be replaced with "CA", while the token pair "rhode" followed by "island" would become the single token "RI".--Because filters consume one `TokenStream` and produce a new `TokenStream`, they can be chained one after another indefinitely. Each filter in the chain in turn processes the tokens produced by its predecessor. The order in which you specify the filters is therefore significant. Typically, the most general filtering is done first, and later filtering stages are more specialized.--[.dynamic-tabs]----[example.tab-pane#byname-filterexample]-====-[.tab-label]*With name*-[source,xml]------<fieldType name="text" class="solr.TextField">- <analyzer>- <tokenizer name="standard"/>- <filter name="lowercase"/>- <filter name="englishPorter"/>- </analyzer>-</fieldType>------====-[example.tab-pane#byclass-filterexample]-====-[.tab-label]*With class name (legacy)*-[source,xml]------<fieldType name="text" class="solr.TextField">- <analyzer>- <tokenizer class="solr.StandardTokenizerFactory"/>- <filter class="solr.LowerCaseFilterFactory"/>- <filter class="solr.EnglishPorterFilterFactory"/>- </analyzer>-</fieldType>------====-----This example starts with Solr's standard tokenizer, which breaks the field's text into tokens. All the tokens are then set to lowercase, which will facilitate case-insensitive matching at query time.--The last filter in the above example is a stemmer filter that uses the Porter stemming algorithm. A stemmer is basically a set of mapping rules that maps the various forms of a word back to the base, or _stem_, word from which they derive. For example, in English the words "hugs", "hugging" and "hugged" are all forms of the stem word "hug". The stemmer will replace all of these terms with "hug", which is what will be indexed. This means that a query for "hug" will match the term "hugged", but not "huge".--Conversely, applying a stemmer to your query terms will allow queries containing non stem terms, like "hugging", to match documents with different variations of the same stem word, such as "hugged". This works because both the indexer and the query will map to the same stem ("hug").--Word stemming is, obviously, very language specific. Solr includes several language-specific stemmers created by the http://snowball.tartarus.org/[Snowball] generator that are based on the Porter stemming algorithm. The generic Snowball Porter Stemmer Filter can be used to configure any of these language stemmers. Solr also includes a convenience wrapper for the English Snowball stemmer. There are also several purpose-built stemmers for non-English languages. These stemmers are described in <<language-analysis.adoc#,Language Analysis>>.diff --git a/solr/solr-ref-guide/src/about-this-guide.adoc b/solr/solr-ref-guide/src/about-this-guide.adocindex 0eff93763eb..4f6a71af83a 100644--- a/solr/solr-ref-guide/src/about-this-guide.adoc+++ b/solr/solr-ref-guide/src/about-this-guide.adoc@@ -48,7 +48,7 @@ In many cases, this is is in the `server/solr` directory of your installation. HIn several cases of this Guide, our examples are built from the the "techproducts" example (i.e., you have started Solr with the command `bin/solr -e techproducts`). In this case, `solr.home` will be a sub-directory of the `example/` directory created for you automatically.-See also the section <<solr-configuration-files.adoc#solr-home,Solr Home>> for further details on what is contained in this directory.+See also the section <<configuration-files.adoc#solr-home,Solr Home>> for further details on what is contained in this directory.== API Examplesdiff --git a/solr/solr-ref-guide/src/about-tokenizers.adoc b/solr/solr-ref-guide/src/about-tokenizers.adocdeleted file mode 100644index b2d0c3b665d..00000000000--- a/solr/solr-ref-guide/src/about-tokenizers.adoc+++ /dev/null@@ -1,63 +0,0 @@-= About Tokenizers-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The job of a <<tokenizers.adoc#,tokenizer>> is to break up a stream of text into tokens, where each token is (usually) a sub-sequence of the characters in the text. An analyzer is aware of the field it is configured for, but a tokenizer is not. Tokenizers read from a character stream (a Reader) and produce a sequence of Token objects (a TokenStream).--Characters in the input stream may be discarded, such as whitespace or other delimiters. They may also be added to or replaced, such as mapping aliases or abbreviations to normalized forms. A token contains various metadata in addition to its text value, such as the location at which the token occurs in the field. Because a tokenizer may produce tokens that diverge from the input text, you should not assume that the text of the token is the same text that occurs in the field, or that its length is the same as the original text. It's also possible for more than one token to have the same position or refer to the same offset in the original text. Keep this in mind if you use token metadata for things like highlighting search results in the field text.--[.dynamic-tabs]----[example.tab-pane#byname-tok]-====-[.tab-label]*With name*-[source,xml]------<fieldType name="text" class="solr.TextField">- <analyzer>- <tokenizer name="standard"/>- </analyzer>-</fieldType>------====-[example.tab-pane#byclass-tok]-====-[.tab-label]*With class name (legacy)*-[source,xml]------<fieldType name="text" class="solr.TextField">- <analyzer>- <tokenizer class="solr.StandardTokenizerFactory"/>- </analyzer>-</fieldType>------====-----The class named in the tokenizer element is not the actual tokenizer, but rather a class that implements the `TokenizerFactory` API. This factory class will be called upon to create new tokenizer instances as needed. Objects created by the factory must derive from `Tokenizer`, which indicates that they produce sequences of tokens. If the tokenizer produces tokens that are usable as is, it may be the only component of the analyzer. Otherwise, the tokenizer's output tokens will serve as input to the first filter stage in the pipeline.--A `TypeTokenFilterFactory` is available that creates a `TypeTokenFilter` that filters tokens based on their TypeAttribute, which is set in `factory.getStopTypes`.--For a complete list of the available TokenFilters, see the section <<tokenizers.adoc#,Tokenizers>>.--== When to Use a CharFilter vs. a TokenFilter--There are several pairs of CharFilters and TokenFilters that have related (i.e., `MappingCharFilter` and `ASCIIFoldingFilter`) or nearly identical (i.e., `PatternReplaceCharFilterFactory` and `PatternReplaceFilterFactory`) functionality and it may not always be obvious which is the best choice.--The decision about which to use depends largely on which Tokenizer you are using, and whether you need to preprocess the stream of characters.--For example, suppose you have a tokenizer such as `StandardTokenizer` and although you are pretty happy with how it works overall, you want to customize how some specific characters behave. You could modify the rules and re-build your own tokenizer with JFlex, but it might be easier to simply map some of the characters before tokenization with a `CharFilter`.diff --git a/solr/solr-ref-guide/src/collection-aliasing.adoc b/solr/solr-ref-guide/src/alias-management.adocsimilarity index 63%rename from solr/solr-ref-guide/src/collection-aliasing.adocrename to solr/solr-ref-guide/src/alias-management.adocindex c019fc7a3e1..18f56580231 100644--- a/solr/solr-ref-guide/src/collection-aliasing.adoc+++ b/solr/solr-ref-guide/src/alias-management.adoc@@ -1,4 +1,4 @@-= Collection Aliasing+= Alias Management:toclevels: 1// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file@@ -17,69 +17,68 @@// specific language governing permissions and limitations// under the License.-A collection alias is a virtual collection which Solr treats the same as a normal collection. The alias collection may point to one or more real collections.+A collection alias is a virtual collection which Solr treats the same as a normal collection.+The alias collection may point to one or more real collections.Some use cases for collection aliasing:* Time series data* Reindexing content behind the scenes+For an overview of aliases in Solr, see the section <<aliases.adoc#,Aliases>>.+[[createalias]]== CREATEALIAS: Create or Modify an Alias for a CollectionThe `CREATEALIAS` action will create a new alias pointing to one or more collections.Aliases come in 2 flavors: standard and routed.-*Standard aliases* are simple: CREATEALIAS registers the alias name with the names of one or more collections provided- by the command.+*Standard aliases* are simple: `CREATEALIAS` registers the alias name with the names of one or more collections provided by the command.If an existing alias exists, it is replaced/updated.-A standard alias can serve as a means to rename a collection, and can be used to atomically swap-which backing/underlying collection is "live" for various purposes.-When Solr searches an alias pointing to multiple collections, Solr will search all shards of all the collections as an- aggregated whole.-While it is possible to send updates to an alias spanning multiple collections, standard aliases have no logic for- distributing documents among the referenced collections so all updates will go to the first collection in the list.++A standard alias can serve as a means to rename a collection, and can be used to atomically swap which backing/underlying collection is "live" for various purposes.++When Solr searches an alias pointing to multiple collections, Solr will search all shards of all the collections as an aggregated whole.+While it is possible to send updates to an alias spanning multiple collections, standard aliases have no logic for distributing documents among the referenced collections so all updates will go to the first collection in the list.`/admin/collections?action=CREATEALIAS&name=_name_&collections=_collectionlist_`-*Routed aliases* are aliases with additional capabilities to act as a kind of super-collection that route- updates to the correct collection. Routing is data driven and may be based on a temporal field or on categories- specified in a field (normally string based).-See <<aliases.adoc#routed-aliases,Routed Aliases>> for some important high-level information- before getting started.+*Routed aliases* are aliases with additional capabilities to act as a kind of super-collection that route updates to the correct collection.++Routing is data driven and may be based on a temporal field or on categories specified in a field (normally string based).+See <<aliases.adoc#routed-aliases,Routed Aliases>> for some important high-level information before getting started.[source,text]-----localhost:8983/solr/admin/collections?action=CREATEALIAS&name=timedata&router.start=NOW/DAY&router.field=evt_dt&router.name=time&router.interval=%2B1DAY&router.maxFutureMs=3600000&create-collection.collection.configName=myConfig&create-collection.numShards=2+$ http://localhost:8983/solr/admin/collections?action=CREATEALIAS&name=timedata&router.start=NOW/DAY&router.field=evt_dt&router.name=time&router.interval=%2B1DAY&router.maxFutureMs=3600000&create-collection.collection.configName=myConfig&create-collection.numShards=2-----If run on Jan 15, 2018, the above will create an time routed alias named timedata, that contains collections with names prefixed-with `timedata` and an initial collection named `timedata_2018_01_15` will be created immediately. Updates sent to this-alias with a (required) value in `evt_dt` that is before or after 2018-01-15 will be rejected, until the last 60-minutes of 2018-01-15. After 2018-01-15T23:00:00 documents for either 2018-01-15 or 2018-01-16 will be accepted.-As soon as the system receives a document for an allowable time window for which there is no collection it will-automatically create the next required collection (and potentially any intervening collections if `router.interval` is-smaller than `router.maxFutureMs`). Both the initial collection and any subsequent collections will be created using-the specified configset. All collection creation parameters other than `name` are allowed, prefixed+If run on Jan 15, 2018, the above will create an time routed alias named timedata, that contains collections with names prefixed with `timedata` and an initial collection named `timedata_2018_01_15` will be created immediately.+Updates sent to this alias with a (required) value in `evt_dt` that is before or after 2018-01-15 will be rejected, until the last 60 minutes of 2018-01-15.+After 2018-01-15T23:00:00 documents for either 2018-01-15 or 2018-01-16 will be accepted.+As soon as the system receives a document for an allowable time window for which there is no collection it will automatically create the next required collection (and potentially any intervening collections if `router.interval` is+smaller than `router.maxFutureMs`).+Both the initial collection and any subsequent collections will be created using+the specified configset.+All collection creation parameters other than `name` are allowed, prefixedby `create-collection.`-This means that one could, for example, partition their collections by day, and within each daily collection route-the data to shards based on customer id. Such shards can be of any type (NRT, PULL or TLOG), and rule-based replica+This means that one could, for example, partition their collections by day, and within each daily collection route the data to shards based on customer id.+Such shards can be of any type (NRT, PULL or TLOG), and rule-based replicaplacement strategies may also be used.The values supplied in this command for collection creation will be retainedin alias properties, and can be verified by inspecting `aliases.json` in ZooKeeper.-NOTE: Presently only updates are routed and queries are distributed to all collections in the alias, but future-features may enable routing of the query to the single appropriate collection based on a special parameter or perhaps-a filter on the routed field.+NOTE: Only updates are routed and queries are distributed to all collections in the alias.=== CREATEALIAS Parameters`name`::-The alias name to be created. This parameter is required. If the alias is to be routed it also functions-as a prefix for the names of the dependent collections that will be created. It must therefore adhere to normal-requirements for collection naming.+The alias name to be created.+This parameter is required.+If the alias is to be routed it also functions as a prefix for the names of the dependent collections that will be created.+It must therefore adhere to normal requirements for collection naming.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -87,85 +86,84 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou==== Standard Alias Parameters`collections`::-A comma-separated list of collections to be aliased. The collections must already exist in the cluster.-This parameter signals the creation of a standard alias. If it is present all routing parameters are-prohibited. If routing parameters are present this parameter is prohibited.+A comma-separated list of collections to be aliased.+The collections must already exist in the cluster.+This parameter signals the creation of a standard alias.+If it is present all routing parameters are prohibited.+If routing parameters are present this parameter is prohibited.==== Routed Alias ParametersMost routed alias parameters become _alias properties_ that can subsequently be inspected and <<aliasprop,modified>>.`router.name`::-The type of routing to use. Presently only `time` and `category` and `Dimensional[]` are valid.-In the case of a multi dimensional routed alias (A. K. A. "DRA", see <<aliases.adoc#dimensional-routed-aliases,Aliases>>-documentation), it is required to express all the dimensions in the same order that they will appear in the dimension-array. The format for a DRA router.name is Dimensional[dim1,dim2] where dim1 and dim2 are valid router.name-values for each sub-dimension. Note that DRA's are very new, and only 2D DRA's are presently supported. Higher-numbers of dimensions will be supported soon. See examples below for further clarification on how to configure-individual dimensions. This parameter is required.+The type of routing to use.+Presently only `time` and `category` and `Dimensional[]` are valid.+In the case of a multi dimensional routed alias (aka "DRA", see <<aliases.adoc#dimensional-routed-aliases,Aliases>>), it is required to express all the dimensions in the same order that they will appear in the dimension+array.+The format for a DRA router.name is Dimensional[dim1,dim2] where dim1 and dim2 are valid router.name values for each sub-dimension.+Note that DRA's are very new, and only 2D DRA's are presently supported.+Higher numbers of dimensions will be supported soon.+See examples below for further clarification on how to configure+individual dimensions.+This parameter is required.`router.field`::The field to inspect to determine which underlying collection an incoming document should be routed to.This field is required on all incoming documents.`create-collection.*`::-The `*` wildcard can be replaced with any parameter from the <<collection-management.adoc#create,CREATE>> command except `name`. All other fields-are identical in requirements and naming except that we insist that the configset be explicitly specified.+The `*` wildcard can be replaced with any parameter from the <<collection-management.adoc#create,CREATE>> command except `name`.+All other fields are identical in requirements and naming except that we insist that the configset be explicitly specified.The configset must be created beforehand, either uploaded or copied and modified.It's probably a bad idea to use "data driven" mode as schema mutations might happen concurrently leading to errors.==== Time Routed Alias Parameters`router.start`::-The start date/time of data for this time routed alias in Solr's standard date/time format (i.e., ISO-8601 or "NOW"-optionally with <<working-with-dates.adoc#date-math,date math>>).+The start date/time of data for this time routed alias in Solr's standard date/time format (i.e., ISO-8601 or "NOW" optionally with <<date-formatting-math.adoc#date-math,date math>>).+The first collection created for the alias will be internally named after this value.-If a document is submitted with an earlier value for router.field then the earliest collection the alias points to then-it will yield an error since it can't be routed. This date/time MUST NOT have a milliseconds component other than 0.-Particularly, this means `NOW` will fail 999 times out of 1000, though `NOW/SECOND`, `NOW/MINUTE`, etc. will work-just fine. This parameter is required.+If a document is submitted with an earlier value for router.field then the earliest collection the alias points to then it will yield an error since it can't be routed.+This date/time MUST NOT have a milliseconds component other than 0.+Particularly, this means `NOW` will fail 999 times out of 1000, though `NOW/SECOND`, `NOW/MINUTE`, etc., will work just fine.+This parameter is required.`TZ`::-The timezone to be used when evaluating any date math in router.start or router.interval. This is equivalent to the-same parameter supplied to search queries, but understand in this case it's persisted with most of the other parameters+The timezone to be used when evaluating any date math in router.start or router.interval.+This is equivalent to the same parameter supplied to search queries, but understand in this case it's persisted with most of the other parametersas an alias property.+-If GMT-4 is supplied for this value then a document dated 2018-01-14T21:00:00:01.2345Z would be stored in the-myAlias_2018-01-15_01 collection (assuming an interval of +1HOUR).+If GMT-4 is supplied for this value then a document dated 2018-01-14T21:00:00:01.2345Z would be stored in the myAlias_2018-01-15_01 collection (assuming an interval of +1HOUR).+The default timezone is UTC.`router.interval`::A date math expression that will be appended to a timestamp to determine the next collection in the series.-Any date math expression that can be evaluated if appended to a timestamp of the form 2018-01-15T16:17:18 will-work here.+Any date math expression that can be evaluated if appended to a timestamp of the form 2018-01-15T16:17:18 will work here.+This parameter is required.`router.maxFutureMs`::-The maximum milliseconds into the future that a document is allowed to have in `router.field` for it to be accepted-without error. If there was no limit, than an erroneous value could trigger many collections to be created.+The maximum milliseconds into the future that a document is allowed to have in `router.field` for it to be accepted without error.+If there was no limit, than an erroneous value could trigger many collections to be created.+-The default is 10 minutes.+The default is `600000` or 10 minutes.`router.preemptiveCreateMath`::A date math expression that results in early creation of new collections.+-If a document arrives with a timestamp that is after the end time of the most recent collection minus this-interval, then the next (and only the next) collection will be created asynchronously. Without this setting, collections are created-synchronously when required by the document time stamp and thus block the flow of documents until the collection-is created (possibly several seconds). Preemptive creation reduces these hiccups. If set to enough time (perhaps-an hour or more) then if there are problems creating a collection, this window of time might be enough to take-corrective action. However after a successful preemptive creation, the collection is consuming resources without-being used, and new documents will tend to be routed through it only to be routed elsewhere. Also, note that-`router.autoDeleteAge` is currently evaluated relative to the date of a newly created collection, and so you may-want to increase the delete age by the preemptive window amount so that the oldest collection isn't deleted too-soon. Note that it has to be possible to subtract the interval specified from a date, so if prepending a-minus sign creates invalid date math, this will cause an error. Also note that a document that is itself-destined for a collection that does not exist will still trigger synchronous creation up to that destination collection-but will not trigger additional async preemptive creation. Only one type of collection creation can happen-per document.+If a document arrives with a timestamp that is after the end time of the most recent collection minus this interval, then the next (and only the next) collection will be created asynchronously.+Without this setting, collections are created synchronously when required by the document time stamp and thus block the flow of documents until the collection is created (possibly several seconds).+Preemptive creation reduces these hiccups.+If set to enough time (perhaps an hour or more) then if there are problems creating a collection, this window of time might be enough to take+corrective action.+However after a successful preemptive creation, the collection is consuming resources without being used, and new documents will tend to be routed through it only to be routed elsewhere.+Also, note that `router.autoDeleteAge` is currently evaluated relative to the date of a newly created collection, so you may want to increase the delete age by the preemptive window amount so that the oldest collection isn't deleted too+soon.+Note that it has to be possible to subtract the interval specified from a date, so if prepending a minus sign creates invalid date math, this will cause an error.+Also note that a document that is itself destined for a collection that does not exist will still trigger synchronous creation up to that destination collection but will not trigger additional async preemptive creation.+Only one type of collection creation can happen per document.Example: `90MINUTES`.+This property is blank by default indicating just-in-time, synchronous creation of new collections.@@ -173,8 +171,7 @@ This property is blank by default indicating just-in-time, synchronous creation`router.autoDeleteAge`::A date math expression that results in the oldest collections getting deleted automatically.+-The date math is relative to the timestamp of a newly created collection (typically close to the current time),-and thus this must produce an earlier time via rounding and/or subtracting.+The date math is relative to the timestamp of a newly created collection (typically close to the current time), and thus this must produce an earlier time via rounding and/or subtracting.Collections to be deleted must have a time range that is entirely before the computed age.Collections are considered for deletion immediately prior to new collections getting created.Example: `/DAY-90DAYS`.@@ -188,15 +185,14 @@ The maximum number of categories allowed for this alias.This setting safeguards against the inadvertent creation of an infinite number of collections in the event of bad data.`router.mustMatch`::-A regular expression that the value of the field specified by `router.field` must match before a corresponding-collection will be created. Note that changing this setting after data has been added will not alter the data already-indexed. Any valid Java regular expression pattern may be specified. This expression is pre-compiled at the start of-each request so batching of updates is strongly recommended. Overly complex patterns will produce cpu-or garbage collecting overhead during indexing as determined by the JVM's implementation of regular expressions.+A regular expression that the value of the field specified by `router.field` must match before a corresponding collection will be created.+Note that changing this setting after data has been added will not alter the data already indexed.+Any valid Java regular expression pattern may be specified.+This expression is pre-compiled at the start of each request so batching of updates is strongly recommended.+Overly complex patterns will produce CPU or garbage collection overhead during indexing as determined by the JVM's implementation of regular expressions.==== Dimensional Routed Alias Parameters-`router.#.`::This prefix denotes which position in the dimension array is being referred to for purposes of dimension configuration.For example in a Dimensional[time,category] router.0.start would be used to set the start time for the time dimension.@@ -205,8 +201,8 @@ For example in a Dimensional[time,category] router.0.start would be used to set=== CREATEALIAS ResponseThe output will simply be a responseHeader with details of the time it took to process the request.-To confirm the creation of the alias, you can look in the Solr Admin UI, under the Cloud section and find the-`aliases.json` file. The initial collection for routed aliases should also be visible in various parts of the admin UI.+To confirm the creation of the alias, you can look in the Solr Admin UI, under the Cloud section and find the `aliases.json` file.+The initial collection for routed aliases should also be visible in various parts of the admin UI.=== Examples using CREATEALIASCreate an alias named "testalias" and link it to the collections named "foo" and "bar".@@ -522,7 +518,8 @@ http://localhost:8983/solr/admin/collections?action=LISTALIASES&wt=xml[[aliasprop]]== ALIASPROP: Modify Alias Properties for a Collection-The `ALIASPROP` action modifies the properties (metadata) on an alias. If a key is set with a value that is empty it will be removed.+The `ALIASPROP` action modifies the properties (metadata) on an alias.+If a key is set with a value that is empty it will be removed.[.dynamic-tabs]--@@ -556,14 +553,15 @@ curl -X POST http://localhost:8983/api/collections -H 'Content-Type: application---WARNING: This command allows you to revise any property. No alias specific validation is performed.- Routed aliases may cease to function, function incorrectly or cause errors if property values- are set carelessly.+WARNING: This command allows you to revise any property.+No alias specific validation is performed.+Routed aliases may cease to function, function incorrectly, or cause errors if property values are set carelessly.=== ALIASPROP Parameters`name`::-The alias name on which to set properties. This parameter is required.+The alias name on which to set properties.+This parameter is required.`property._name_=_value_` (v1)::Set property _name_ to _value_.@@ -577,8 +575,7 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou=== ALIASPROP ResponseThe output will simply be a responseHeader with details of the time it took to process the request.-To confirm the creation of the property or properties, you can look in the Solr Admin UI, under the Cloud section and-find the `aliases.json` file or use the LISTALIASES api command.+To confirm the creation of the property or properties, you can look in the Solr Admin UI, under the Cloud section and find the `aliases.json` file or use the LISTALIASES api command.[[deletealias]]== DELETEALIAS: Delete a Collection Alias@@ -616,7 +613,8 @@ curl -X POST http://localhost:8983/api/collections -H 'Content-Type: application=== DELETEALIAS Parameters`name`::-The name of the alias to delete. This parameter is required.+The name of the alias to delete.+This parameter is required.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -624,5 +622,4 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou=== DELETEALIAS ResponseThe output will simply be a responseHeader with details of the time it took to process the request.-To confirm the removal of the alias, you can look in the Solr Admin UI, under the Cloud section, and-find the `aliases.json` file.+To confirm the removal of the alias, you can look in the Solr Admin UI, under the Cloud section, and find the `aliases.json` file.diff --git a/solr/solr-ref-guide/src/aliases.adoc b/solr/solr-ref-guide/src/aliases.adocindex 57cdf6ebe5b..e30d712a866 100644--- a/solr/solr-ref-guide/src/aliases.adoc+++ b/solr/solr-ref-guide/src/aliases.adoc@@ -16,34 +16,31 @@// specific language governing permissions and limitations// under the License.--SolrCloud has the ability to query one or more collections via an alternative name. These-alternative names for collections are known as aliases, and are useful when you want to:+SolrCloud has the ability to query one or more collections via an alternative name.+These alternative names for collections are known as aliases, and are useful when you want to:. Atomically switch to using a newly (re)indexed collection with zero down time (by re-defining the alias). Insulate the client programming versus changes in collection names. Issue a single query against several collections with identical schemas-There are two types of aliases: standard aliases and routed aliases. Within routed aliases, there are two types: category-routed aliases and time-routed aliases. These types are discussed in this section.+There are two types of aliases: standard aliases and routed aliases.+Within routed aliases, there are two types: category-routed aliases and time-routed aliases.+These types are discussed in this section.-It's possible to send collection update commands to aliases, but only to those that either resolve to a single collection-or those that define the routing between multiple collections (<<Routed Aliases>>). In other cases update commands are-rejected with an error since there is no logic by which to distribute documents among the multiple collections.+It's possible to send collection update commands to aliases, but only to those that either resolve to a single collection or those that define the routing between multiple collections (<<Routed Aliases>>).+In other cases update commands are rejected with an error since there is no logic by which to distribute documents among the multiple collections.== Standard Aliases-Standard aliases are created and updated using the <<collection-aliasing.adoc#createalias,CREATEALIAS>> command.+Standard aliases are created and updated using the <<alias-management.adoc#createalias,CREATEALIAS>> command.-The current list of collections that are members of an alias can be verified via the- <<cluster-node-management.adoc#clusterstatus,CLUSTERSTATUS>> command.+The current list of collections that are members of an alias can be verified via the <<cluster-node-management.adoc#clusterstatus,CLUSTERSTATUS>> command.-The full definition of all aliases including metadata about that alias (in the case of routed aliases, see below)- can be verified via the <<collection-aliasing.adoc#listaliases,LISTALIASES>> command.+The full definition of all aliases including metadata about that alias (in the case of routed aliases, see below) can be verified via the <<alias-management.adoc#listaliases,LISTALIASES>> command.-Alternatively this information is available by checking `/aliases.json` in ZooKeeper with either the native ZooKeeper- client or in the <<cloud-screens.adoc#tree-view,tree page>> of the cloud menu in the admin UI.+Alternatively this information is available by checking `/aliases.json` in ZooKeeper with either the native ZooKeeper client or in the <<cloud-screens.adoc#tree-view,tree page>> of the cloud menu in the admin UI.-Aliases may be deleted via the <<collection-aliasing.adoc#deletealias,DELETEALIAS>> command.+Aliases may be deleted via the <<alias-management.adoc#deletealias,DELETEALIAS>> command.When deleting an alias, underlying collections are *unaffected*.[TIP]@@ -51,154 +48,122 @@ When deleting an alias, underlying collections are *unaffected*.Any alias (standard or routed) that references multiple collections may complicate relevancy.By default, SolrCloud scores documents on a per-shard basis.-With multiple collections in an alias this is always a problem, so if you have a use case for which BM25 or- TF/IDF relevancy is important you will want to turn on one of the- <<distributed-requests.adoc#distributedidf,ExactStatsCache>> implementations.+With multiple collections in an alias this is always a problem, so if you have a use case for which BM25 or TF/IDF relevancy is important you will want to turn on one of the <<solrcloud-distributed-requests.adoc#distributedidf,ExactStatsCache>> implementations.-However, for analytical use cases where results are sorted on numeric, date, or alphanumeric field values, rather- than relevancy calculations, this is not a problem.+However, for analytical use cases where results are sorted on numeric, date, or alphanumeric field values, rather than relevancy calculations, this is not a problem.====== Routed Aliases-To address the update limitations associated with standard aliases and provide additional useful features, the concept of- routed aliases has been developed.-There are presently two types of routed alias: time routed and category routed. These are described in detail below,- but share some common behavior.+To address the update limitations associated with standard aliases and provide additional useful features, the concept of routed aliases has been developed.+There are presently two types of routed alias: time routed and category routed.+These are described in detail below, but share some common behavior.-When processing an update for a routed alias, Solr initializes its- <<update-request-processors.adoc#,UpdateRequestProcessor>> chain as usual, but- when `DistributedUpdateProcessor` (DUP) initializes, it detects that the update targets a routed alias and injects- `RoutedAliasUpdateProcessor` (RAUP) in front of itself.-RAUP, in coordination with the Overseer, is the main part of a routed alias, and must immediately precede DUP. It is not- possible to configure custom chains with other types of UpdateRequestProcessors between RAUP and DUP.+When processing an update for a routed alias, Solr initializes its <<update-request-processors.adoc#,UpdateRequestProcessor>> chain as usual, but when `DistributedUpdateProcessor` (DUP) initializes, it detects that the update targets a routed alias and injects `RoutedAliasUpdateProcessor` (RAUP) in front of itself.+RAUP, in coordination with the Overseer, is the main part of a routed alias, and must immediately precede DUP.+It is not possible to configure custom chains with other types of UpdateRequestProcessors between RAUP and DUP.-Ideally, as a user of a routed alias, you needn't concern yourself with the particulars of the collection naming pattern- since both queries and updates may be done via the alias.+Ideally, as a user of a routed alias, you needn't concern yourself with the particulars of the collection naming pattern since both queries and updates may be done via the alias.When adding data, you should usually direct documents to the alias (e.g., reference the alias name instead of any collection).The Solr server and `CloudSolrClient` will direct an update request to the first collection that an alias points to.Once the server receives the data it will perform the necessary routing.-WARNING: It's extremely important with all routed aliases that the route values NOT change. Reindexing a document-with a different route value for the same ID produces two distinct documents with the same ID accessible via the alias.+WARNING: It's extremely important with all routed aliases that the route values NOT change.+Reindexing a document with a different route value for the same ID produces two distinct documents with the same ID accessible via the alias.All query time behavior of the routed alias is *_undefined_* and not easily predictable once duplicate ID's exist.-CAUTION: It is a bad idea to use "data driven" mode (aka <<schemaless-mode.adoc#,schemaless-mode>>) with-routed aliases, as duplicate schema mutations might happen concurrently leading to errors.+CAUTION: It is a bad idea to use "data driven" mode (aka <<schemaless-mode.adoc#,schemaless-mode>>) with routed aliases, as duplicate schema mutations might happen concurrently leading to errors.=== Time Routed Aliases-Time Routed Aliases (TRAs) are a SolrCloud feature that manages an alias and a time sequential- series of collections.+Time Routed Aliases (TRAs) are a SolrCloud feature that manages an alias and a time sequential series of collections.-It automatically creates new collections and (optionally) deletes old ones as it routes documents to the correct- collection based on its timestamp.-This approach allows for indefinite indexing of data without degradation of performance otherwise experienced due to the- continuous growth of a single index.+It automatically creates new collections and (optionally) deletes old ones as it routes documents to the correct collection based on its timestamp.+This approach allows for indefinite indexing of data without degradation of performance otherwise experienced due to the continuous growth of a single index.-If you need to store a lot of timestamped data in Solr, such as logs or IoT sensor data, then this feature probably- makes more sense than creating one sharded hash-routed collection.+If you need to store a lot of timestamped data in Solr, such as logs or IoT sensor data, then this feature probably makes more sense than creating one sharded hash-routed collection.==== How It Works-First you create a time routed aliases using the <<collection-aliasing.adoc#createalias,CREATEALIAS>> command with the-desired router settings.-Most of the settings are editable at a later time using the <<collection-aliasing.adoc#aliasprop,ALIASPROP>> command.+First you create a time routed aliases using the <<alias-management.adoc#createalias,CREATEALIAS>> command with the desired router settings.+Most of the settings are editable at a later time using the <<alias-management.adoc#aliasprop,ALIASPROP>> command.The first collection will be created automatically, along with an alias pointing to it.Each underlying Solr "core" in a collection that is a member of a TRA has a special core property referencing the alias.-The name of each collection is comprised of the TRA name and the start timestamp (UTC), with trailing zeros and symbols- truncated.--The collections list for a TRA is always reverse sorted, and thus the connection path of the request will route to the- lead collection. Using `CloudSolrClient` is preferable as it can reduce the number of underlying physical HTTP requests by one.-If you know that a particular set of documents to be delivered is going to a particular older collection then you could- direct it there from the client side as an optimization but it's not necessary. `CloudSolrClient` does not (yet) do this.--RAUP first reads TRA configuration from the alias properties when it is initialized. As it sees each document, it checks for- changes to TRA properties, updates its cached configuration if needed, and then determines which collection the- document belongs to:--* If RAUP needs to send it to a time segment represented by a collection other than the one that- the client chose to communicate with, then it will do so using mechanisms shared with DUP.- Once the document is forwarded to the correct collection (i.e., the correct TRA time segment), it skips directly to- DUP on the target collection and continues normally, potentially being routed again to the correct shard & replica- within the target collection.--* If it belongs in the current collection (which is usually the case if processing events as they occur), the document- passes through to DUP. DUP does its normal collection-level processing that may involve routing the document- to another shard & replica.--* If the timestamp on the document is more recent than the most recent TRA segment, then a new collection needs to be- added at the front of the TRA.- RAUP will create this collection, add it to the alias, and then forward the document to the collection it just created.- This can happen recursively if more than one collection needs to be created.+The name of each collection is comprised of the TRA name and the start timestamp (UTC), with trailing zeros and symbols truncated.++The collections list for a TRA is always reverse sorted, and thus the connection path of the request will route to the lead collection.+Using `CloudSolrClient` is preferable as it can reduce the number of underlying physical HTTP requests by one.+If you know that a particular set of documents to be delivered is going to a particular older collection then you could direct it there from the client side as an optimization but it's not necessary.+`CloudSolrClient` does not (yet) do this.++RAUP first reads TRA configuration from the alias properties when it is initialized.+As it sees each document, it checks for changes to TRA properties, updates its cached configuration if needed, and then determines which collection the document belongs to:++* If RAUP needs to send it to a time segment represented by a collection other than the one that the client chose to communicate with, then it will do so using mechanisms shared with DUP.+Once the document is forwarded to the correct collection (i.e., the correct TRA time segment), it skips directly to DUP on the target collection and continues normally, potentially being routed again to the correct shard & replica within the target collection.++* If it belongs in the current collection (which is usually the case if processing events as they occur), the document passes through to DUP.+DUP does its normal collection-level processing that may involve routing the document to another shard & replica.++* If the timestamp on the document is more recent than the most recent TRA segment, then a new collection needs to be added at the front of the TRA.+RAUP will create this collection, add it to the alias, and then forward the document to the collection it just created.+This can happen recursively if more than one collection needs to be created.+-Each time a new collection is added, the oldest collections in the TRA are examined for possible deletion, if that has- been configured.+Each time a new collection is added, the oldest collections in the TRA are examined for possible deletion, if that has been configured.All this happens synchronously, potentially adding seconds to the update request and indexing latency.+-If `router.preemptiveCreateMath` is configured and if the document arrives within this window then it will occur-asynchronously. See <<collection-aliasing.adoc#time-routed-alias-parameters,Time Routed Alias Parameters>> for more information.+If `router.preemptiveCreateMath` is configured and if the document arrives within this window then it will occur asynchronously.+See <<alias-management.adoc#time-routed-alias-parameters,Time Routed Alias Parameters>> for more information.Any other type of update like a commit or delete is routed by RAUP to all collections.-Generally speaking, this is not a performance concern. When Solr receives a delete or commit wherein nothing is deleted-or nothing needs to be committed, then it's pretty cheap.+Generally speaking, this is not a performance concern.+When Solr receives a delete or commit wherein nothing is deleted or nothing needs to be committed, then it's pretty cheap.==== Limitations & Assumptions-* Only *time* routed aliases are supported. If you instead have some other sequential number, you could fake it- as a time (e.g., convert to a timestamp assuming some epoch and increment).+* Only *time* routed aliases are supported.+If you instead have some other sequential number, you could fake it as a time (e.g., convert to a timestamp assuming some epoch and increment).+The smallest possible interval is one second.-No other routing scheme is supported, although this feature was developed with considerations that it could be- extended/improved to other schemes.+No other routing scheme is supported, although this feature was developed with considerations that it could be extended/improved to other schemes.-* The underlying collections form a contiguous sequence without gaps. This will not be suitable when there are- large gaps in the underlying data, as Solr will insist that there be a collection for each increment. This- is due in part to Solr calculating the end time of each interval collection based on the timestamp of- the next collection, since it is otherwise not stored in any way.+* The underlying collections form a contiguous sequence without gaps.+This will not be suitable when there are large gaps in the underlying data, as Solr will insist that there be a collection for each increment.+This is due in part to Solr calculating the end time of each interval collection based on the timestamp of the next collection, since it is otherwise not stored in any way.-* Avoid sending updates to the oldest collection if you have also configured that old collections should be- automatically deleted. It could lead to exceptions bubbling back to the indexing client.+* Avoid sending updates to the oldest collection if you have also configured that old collections should be automatically deleted.+It could lead to exceptions bubbling back to the indexing client.=== Category Routed Aliases-Category Routed Aliases (CRAs) are a feature to manage aliases and a set of dependent collections-based on the value of a single field.+Category Routed Aliases (CRAs) are a feature to manage aliases and a set of dependent collections based on the value of a single field.-CRAs automatically create new collections but because the partitioning is on categorical information rather than continuous-numerically based values there's no logic for automatic deletion. This approach allows for simplified indexing of data-that must be segregated into collections for cluster management or security reasons.+CRAs automatically create new collections but because the partitioning is on categorical information rather than continuous numerically based values there's no logic for automatic deletion.+This approach allows for simplified indexing of data that must be segregated into collections for cluster management or security reasons.==== How It Works-First you create a category routed alias using the <<collection-aliasing.adoc#createalias,CREATEALIAS>> command with the-desired router settings.- Most of the settings are editable at a later time using the <<collection-aliasing.adoc#aliasprop,ALIASPROP>> command.+First you create a category routed alias using the <<alias-management.adoc#createalias,CREATEALIAS>> command with the desired router settings.+Most of the settings are editable at a later time using the <<alias-management.adoc#aliasprop,ALIASPROP>> command.-The alias will be created with a special place-holder collection which will always be named- `myAlias\__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA\__TEMP`. The first document indexed into the CRA- will create a second collection named `myAlias__CRA__foo` (for a routed field value of `foo`). The second document- indexed will cause the temporary place holder collection to be deleted. Thereafter collections will be created whenever- a new value for the field is encountered.+The alias will be created with a special place-holder collection which will always be named `myAlias\__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA\__TEMP`.+The first document indexed into the CRA will create a second collection named `myAlias__CRA__foo` (for a routed field value of `foo`). The second document+ indexed will cause the temporary place holder collection to be deleted.+Thereafter collections will be created whenever a new value for the field is encountered.-CAUTION: To guard against runaway collection creation options for limiting the total number of categories, and for-rejecting values that don't match, a regular expression parameter is provided (see <<collection-aliasing.adoc#category-routed-alias-parameters,Category Routed Alias Parameters>> for-details).+CAUTION: To guard against runaway collection creation options for limiting the total number of categories, and for rejecting values that don't match, a regular expression parameter is provided (see <<alias-management.adoc#category-routed-alias-parameters,Category Routed Alias Parameters>> for details).+-Note that by providing very large or very permissive values for these options you are accepting the risk that-garbled data could potentially create thousands of collections and bring your cluster to a grinding halt.+Note that by providing very large or very permissive values for these options you are accepting the risk that garbled data could potentially create thousands of collections and bring your cluster to a grinding halt.Field values (and thus the collection names) are case sensitive.-As elsewhere in Solr, manipulation and-cleaning of the data is expected to be done by external processes before data is sent to Solr, with one exception.-Throughout Solr there are limitations on the allowable characters in collection names. Any characters other than ASCII-alphanumeric characters (`A-Za-z0-9`), hyphen (`-`) or underscore (`_`) are replaced with an underscore when calculating-the collection name for a category. For a CRA named `myAlias` the following table shows how collection names would be-calculated:+As elsewhere in Solr, manipulation and cleaning of the data is expected to be done by external processes before data is sent to Solr, with one exception.+Throughout Solr there are limitations on the allowable characters in collection names.+Any characters other than ASCII alphanumeric characters (`A-Za-z0-9`), hyphen (`-`) or underscore (`_`) are replaced with an underscore when calculating+the collection name for a category.+For a CRA named `myAlias` the following table shows how collection names would be calculated:|===|Value |CRA Collection Name@@ -226,78 +191,66 @@ calculated:|===-Since collection creation can take upwards of 1-3 seconds, systems inserting data in a CRA should be- constructed to handle such pauses whenever a new collection is created.+Since collection creation can take upwards of 1-3 seconds, systems inserting data in a CRA should be constructed to handle such pauses whenever a new collection is created.Unlike time routed aliases, there is no way to predict the next value so such pauses are unavoidable.-There is no automated means of removing a category. If a category needs to be removed from a CRA-the following procedure is recommended:+There is no automated means of removing a category.+If a category needs to be removed from a CRA the following procedure is recommended:// TODO: This should have example instructions-. Ensure that no documents with the value corresponding to the category to be removed will be sent- either by stopping indexing or by fixing the incoming data stream+. Ensure that no documents with the value corresponding to the category to be removed will be sent either by stopping indexing or by fixing the incoming data stream. Modify the alias definition in ZooKeeper, removing the collection corresponding to the category.-. Delete the collection corresponding to the category. Note that if the collection is not removed- from the alias first, this step will fail.+. Delete the collection corresponding to the category.+Note that if the collection is not removed from the alias first, this step will fail.==== Limitations & Assumptions* CRAs are presently unsuitable for non-English data values due to the limits on collection names.- This can be worked around by duplicating the route value to a *_url safe_* Base64-encoded field- and routing on that value instead.+This can be worked around by duplicating the route value to a *_url safe_* Base64-encoded field and routing on that value instead.-* The check for the __CRA__ infix is independent of the regular expression validation and occurs after- the name of the collection to be created has been calculated. It may not be avoided and is necessary- to support future features.+* The check for the __CRA__ infix is independent of the regular expression validation and occurs after the name of the collection to be created has been calculated.+It may not be avoided and is necessary to support future features.=== Dimensional Routed Aliases-For cases where the desired segregation of of data relates to two fields and combination into a single-field during indexing is impractical, or the TRA behavior is desired across multiple categories,-Dimensional Routed aliases may be used. This feature has been designed to handle an arbitrary number-and combination of category and time dimensions in any order, but users are cautioned to carefully-consider the total number of collections that will result from such configurations. Collection counts-in the high hundreds or low 1000's begin to pose significant challenges with ZooKeeper.+For cases where the desired segregation of of data relates to two fields and combination into a single field during indexing is impractical, or the TRA behavior is desired across multiple categories, Dimensional Routed aliases may be used.+This feature has been designed to handle an arbitrary number and combination of category and time dimensions in any order, but users are cautioned to carefully consider the total number of collections that will result from such configurations.+Collection counts in the high hundreds or low 1000's begin to pose significant challenges with ZooKeeper.-NOTE: DRA's are a new feature and presently only 2 dimensions are supported. More dimensions will-be supported in the future (see https://issues.apache.org/jira/browse/SOLR-13628 for progress)+NOTE: DRA's are a new feature and presently only 2 dimensions are supported.+More dimensions will be supported in the future (see https://issues.apache.org/jira/browse/SOLR-13628 for progress)==== How It Works-First you create a dimensional routed alias with the desired router settings for each dimension. See the-<<collection-aliasing.adoc#createalias,CREATEALIAS>> command documentation for details on how to specify the-per-dimension configuration. Typical collection names will be of the form (example is for category x time example,-with 30 minute intervals):+First you create a dimensional routed alias with the desired router settings for each dimension.+See the <<alias-management.adoc#createalias,CREATEALIAS>> command documentation for details on how to specify the per-dimension configuration.+Typical collection names will be of the form (example is for category x time example, with 30 minute intervals):- myalias__CRA__someCategory__TRA__2019-07-01_00_30+[source,text]+myalias__CRA__someCategory__TRA__2019-07-01_00_30Note that the initial collection will be a throw away place holder for any DRA containing a category based dimension.-Name generation for each sub-part of a collection name is identical to the corresponding potion of the component-dimension type. (e.g., a category value generating __CRA__ or __TRA__ would still produce an error)--WARNING: The prior warning about reindexing documents with different route value applies to every dimension of-a DRA. DRA's are inappropriate for documents where categories or timestamps used in routing will change (this of-course applies to other route values in future RA types too).--As with all Routed Aliases, DRA's impose some costs if your data is not well behaved. In addition to the-normal caveats of each component dimension there is a need for care in sending new categories after the DRA has been-running for a while. Ordered Dimensions (time) behave slightly differently from Unordered (category) dimensions.-Ordered dimensions rely on the iteration order of the collections in the alias and therefore cannot tolerate the-generation of collection names out of order. The this means that of this is that when an ordered dimension such as time-is a component of a DRA and the DRA experiences receipt of a document with a novel category with a time value-corresponding to a time slice other than the starting time-slice for the time dimension, several collections will-need to be created before the document can be indexed. This "new category effect" is identical to the behavior-you would get with a TRA if you picked a start-date too far in the past.--For example given a Dimensional[time,category] DRA with start time of 2019-07-01T00:00:00Z the pattern of collections-created for 4 documents might look like this:+Name generation for each sub-part of a collection name is identical to the corresponding potion of the component dimension type (e.g., a category value generating __CRA__ or __TRA__ would still produce an error).++WARNING: The prior warning about reindexing documents with different route value applies to every dimension of a DRA.+DRA's are inappropriate for documents where categories or timestamps used in routing will change (this of course applies to other route values in future RA types too).++As with all Routed Aliases, DRA's impose some costs if your data is not well behaved.+In addition to the normal caveats of each component dimension there is a need for care in sending new categories after the DRA has been running for a while.+Ordered Dimensions (time) behave slightly differently from Unordered (category) dimensions.+Ordered dimensions rely on the iteration order of the collections in the alias and therefore cannot tolerate the generation of collection names out of order.+This means that of this is that when an ordered dimension such as time is a component of a DRA and the DRA experiences receipt of a document with a novel category with a time value corresponding to a time-slice other than the starting time-slice for the time dimension, several collections will need to be created before the document can be indexed.+This "new category effect" is identical to the behavior you would get with a TRA if you picked a start-date too far in the past.++For example given a Dimensional[time,category] DRA with start time of 2019-07-01T00:00:00Z the pattern of collections created for 4 documents might look like this:*No documents**Aliased collections:*- // temp avoids empty alias error conditions- myalias__TRA__2019-07-01__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA_TEMP+[source,text]+// temp avoids empty alias error conditions+myalias__TRA__2019-07-01__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA_TEMP*Doc 1*@@ -306,9 +259,10 @@ created for 4 documents might look like this:*Aliased collections:*- // temp retained to avoid empty alias during race with collection creation- myalias__TRA__2019-07-01__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA_TEMP- myalias__TRA__2019-07-01__CRA__someCategory+[source,text]+// temp retained to avoid empty alias during race with collection creation+myalias__TRA__2019-07-01__CRA__NEW_CATEGORY_ROUTED_ALIAS_WAITING_FOR_DATA_TEMP+myalias__TRA__2019-07-01__CRA__someCategory*Doc 2*@@ -317,10 +271,11 @@ created for 4 documents might look like this:*Aliased collections:*- // temp can now be deleted without risk of having an empty alias.- myalias__TRA__2019-07-01__CRA__someCategory- myalias__TRA__2019-07-01__CRA__otherCategory // 2 collections created in one update- myalias__TRA__2019-07-02__CRA__otherCategory+[source,text]+// temp can now be deleted without risk of having an empty alias.+myalias__TRA__2019-07-01__CRA__someCategory+myalias__TRA__2019-07-01__CRA__otherCategory // 2 collections created in one update+myalias__TRA__2019-07-02__CRA__otherCategory*Doc 3*@@ -329,12 +284,13 @@ created for 4 documents might look like this:*Aliased collections:*- myalias__TRA__2019-07-01__CRA__someCategory- myalias__TRA__2019-07-01__CRA__otherCategory- myalias__TRA__2019-07-02__CRA__otherCategory- myalias__TRA__2019-07-01__CRA__thirdCategory // 3 collections created in one update!- myalias__TRA__2019-07-02__CRA__thirdCategory- myalias__TRA__2019-07-03__CRA__thirdCategory+[source,text]+myalias__TRA__2019-07-01__CRA__someCategory+myalias__TRA__2019-07-01__CRA__otherCategory+myalias__TRA__2019-07-02__CRA__otherCategory+myalias__TRA__2019-07-01__CRA__thirdCategory // 3 collections created in one update!+myalias__TRA__2019-07-02__CRA__thirdCategory+myalias__TRA__2019-07-03__CRA__thirdCategory*Doc 4*@@ -343,26 +299,22 @@ created for 4 documents might look like this:*Aliased collections:*- myalias__TRA__2019-07-01__CRA__someCategory- myalias__TRA__2019-07-01__CRA__otherCategory- myalias__TRA__2019-07-02__CRA__otherCategory- myalias__TRA__2019-07-01__CRA__thirdCategory- myalias__TRA__2019-07-02__CRA__thirdCategory- myalias__TRA__2019-07-03__CRA__thirdCategory- myalias__TRA__2019-07-02__CRA__someCategory // 2 collections created in one update- myalias__TRA__2019-07-03__CRA__someCategory+[source,text]+myalias__TRA__2019-07-01__CRA__someCategory+myalias__TRA__2019-07-01__CRA__otherCategory+myalias__TRA__2019-07-02__CRA__otherCategory+myalias__TRA__2019-07-01__CRA__thirdCategory+myalias__TRA__2019-07-02__CRA__thirdCategory+myalias__TRA__2019-07-03__CRA__thirdCategory+myalias__TRA__2019-07-02__CRA__someCategory // 2 collections created in one update+myalias__TRA__2019-07-03__CRA__someCategory-Therefore the sweet spot for DRA's is for a data set with a well standardized set of dimensions that are not changing-and where the full set of permutations occur regularly. If a new category is introduced at a later date and-indexing latency is an important SLA feature, there are a couple strategies to mitigate this effect:+Therefore the sweet spot for DRA's is for a data set with a well standardized set of dimensions that are not changing and where the full set of permutations occur regularly.+If a new category is introduced at a later date and indexing latency is an important SLA feature, there are a couple strategies to mitigate this effect:-* If the number of extra time slices to be created is not very large, then sending a single document out of band from- regular indexing, and waiting for collection creation to complete before allowing the new category to be sent via the- SLA constrained process.+* If the number of extra time slices to be created is not very large, then sending a single document out of band from regular indexing, and waiting for collection creation to complete before allowing the new category to be sent via the SLA constrained process.-* If the above procedure is likely to create an extreme number of collections, and the earliest possible document in- the new category is known, the start time for the time dimension may be adjusted using the- <<collection-aliasing.adoc#aliasprop,ALIASPROP>> command+* If the above procedure is likely to create an extreme number of collections, and the earliest possible document in the new category is known, the start time for the time dimension may be adjusted using the <<alias-management.adoc#aliasprop,ALIASPROP>> command=== Improvement Possibilities@@ -371,8 +323,7 @@ Some _potential_ areas for improvement that _are not implemented yet_ are:* *TRAs*: Searches with time filters should only go to applicable collections.-* *TRAs*: Ways to automatically optimize (or reduce the resources of) older collections that aren't expected to receive more- updates, and might have less search demand.+* *TRAs*: Ways to automatically optimize (or reduce the resources of) older collections that aren't expected to receive more updates, and might have less search demand.* *CRAs*: Intrinsic support for non-English text via Base64 encoding.@@ -380,34 +331,29 @@ Some _potential_ areas for improvement that _are not implemented yet_ are:* *DRAs*: Support for more than 2 dimensions.-* `CloudSolrClient` could route documents to the correct collection based on the route value instead always picking the- latest/first.+* `CloudSolrClient` could route documents to the correct collection based on the route value instead always picking the latest/first.-* Presently only updates are routed and queries are distributed to all collections in the alias, but future- features might enable routing of the query to the single appropriate collection based on a special parameter or perhaps- a filter on the routed field.+* Presently only updates are routed and queries are distributed to all collections in the alias, but future features might enable routing of the query to the single appropriate collection based on a special parameter or perhaps a filter on the routed field.* Collections might be constrained by their size instead of or in addition to time or category value.- This might be implemented as another type of routed alias, or possibly as an option on the existing routed aliases+This might be implemented as another type of routed alias, or possibly as an option on the existing routed aliases-* Option for deletion of aliases that also deletes the underlying collections in one step. Routed Aliases may quickly- create more collections than expected during initial testing. Removing them after such events is overly tedious.+* Option for deletion of aliases that also deletes the underlying collections in one step.+Routed Aliases may quickly create more collections than expected during initial testing.+Removing them after such events is overly tedious.As always, patches and pull requests are welcome!== Collection Commands and Aliases-Starting with version 8.1 SolrCloud supports using alias names in collection commands where normally a-collection name is expected. This works only when the following criteria are satisfied:+SolrCloud supports using alias names in collection commands where normally a collection name is expected.+This works only when the following criteria are satisfied:* a request parameter `followAliases=true` is used* an alias must not refer to more than one collection* an alias must not refer to a <<Routed Aliases,Routed Alias>>-If all criteria are satisfied then the command will resolve all alias names and operate on the collections the aliases-refer to as if it was invoked with the collection names instead. Otherwise the command will not be executed and-an exception will be thrown.+If all criteria are satisfied then the command will resolve all alias names and operate on the collections the aliases refer to as if it was invoked with the collection names instead.+Otherwise the command will not be executed and an exception will be thrown.The `followAliases=true` parameter should be used with care so that the resolved targets are indeed the intended ones.-In case of multi-level aliases or shadow aliases (an alias with the same name as an existing collection but pointing-to other collections) the use of this option is strongly discouraged because effects may be difficult to-predict correctly.+In case of multi-level aliases or shadow aliases (an alias with the same name as an existing collection but pointing to other collections) the use of this option is strongly discouraged because effects may be difficult to predict correctly.diff --git a/solr/solr-ref-guide/src/analysis-screen.adoc b/solr/solr-ref-guide/src/analysis-screen.adocindex ec075c79d13..39dc213f042 100644--- a/solr/solr-ref-guide/src/analysis-screen.adoc+++ b/solr/solr-ref-guide/src/analysis-screen.adoc@@ -16,16 +16,56 @@// specific language governing permissions and limitations// under the License.-The Analysis screen lets you inspect how data will be handled according to the field, field type and dynamic field configurations found in your Schema. You can analyze how content would be handled during indexing or during query processing and view the results separately or at the same time. Ideally, you would want content to be handled consistently, and this screen allows you to validate the settings in the field type or field analysis chains.+Once you've <<field-type-definitions-and-properties.adoc#,defined a field type in your Schema>>, and specified the analysis steps that you want applied to it, you should test it out to make sure that it behaves the way you expect it to.-Enter content in one or both boxes at the top of the screen, and then choose the field or field type definitions to use for analysis.+Luckily, there is a very handy page in the Solr <<solr-admin-ui.adoc#,admin interface>> that lets you do just that.+You can invoke the analyzer for any text field, provide sample input, and display the resulting token stream.-image::images/analysis-screen/analysis_normal.png[image,height=400]+For example, let's look at some of the "Text" field types available in the `bin/solr -e techproducts` example configuration, and use the Analysis Screen (`\http://localhost:8983/solr/#/techproducts/analysis`) to compare how the tokens produced at index time for the sentence "Running an Analyzer" match up with a slightly different query text of "run my analyzer".-If you click the *Verbose Output* check box, you see more information, including more details on the transformations to the input (such as, convert to lower case, strip extra characters, etc.) including the raw bytes, type and detailed position information at each stage. The information displayed will vary depending on the settings of the field or field type. Each step of the process is displayed in a separate section, with an abbreviation for the tokenizer or filter that is applied in that step. Hover or click on the abbreviation, and you'll see the name and path of the tokenizer or filter.+We can begin with `text_ws` - one of the most simplified Text field types available:-image::images/analysis-screen/analysis_verbose.png[image,height=400]+image::images/analysis-screen/analysis_normal.png[Normal Analysis]-In the example screenshot above, several transformations are applied to the input "Running is a sport." The words "is" and "a" have been removed and the word "running" has been changed to its basic form, "run". This is because we are using the field type `text_en` in this scenario, which is configured to remove stop words (small words that usually do not provide a great deal of context) and "stem" terms when possible to find more possible matches (this is particularly helpful with plural forms of words). If you click the question mark next to the *Analyze Fieldname/Field Type* pull-down menu, the <<schema-browser-screen.adoc#,Schema Browser window>> will open, showing you the settings for the field specified.+This shows a simple output of only the tokens produced by each step of analysis (a single step in this case).+The tokenizer is shown with an abbreviation, hover or click on this to see the full name.-The section <<understanding-analyzers-tokenizers-and-filters.adoc#,Understanding Analyzers, Tokenizers, and Filters>> describes in detail what each option is and how it may transform your data and the section <<running-your-analyzer.adoc#,Running Your Analyzer>> has specific examples for using the Analysis screen.+If we enable *Verbose Output* by checking the box, more details are shown:++image::images/analysis-screen/analysis_compare_0.png[Verbose Analysis]++By looking at the start and end positions for each term, we can see that the only thing this field type does is tokenize text on whitespace.+Notice in this image that the term "Running" has a start position of 0 and an end position of 7, while "an" has a start position of 8 and an end position of 10, and "Analyzer" starts at 11 and ends at 19.+If the whitespace between the terms was also included, the count would be 21; since it is 19, we know that whitespace has been removed from this query.++Note also that the indexed terms and the query terms are still very different.+"Running" doesn't match "run", "Analyzer" doesn't match "analyzer" (to a computer), and obviously "an" and "my" are totally different words.+If our objective is to allow queries like "run my analyzer" to match indexed text like "Running an Analyzer" then we will need to pick a different field type with index and query time text analysis that does more processing of the inputs.++In particular we want:++* Case insensitivity, so "Analyzer" and "analyzer" match.+* Stemming, so words like "Run" and "Running" are considered equivalent terms.+* Stop Word Pruning, so small words like "an" and "my" don't affect the query.++For our next attempt, let's try the `text_general` field type:++image::images/analysis-screen/analysis_compare_1.png[image]++With the verbose output enabled, we can see how each stage of our new analyzers modify the tokens they receive before passing them on to the next stage.+As we scroll down to the final output, we can see that we do start to get a match on "analyzer" from each input string, thanks to the "LCF" stage, which if you hover over with your mouse, you'll see is the `LowerCaseFilter`:++image::images/analysis-screen/analysis_compare_2.png[image]++The `text_general` field type is designed to be generally useful for any language, and it has definitely gotten us closer to our objective than `text_ws` from our first example by solving the problem of case sensitivity.+It's still not quite what we are looking for because we don't see stemming or stopword rules being applied.+So now let us try the `text_en` field type:++image::images/analysis-screen/analysis_compare_3.png[image]++Now we can see the "SF" (`StopFilter`) stage of the analyzers solving the problem of removing Stop Words ("an"), and as we scroll down, we also see the "PSF" (`PorterStemFilter`) stage apply stemming rules suitable for our English language input, such that the terms produced by our "index analyzer" and the terms produced by our "query analyzer" match the way we expect.++image::images/analysis-screen/analysis_compare_4.png[image]+++At this point, we can continue to experiment with additional inputs, verifying that our analyzers produce matching tokens when we expect them to match, and disparate tokens when we do not expect them to match, as we iterate and tweak our field type configuration.diff --git a/solr/solr-ref-guide/src/analytics-expression-sources.adoc b/solr/solr-ref-guide/src/analytics-expression-sources.adocindex 7ac68122a06..4096afa1704 100644--- a/solr/solr-ref-guide/src/analytics-expression-sources.adoc+++ b/solr/solr-ref-guide/src/analytics-expression-sources.adoc@@ -77,7 +77,7 @@ There are two possible ways of specifying constant strings, as shown below.=== DatesDates can be specified in the same way as they are in Solr queries. Just use ISO-8601 format.-For more information, refer to the <<working-with-dates.adoc#,Working with Dates>> section.+For more information, refer to the <<date-formatting-math.adoc#,Working with Dates>> section.* `2017-07-17T19:35:08Z`diff --git a/solr/solr-ref-guide/src/analytics.adoc b/solr/solr-ref-guide/src/analytics.adocindex 36708265865..6bcbe9602d4 100644--- a/solr/solr-ref-guide/src/analytics.adoc+++ b/solr/solr-ref-guide/src/analytics.adoc@@ -57,7 +57,7 @@ For these changes to take effect, restart Solr or reload the core or collection.== Request SyntaxAn Analytics request is passed to Solr with the parameter `analytics` in a request sent to the-<<requesthandlers-and-searchcomponents-in-solrconfig.adoc#search-handlers,Search Handler>>.+<<requesthandlers-searchcomponents.adoc#search-handlers,Search Handler>>.Since the analytics request is sent inside of a search handler request, it will compute results based on the result set determined by the search handler.For example, this curl command encodes and POSTs a simple analytics request to the the search handler:diff --git a/solr/solr-ref-guide/src/analyzers.adoc b/solr/solr-ref-guide/src/analyzers.adocindex 36ca4c57326..ac5b26e34da 100644--- a/solr/solr-ref-guide/src/analyzers.adoc+++ b/solr/solr-ref-guide/src/analyzers.adoc@@ -20,7 +20,10 @@ An analyzer examines the text of fields and generates a token stream.Analyzers are specified as a child of the `<fieldType>` element in the `schema.xml` configuration file (in the same `conf/` directory as `solrconfig.xml`).-In normal usage, only fields of type `solr.TextField` or `solr.SortableTextField` will specify an analyzer. The simplest way to configure an analyzer is with a single `<analyzer>` element whose class attribute is a fully qualified Java class name. The named class must derive from `org.apache.lucene.analysis.Analyzer`. For example:+In normal usage, only fields of type `solr.TextField` or `solr.SortableTextField` will specify an analyzer.+The simplest way to configure an analyzer is with a single `<analyzer>` element whose class attribute is a fully qualified Java class name.+The named class must derive from `org.apache.lucene.analysis.Analyzer`.+For example:[source,xml]----@@ -29,9 +32,13 @@ In normal usage, only fields of type `solr.TextField` or `solr.SortableTextField</fieldType>-----In this case a single class, `WhitespaceAnalyzer`, is responsible for analyzing the content of the named text field and emitting the corresponding tokens. For simple cases, such as plain English prose, a single analyzer class like this may be sufficient. But it's often necessary to do more complex analysis of the field content.+In this case a single class, `WhitespaceAnalyzer`, is responsible for analyzing the content of the named text field and emitting the corresponding tokens.+For simple cases, such as plain English prose, a single analyzer class like this may be sufficient.+But it's often necessary to do more complex analysis of the field content.-Even the most complex analysis requirements can usually be decomposed into a series of discrete, relatively simple processing steps. As you will soon discover, the Solr distribution comes with a large selection of tokenizers and filters that covers most scenarios you are likely to encounter. Setting up an analyzer chain is very straightforward; you specify a simple `<analyzer>` element (no class attribute) with child elements that name factory classes for the tokenizer and filters to use, in the order you want them to run.+Even the most complex analysis requirements can usually be decomposed into a series of discrete, relatively simple processing steps.+As you will soon discover, the Solr distribution comes with a large selection of tokenizers and filters that covers most scenarios you are likely to encounter.+Setting up an analyzer chain is very straightforward; you specify a simple `<analyzer>` element (no class attribute) with child elements that name factory classes for the tokenizer and filters to use, in the order you want them to run.For example:@@ -71,21 +78,30 @@ Note that classes in the `org.apache.lucene.analysis` package may be referred to====---In this case, no Analyzer class was specified on the `<analyzer>` element. Rather, a sequence of more specialized classes are wired together and collectively act as the Analyzer for the field. The text of the field is passed to the first item in the list (`solr.StandardTokenizerFactory`), and the tokens that emerge from the last one (`solr.EnglishPorterFilterFactory`) are the terms that are used for indexing or querying any fields that use the "nametext" `fieldType`.+In this case, no Analyzer class was specified on the `<analyzer>` element.+Rather, a sequence of more specialized classes are wired together and collectively act as the Analyzer for the field.+The text of the field is passed to the first item in the list (`solr.StandardTokenizerFactory`), and the tokens that emerge from the last one (`solr.EnglishPorterFilterFactory`) are the terms that are used for indexing or querying any fields that use the "nametext" `fieldType`..Field Values versus Indexed Terms[IMPORTANT]====-The output of an Analyzer affects the _terms_ indexed in a given field (and the terms used when parsing queries against those fields) but it has no impact on the _stored_ value for the fields. For example: an analyzer might split "Brown Cow" into two indexed terms "brown" and "cow", but the stored value will still be a single String: "Brown Cow"+The output of an Analyzer affects the _terms_ indexed in a given field (and the terms used when parsing queries against those fields) but it has no impact on the _stored_ value for the fields.+For example: an analyzer might split "Brown Cow" into two indexed terms "brown" and "cow", but the stored value will still be a single String: "Brown Cow"====== Analysis Phases-Analysis takes place in two contexts. At index time, when a field is being created, the token stream that results from analysis is added to an index and defines the set of terms (including positions, sizes, and so on) for the field. At query time, the values being searched for are analyzed and the terms that result are matched against those that are stored in the field's index.+Analysis takes place in two contexts.+At index time, when a field is being created, the token stream that results from analysis is added to an index and defines the set of terms (including positions, sizes, and so on) for the field.+At query time, the values being searched for are analyzed and the terms that result are matched against those that are stored in the field's index.-In many cases, the same analysis should be applied to both phases. This is desirable when you want to query for exact string matches, possibly with case-insensitivity, for example. In other cases, you may want to apply slightly different analysis steps during indexing than those used at query time.+In many cases, the same analysis should be applied to both phases.+This is desirable when you want to query for exact string matches, possibly with case-insensitivity, for example.+In other cases, you may want to apply slightly different analysis steps during indexing than those used at query time.-If you provide a simple `<analyzer>` definition for a field type, as in the examples above, then it will be used for both indexing and queries. If you want distinct analyzers for each phase, you may include two `<analyzer>` definitions distinguished with a type attribute. For example:+If you provide a simple `<analyzer>` definition for a field type, as in the examples above, then it will be used for both indexing and queries.+If you want distinct analyzers for each phase, you may include two `<analyzer>` definitions distinguished with a type attribute.+For example:[.dynamic-tabs]--@@ -129,21 +145,22 @@ If you provide a simple `<analyzer>` definition for a field type, as in the exam====---In this theoretical example, at index time the text is tokenized, the tokens are set to lowercase, any that are not listed in `keepwords.txt` are discarded and those that remain are mapped to alternate values as defined by the synonym rules in the file `syns.txt`. This essentially builds an index from a restricted set of possible values and then normalizes them to values that may not even occur in the original text.+In this theoretical example, at index time the text is tokenized, the tokens are set to lowercase, any that are not listed in `keepwords.txt` are discarded and those that remain are mapped to alternate values as defined by the synonym rules in the file `syns.txt`.+This essentially builds an index from a restricted set of possible values and then normalizes them to values that may not even occur in the original text.-At query time, the only normalization that happens is to convert the query terms to lowercase. The filtering and mapping steps that occur at index time are not applied to the query terms. Queries must then, in this example, be very precise, using only the normalized terms that were stored at index time.+At query time, the only normalization that happens is to convert the query terms to lowercase.+The filtering and mapping steps that occur at index time are not applied to the query terms.+Queries must then, in this example, be very precise, using only the normalized terms that were stored at index time.=== Analysis for Multi-Term Expansion-In some types of queries (i.e., Prefix, Wildcard, Regex, etc.) the input provided-by the user is not natural language intended for Analysis. Things like Synonyms-or Stop word filtering do not work in a logical way in these types of Queries.+In some types of queries (i.e., Prefix, Wildcard, Regex, etc.) the input provided by the user is not natural language intended for Analysis.+Things like Synonyms or Stop word filtering do not work in a logical way in these types of Queries.-When Solr needs to perform analysis for a query that results in multi-term-expansion, then the `normalize` method is called for each factory in the filter-chain. Factories that provide filters that do not make sense in this context-will return their inputs unchanged. Normalization applies to both CharFilters-and TokenFilters+When Solr needs to perform analysis for a query that results in multi-term expansion, then the `normalize` method is called for each factory in the filter chain.+Factories that provide filters that do not make sense in this context+will return their inputs unchanged.+Normalization applies to both CharFilters and TokenFilters.For most use cases, this provides the best possible behavior, but if you wish for absolute control over the analysis performed on these types of queries, you may explicitly define a `multiterm` analyzer to use, such as in the following example:diff --git a/solr/solr-ref-guide/src/audit-logging.adoc b/solr/solr-ref-guide/src/audit-logging.adocindex 3d299d273d2..86b3baf1404 100644--- a/solr/solr-ref-guide/src/audit-logging.adoc+++ b/solr/solr-ref-guide/src/audit-logging.adoc@@ -20,12 +20,12 @@ Solr has the ability to log an audit trail of all HTTP requests entering the sysAudit loggers are pluggable to suit any possible format or log destination.[quote]-An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, or event. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia])+An audit trail (also called audit log) is a security-relevant chronological record, set of records, and/or destination and source of records that provide documentary evidence of the sequence of activities that have affected at any time a specific operation, procedure, event, or device. (https://en.wikipedia.org/wiki/Audit_trail[Wikipedia])-== Configuration in security.json+== Configuring Audit LoggingAudit logging is configured in `security.json` under the `auditlogging` key.-The example `security.json` below configures synchronous audit logging to Solr default log file.+The example below uses plugin defaults to configure synchronous audit logging to Solr's default log file.[source,json]----@@ -36,7 +36,10 @@ The example `security.json` below configures synchronous audit logging to Solr d}-----By default any AuditLogger plugin configured will log asynchronously in the background to avoid slowing down the requests. To make audit logging happen synchronously, add the parameter `async: false`. For async logging, you may optionally also configure queue size, number of threads and whether it should block when the queue is full or discard events:+By default any audit logging plugin will log asynchronously in the background to avoid slowing down the requests.+To make audit logging happen synchronously, add the parameter `async` with a value of `false`.++When using asynchronous logging, you may optionally also configure queue size, number of threads, and whether it should block when the queue is full or discard events:[source,json]----@@ -52,49 +55,130 @@ By default any AuditLogger plugin configured will log asynchronously in the back}-----The defaults are `async: true`, `blockAsync: false`, `queueSize: 4096`. The default for `numThreads` is 2, or if the server has more than 4 CPU-cores then we use CPU-cores/2.+=== Audit Logging Parameters+These parameters are:++`class`::+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The audit logging plugin class name.+Either `solr.SolrLogAuditLoggingPlugin` or `solr.MultiDestinationAuditLogger` (described below in the section <<Chaining Multiple Loggers>>).++`async`::+++[%autowidth,frame=none]+|===+|Optional |Default: `true`+|===+++Defines if events are logged asynchronously.+This defaults to `true` to avoid slowing down requests.+However, if you are confident in the performance characteristics of your system and need events logged synchronously, you can change this to `false`.++`blockAsync`::+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++Defines if requests should be blocked if the queue is full.+The default of `false` will discard unlogged events.+Only used when `async=true`.++`numThreads`::+++[%autowidth,frame=none]+|===+|Optional |Default: `2`+|===+++The number of threads available to audit logging.+If the number of CPU-cores available to the server is higher than 4, then the default is modified to `CPU-cores / 2`.+Only used when `async=true`.++`queueSize`::+++[%autowidth,frame=none]+|===+|Optional |Default: `4096`+|===+++The size of the queue.+Only used when `async=true`.++`eventTypes`::+++[%autowidth,frame=none]+|===+|Optional |Default: `["REJECTED", "ANONYMOUS_REJECTED", "UNAUTHORIZED", "COMPLETED", "ERROR"]`+|===+++The event types to log.+See the section <<Event Types>> below for type options.++`muteRules`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Defines the circumstances when events should not be logged (muted).+Possible rules can exclude requests from certain users, IPs, paths, or request parameters.+See the section <<Muting Certain Events>> below for mute rule options.[#audit-event-types]=== Event Types++The event types logged can be configured with the `eventTypes` parameter.+By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged.+These are the event types triggered by the framework:[%header,format=csv,separator=;]|===EventType;Usage-AUTHENTICATED;User successfully authenticated-REJECTED;Authentication request rejected-ANONYMOUS;Request proceeds with unknown user-ANONYMOUS_REJECTED;Request from unknown user rejected-AUTHORIZED;Authorization succeeded-UNAUTHORIZED;Authorization failed-COMPLETED;Request completed successfully-ERROR;Request was not executed due to an error+`AUTHENTICATED`;User successfully authenticated+`REJECTED`;Authentication request rejected+`ANONYMOUS`;Request proceeds with unknown user+`ANONYMOUS_REJECTED`;Request from unknown user rejected+`AUTHORIZED`;Authorization succeeded+`UNAUTHORIZED`;Authorization failed+`COMPLETED`;Request completed successfully+`ERROR`;Request was not executed due to an error|===-By default only the final event types `REJECTED`, `ANONYMOUS_REJECTED`, `UNAUTHORIZED`, `COMPLETED` and `ERROR` are logged. What eventTypes are logged can be configured with the `eventTypes` configuration parameter.-=== Muting Certain Events-The configuration parameter `muteRules` lets you mute logging for certain events. You may specify multiple rules and combination of rules that will cause muting. You can mute by request type, username, collection name, path, request parameters or IP address. We'll explain through examples:-The below example will mute logging for all `SEARCH` requests as well as all requests made my user `johndoe` or from IP address `192.168.0.10`:+The configuration parameter `muteRules` lets you mute logging for certain events.+You may specify multiple rules and combination of rules that will cause muting.+You can mute by request type, username, collection name, path, request parameters or IP address.++The following example uses `muteRules` to mute audit logging for three categories of requests: any `SEARCH` requests, any requests made by user `johndoe`, and any requests from IP address `192.168.0.10`:[source,json]----{"auditlogging":{- "class": "solr.SolrLogAuditLoggerPlugin"+ "class": "solr.SolrLogAuditLoggerPlugin","muteRules": [ "type:SEARCH", "user:johndoe", "ip:192.168.0.10" ]}}-----An mute rule may also be a list, in which case all must be true for muting to happen. The configuration below has three mute rules:+A mute rule may also be a list, in which case all items in the list must be true for muting to happen.+The configuration below has three mute rules:[source,json]----{"auditlogging":{- "class": "solr.SolrLogAuditLoggerPlugin"+ "class": "solr.SolrLogAuditLoggerPlugin","muteRules": ["ip:192.168.0.10", <1>[ "path:/admin/collections", "param:action=LIST" ], <2>@@ -104,24 +188,27 @@ An mute rule may also be a list, in which case all must be true for muting to ha}-----<1> The first will mute all events from client IP `192.168.0.10`-<2> The second rule will mute collection admin requests with `action=LIST`-<3> The third rule will mute collection admin requests for the collection named `test`+<1> This will mute all events from client IP `192.168.0.10`.+<2> This rule will mute Collection API requests with `action=LIST`.+<3> The final rule will mute Collection API requests for the collection named `test`.Note how you can mix single string rules with lists of rules that must all match:-*Valid mute rules are:*+Options for mute rules are:-* `type:<request-type>` (request-type by name: `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, `UNKNOWN`)-* `collection:<collection-name>` (collection by name)-* `user:<userid>` (user by userid)-* `path:</path/to/handler>` (request path relative to `/solr` or for search/update requests relative to collection. Path is prefix matched, i.e., `/admin` will mute any sub path as well.-* `ip:<ip-address>` (IPv4-address)-* `param:<param>=<value>` (request parameter)+* `type:<request-type>`: A request-type by name: `ADMIN`, `SEARCH`, `UPDATE`, `STREAMING`, or `UNKNOWN`.+* `collection:<collection-name>`: A collection by name.+* `user:<userid>`: A user by userid.+* `path:</path/to/handler>`: A request path relative to `/solr` or for search or update requests relative to collection.+Path is prefix matched, i.e., `/admin` will mute any sub path as well.+* `ip:<ip-address>`: An IPv4 address.+* `param:<param>=<value>`: A request parameter.+This will likely mostly be used in conjunction with the `path` rule, as shown in the example above.=== Chaining Multiple Loggers-Using the `MultiDestinationAuditLogger` you can configure multiple audit logger plugins in a chain, to log to multiple destinations, as follows:++Using the `MultiDestinationAuditLogger` multiple audit logger plugins can be configured in a chain to log to multiple destinations.[source,json]----@@ -138,17 +225,25 @@ Using the `MultiDestinationAuditLogger` you can configure multiple audit logger}----+Note that logging to alternate destinations would need to be defined with a custom audit logging plugin.+See the javadocs for the base class at {solr-javadocs}/core/org/apache/solr/security/AuditLoggerPlugin.html[AuditLoggerPlugin].+== Metrics-AuditLoggerPlugins record metrics about count and timing of log requests, as well as queue size for async loggers. The metrics keys are all recorded on the `SECURITY` category, and each metric name are prefixed with a scope of `/auditlogging` and the class name of the logger, e.g., `SolrLogAuditLoggerPlugin`. The individual metrics are:--* `count` (type: meter. Records number and rate of audit logs done)-* `errors` (type: meter. Records number and rate of errors)-* `lost` (type: meter. Records number and rate of events lost due to queue full and `blockAsync=false`)-* `requestTimes` (type: timer. Records latency and perceniles for logging performance)-* `totalTime` (type: counter. Records total time spent)-* `queueCapacity` (type: gauge. Records the max size of the async logging queue)-* `queueSize` (type: gauge. Records the number of events currently waiting in the queue)-* `queuedTime` (type: timer. Records the amount of time events waited in queue. Adding this with requestTimes you get total time from event to logging complete)-* `async` (type: gauge. Tells whether this logger is in async mode)--TIP: If you expect a very high request rate and have a slow audit logger plugin, you may see that the `queueSize` and `queuedTime` metrics increase, and in worst case start dropping events and see an increase in `lost` count. In this case you may want to increas the `numThreads` setting.++Audit logging plugins record metrics about count and timing of log requests, as well as queue size for async loggers.+The metrics keys are all recorded on the `SECURITY` category, and each metric name are prefixed with a scope of `/auditlogging` and the class name of the logger, e.g., `SolrLogAuditLoggerPlugin`.+The individual metrics are:++* `count`: (_meter_) Records number and rate of audit logs written.+* `errors`: (_meter_) Records number and rate of errors.+* `lost`: (_meter_) Records number and rate of events lost if the queue is full and `blockAsync=false`.+* `requestTimes`: (_timer_) Records latency and percentiles for audit logging performance.+* `totalTime`: (_counter_) Records total time spent logging.+* `queueCapacity`: (_gauge_). Records the maximum size of the async logging queue.+* `queueSize`: (_gauge_) Records the number of events currently waiting in the queue.+* `queuedTime`: (_timer_) Records the amount of time events waited in queue.+Adding this with the `requestTimes` metric will show the total time from event to logging complete.+* `async`: (_gauge_) Tells whether this logger is in async mode.++TIP: If you experience a very high request rate and have a slow audit logger plugin, you may see the `queueSize` and `queuedTime` metrics increase, and possibly start dropping events (shown by an increase in `lost` count).+In this case you may want to increase the `numThreads` setting.diff --git a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adocindex 965006b4f8e..62a38ac30f1 100644--- a/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc+++ b/solr/solr-ref-guide/src/authentication-and-authorization-plugins.adoc@@ -1,5 +1,10 @@-= Configuring Authentication, Authorization and Audit Logging-:page-children: basic-authentication-plugin, hadoop-authentication-plugin, kerberos-authentication-plugin, jwt-authentication-plugin, cert-authentication-plugin, rule-based-authorization-plugin, audit-logging+= Configuring Authentication and Authorization+:page-children: basic-authentication-plugin, \+ kerberos-authentication-plugin, \+ jwt-authentication-plugin, \+ cert-authentication-plugin, \+ hadoop-authentication-plugin, \+ rule-based-authorization-plugin// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -17,17 +22,20 @@// specific language governing permissions and limitations// under the License.-Solr has security frameworks for supporting authentication, authorization and auditing of users. This allows for verifying a user's identity and for restricting access to resources in a Solr cluster.+Solr has security frameworks for supporting authentication, authorization and auditing of users.+This allows for verifying a user's identity and for restricting access to resources in a Solr cluster.Solr includes some plugins out of the box, and additional plugins can be developed using the authentication, authorization and audit logging frameworks described below.-All authentication, authorization and audit logging plugins can work with Solr whether they are running in SolrCloud mode or standalone mode. All related configuration, including users and permission rules, are stored in a file named `security.json`. When using Solr in standalone mode, this file must be in the `$SOLR_HOME` directory (usually `server/solr`). When using SolrCloud, this file must be located in ZooKeeper.+All authentication, authorization and audit logging plugins can work with Solr whether it is running as a cluster or a single-node installation.+All related configuration, including users and permission rules, are stored in a file named `security.json`.+When running Solr as a user-managed cluster or a single-node installation, this file must be in the `$SOLR_HOME` directory (usually `server/solr`).+When using SolrCloud, this file must be located in ZooKeeper.-The following section describes how to enable plugins with `security.json` and place them in the proper locations for your mode of operation.+== Configuring security.json-== Enable Plugins with security.json--All of the information required to initialize either type of security plugin is stored in a `security.json` file. This file contains 3 sections, one each for authentication, authorization, and audit logging.+All of the information required to initialize security plugins is stored in a `security.json` file.+This file contains 3 sections, one each for authentication, authorization, and audit logging..Sample security.json[source,json]@@ -45,11 +53,14 @@ All of the information required to initialize either type of security plugin is}-----The `/security.json` file needs to be in the proper location before a Solr instance comes up so Solr starts with the security plugin enabled. See the section <<Using security.json with Solr>> below for information on how to do this.+The `/security.json` file needs to be in the proper location before a Solr instance comes up so Solr starts with the security plugin enabled.+See the section <<Using security.json with Solr>> below for information on how to do this.-Depending on the plugin(s) in use, other information will be stored in `security.json` such as user information or rules to create roles and permissions. This information is added through the APIs for each plugin provided by Solr, or, in the case of a custom plugin, the approach designed by you.+Depending on the plugin(s) in use, other information will be stored in `security.json` such as user information or rules to create roles and permissions.+This information is added through the APIs for each plugin provided by Solr, or, in the case of a custom plugin, the approach designed by you.-Here is a more detailed `security.json` example. In this, the Basic authentication and rule-based authorization plugins are enabled, and some data has been added:+Here is a more detailed `security.json` example.+In this, the Basic authentication and rule-based authorization plugins are enabled, and some data has been added:[source,json]----@@ -68,7 +79,7 @@ Here is a more detailed `security.json` example. In this, the Basic authenticati== Using security.json with Solr-=== In SolrCloud Mode+=== In a SolrCloud ClusterWhile configuring Solr to use an authentication or authorization plugin, you will need to upload a `security.json` file to ZooKeeper.@@ -79,7 +90,8 @@ Create the file `security.json` with the contents:{"authentication": {"class": "org.apache.solr.security.KerberosPlugin"}}-----Note that this example defines the `KerberosPlugin` for authentication. You will want to modify this section as appropriate for the plugin you are using.+Note that this example defines the `KerberosPlugin` for authentication.+You will want to modify this section as appropriate for the plugin you are using.Then use the `bin/solr zk` command to upload the file:@@ -88,27 +100,32 @@ Then use the `bin/solr zk` command to upload the file:>bin/solr zk cp ./security.json zk:security.json -z localhost:2181-----NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.+NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.[WARNING]====-Whenever you use any security plugins and store `security.json` in ZooKeeper, we highly recommend that you implement access control in your ZooKeeper nodes. Information about how to enable this is available in the section <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>.+Whenever you use any security plugins and store `security.json` in ZooKeeper, we highly recommend that you implement access control in your ZooKeeper nodes.+Information about how to enable this is available in the section <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>.====-Once `security.json` has been uploaded to ZooKeeper, you should use the appropriate APIs for the plugins you're using to update it. You can edit it manually, but you must take care to remove any version data so it will be properly updated across all ZooKeeper nodes. The version data is found at the end of the `security.json` file, and will appear as the letter "v" followed by a number, such as `{"v":138}`.+Once `security.json` has been uploaded to ZooKeeper, you should use the appropriate APIs for the plugins you're using to update it.+You can edit it manually, but you must take care to remove any version data so it will be properly updated across all ZooKeeper nodes.+The version data is found at the end of the `security.json` file, and will appear as the letter "v" followed by a number, such as `{"v":138}`.-=== In Standalone Mode+=== In a User-Managed Cluster or Single-Node Installation-When running Solr in standalone mode, you need to create the `security.json` file and put it in the `$SOLR_HOME` directory for your installation (this is the same place you have located `solr.xml` and is usually `server/solr`).+When running Solr in either a user-managed cluster or a single-node installation, you create the `security.json` file and put it in the `$SOLR_HOME` directory for your installation (this is the same place you have located `solr.xml` and is usually `server/solr`).-If you are using <<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>, you will need to place `security.json` on each node of the cluster.+With a user-managed cluster, you will need to place `security.json` on each node of the cluster.-You can use the authentication and authorization APIs, but if you are using the legacy scaling model, you will need to make the same API requests on each node separately. You can also edit `security.json` by hand if you prefer.+You can use the authentication and authorization APIs, but with a user-managed cluster you will need to make the same API requests on each node separately.+You can also edit `security.json` by hand if you prefer.[#configuring-authentication]== Authentication-Authentication plugins help in securing the endpoints of Solr by authenticating incoming requests. A custom plugin can be implemented by extending the AuthenticationPlugin class.+Authentication plugins help in securing the endpoints of Solr by authenticating incoming requests.+A custom plugin can be implemented by extending the AuthenticationPlugin class.An authentication plugin consists of two parts:@@ -117,8 +134,8 @@ An authentication plugin consists of two parts:=== Enabling an Authentication Plugin-* Specify the authentication plugin in `/security.json` as in this example:-++Specify the authentication plugin in `/security.json` as in this example:+[source,json]----{@@ -127,8 +144,10 @@ An authentication plugin consists of two parts:"other_data" : "..."}}-----* All of the content in the authentication block of `security.json` would be passed on as a map to the plugin during initialization.-* An authentication plugin can also be used with a standalone Solr instance by passing in `-DauthenticationPlugin=<plugin class name>` during startup.++All of the content in the `authentication` block of `security.json` will be passed as a map to the plugin during initialization.++An authentication plugin can also be used with a single-node Solr instance by passing in `-DauthenticationPlugin=<plugin class name>` during startup.Currently available authentication plugins are:@@ -141,8 +160,8 @@ An authorization plugin can be written for Solr by extending the {solr-javadocs}=== Enabling an Authorization Plugin-* Make sure that the plugin implementation is in the classpath.-* The plugin can then be initialized by specifying the same in `security.json` in the following manner:+The plugin implementation must be in the classpath.+The plugin can then be initialized by specifying the same in `security.json` in the following manner:[source,json]----@@ -153,45 +172,52 @@ An authorization plugin can be written for Solr by extending the {solr-javadocs}}-----All of the content in the `authorization` block of `security.json` would be passed on as a map to the plugin during initialization.+All of the content in the `authorization` block of `security.json` will be passed on as a map to the plugin during initialization.[IMPORTANT]====-Reloading the plugin isn't yet supported and requires a restart of the Solr installation (meaning, the JVM should be restarted, not simply a core reload).+Reloading a plugin isn't yet supported and requires a restart of the Solr installation (meaning, the JVM should be restarted, not simply a core reload).====Currently available authorization plugins are:include::securing-solr.adoc[tag=list-of-authorization-plugins]-[#configuring-audit-logging]-== Audit Logging--<<audit-logging.adoc#,Audit logging>> plugins help you keep an audit trail of events happening in your Solr cluster.-Audit logging may e.g., ship data to an external audit service.-A custom plugin can be implemented by extending the `AuditLoggerPlugin` class.-== Authenticating in the Admin UI-Whenever an authentication plugin is enabled, authentication is also required for all or some operations in the Admin UI. The Admin UI is an AngularJS application running inside your browser, and is treated as any other external client by Solr.+Whenever an authentication plugin is enabled, authentication is also required for all or some operations in the Admin UI.+The Admin UI is an AngularJS application running inside your browser, and is treated as any other external client by Solr.When authentication is required the Admin UI will presented you with a login dialogue. The authentication plugins currently supported by the Admin UI are:* <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>>* <<jwt-authentication-plugin.adoc#,JWT Authentication Plugin>>-If your plugin of choice is not supported, the Admin UI will still let you perform unrestricted operations, while for restricted operations you will need to interact with Solr by sending HTTP requests instead of through the graphical user interface of the Admin UI. All operations supported by Admin UI can be performed through Solr's RESTful APIs.+If your plugin of choice is not supported, the Admin UI will still let you perform unrestricted operations, while for restricted operations you will need to interact with Solr by sending HTTP requests instead of through the graphical user interface of the Admin UI.+All operations supported by Admin UI can be performed through Solr's APIs.== Securing Inter-Node Requests-There are a lot of requests that originate from the Solr nodes itself. For example, requests from overseer to nodes, recovery threads, etc. We call these 'inter-node' request. Solr has a special built-in `PKIAuthenticationPlugin` (see below) that will always be available to secure inter-node traffic.+There are a lot of requests that originate from the Solr nodes itself.+For example, requests from overseer to nodes, recovery threads, etc.+We call these 'inter-node' requests.+Solr has a built-in `PKIAuthenticationPlugin` (described below) that is always available to secure inter-node traffic.-Each Authentication plugin may also decide to secure inter-node requests on its own. They may do this through the so-called `HttpClientBuilder` mechanism, or they may alternatively choose on a per-request basis whether to delegate to PKI or not by overriding a `interceptInternodeRequest()` method from the base class, where any HTTP headers can be set.+Each Authentication plugin may also decide to secure inter-node requests on its own.+They may do this through the so-called `HttpClientBuilder` mechanism, or they may alternatively choose on a per-request basis whether to delegate to PKI or not by overriding a `interceptInternodeRequest()` method from the base class, where any HTTP headers can be set.=== PKIAuthenticationPlugin-The `PKIAuthenticationPlugin` provides a built-in authentication mechanism where each Solr node is a super user and is fully trusted by other Solr nodes through the use of Public Key Infrastructure (PKI). Each Authentication plugn may choose to delegate all or some inter-node traffic to the PKI plugin.+The `PKIAuthenticationPlugin` provides a built-in authentication mechanism where each Solr node is a super user and is fully trusted by other Solr nodes through the use of Public Key Infrastructure (PKI).+Each Authentication plugin may choose to delegate all or some inter-node traffic to the PKI plugin.-For each outgoing request `PKIAuthenticationPlugin` adds a special header `'SolrAuth'` which carries the timestamp and principal encrypted using the private key of that node. The public key is exposed through an API so that any node can read it whenever it needs it. Any node who gets the request with that header, would get the public key from the sender and decrypt the information. If it is able to decrypt the data, the request trusted. It is invalid if the timestamp is more than 5 secs old. This assumes that the clocks of different nodes in the cluster are synchronized. Only traffic from other Solr nodes registered with ZooKeeper is trusted.+For each outgoing request `PKIAuthenticationPlugin` adds a special header `'SolrAuth'` which carries the timestamp and principal encrypted using the private key of that node.+The public key is exposed through an API so that any node can read it whenever it needs it.+Any node who gets the request with that header, would get the public key from the sender and decrypt the information.+If it is able to decrypt the data, the request trusted.+It is invalid if the timestamp is more than 5 seconds old.+This assumes that the clocks of different nodes in the cluster are synchronized.+Only traffic from other Solr nodes registered with ZooKeeper is trusted.-The timeout is configurable through a system property called `pkiauth.ttl`. For example, if you wish to bump up the time-to-live to 10 seconds (10000 milliseconds), start each node with a property `'-Dpkiauth.ttl=10000'`.+The timeout is configurable through a system property called `pkiauth.ttl`.+For example, if you wish to increase the time-to-live to 10 seconds (10000 milliseconds), start each node with a property `'-Dpkiauth.ttl=10000'`.diff --git a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc b/solr/solr-ref-guide/src/backup-restore.adocsimilarity index 78%rename from solr/solr-ref-guide/src/making-and-restoring-backups.adocrename to solr/solr-ref-guide/src/backup-restore.adocindex 4a156f18060..5c05cfddb04 100644--- a/solr/solr-ref-guide/src/making-and-restoring-backups.adoc+++ b/solr/solr-ref-guide/src/backup-restore.adoc@@ -1,4 +1,4 @@-= Making and Restoring Backups+= Backup and Restore// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,31 +18,42 @@If you are worried about data loss, and of course you _should_ be, you need a way to back up your Solr indexes so that you can recover quickly in case of catastrophic failure.-Solr provides two approaches to backing up and restoring Solr cores or collections, depending on how you are running Solr. If you run in SolrCloud mode, you will use the Collections API. If you run Solr in standalone mode, you will use the replication handler.+Solr provides two approaches to backing up and restoring Solr cores or collections, depending on how you are running Solr.+If you run a SolrCloud cluster, you will use the Collections API.+If you run a user-managed cluster or a single-node installation, you will use the replication handler.[NOTE]====-Backups (and Snapshots) capture data that has been <<near-real-time-searching.adoc#commits-and-searching,_hard_ commited>>. Commiting changes using `softCommit=true` may result in changes that are visible in search results but not included in subsequent backups.+Backups (and Snapshots) capture data that has been <<commits-transaction-logs.adoc#hard-commits-vs-soft-commits,hard commited>>.+Committing changes using `softCommit=true` may result in changes that are visible in search results but not included in subsequent backups.Likewise, committing changes using `openSearcher=false` may result in changes committed to disk and included in subsequent backups, even if they are not currently visible in search results.====-== SolrCloud Backups+== SolrCloud Clusters-Support for backups when running SolrCloud is provided with the <<collection-management.adoc#,Collections API>>. This allows the backups to be generated across multiple shards, and restored to the same number of shards and replicas as the original collection.+Support for backups in SolrCloud is provided with the <<collection-management.adoc#,Collections API>>.+This allows the backups to be generated across multiple shards, and restored to the same number of shards and replicas as the original collection.NOTE: SolrCloud Backup/Restore requires a shared file system mounted at the same path on all nodes, or HDFS.Four different API commands are supported:-* `action=BACKUP`: This command backs up Solr indexes and configurations. More information is available in the section <<collection-management.adoc#backup,Backup Collection>>.-* `action=RESTORE`: This command restores Solr indexes and configurations. More information is available in the section <<collection-management.adoc#restore,Restore Collection>>.-* `action=LISTBACKUP`: This command lists the backup points available at a specified location, displaying metadata for each. More information is available in the section <<collection-management.adoc#listbackup,List Backups>>.-* `action=DELETEBACKUP`: This command allows deletion of backup files or whole backups. More information is available in the section <<collection-management.adoc#deletebackup,Delete Backups>>.+* `action=BACKUP`: This command backs up Solr indexes and configurations.+More information is available in the section <<collection-management.adoc#backup,Backup Collection>>.+* `action=RESTORE`: This command restores Solr indexes and configurations.+More information is available in the section <<collection-management.adoc#restore,Restore Collection>>.+* `action=LISTBACKUP`: This command lists the backup points available at a specified location, displaying metadata for each.+More information is available in the section <<collection-management.adoc#listbackup,List Backups>>.+* `action=DELETEBACKUP`: This command allows deletion of backup files or whole backups.+More information is available in the section <<collection-management.adoc#deletebackup,Delete Backups>>.-== Standalone Mode Backups+== User-Managed Clusters and Single-Node Installations-Backups and restoration uses Solr's replication handler. Out of the box, Solr includes implicit support for replication so this API can be used. Configuration of the replication handler can, however, be customized by defining your own replication handler in `solrconfig.xml`. For details on configuring the replication handler, see the section <<index-replication.adoc#configuring-the-replicationhandler,Configuring the ReplicationHandler>>.+Backups and restoration uses Solr's replication handler.+Out of the box, Solr includes implicit support for replication so this API can be used.+Configuration of the replication handler can, however, be customized by defining your own replication handler in `solrconfig.xml`.+For details on configuring the replication handler, see the section <<user-managed-index-replication.adoc#configuring-the-replicationhandler,Configuring the ReplicationHandler>>.=== Backup API@@ -56,25 +67,63 @@ You can trigger a back-up with an HTTP command like this (replace "gettingstartehttp://localhost:8983/solr/gettingstarted/replication?command=backup-----The `backup` command is an asynchronous call, and it will represent data from the latest index commit point. All indexing and search operations will continue to be executed against the index as usual.+The `backup` command is an asynchronous call, and it will represent data from the latest index commit point.+All indexing and search operations will continue to be executed against the index as usual.-Only one backup call can be made against a core at any point in time. While an ongoing backup operation is happening subsequent calls for restoring will throw an exception.+Only one backup call can be made against a core at one time.+While an ongoing backup operation is happening subsequent calls for restoring will throw an exception.The backup request can also take the following additional parameters:`location`::-The path where the backup will be created. If the path is not absolute then the backup path will be relative to Solr's instance directory.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The path where the backup will be created.+If the path is not absolute then the backup path will be relative to Solr's instance directory.`name`::-The snapshot will be created in a directory called `snapshot.<name>`. If a name is not specified then the directory name will have the following format: `snapshot.<yyyyMMddHHmmssSSS>`.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The snapshot will be created in a directory called `snapshot.<name>`.+If a name is not specified then the directory name will have the following format: `snapshot.<_yyyyMMddHHmmssSSS_>`.`numberToKeep`::-The number of backups to keep. If `maxNumberOfBackups` has been specified on the replication handler in `solrconfig.xml`, `maxNumberOfBackups` is always used and attempts to use `numberToKeep` will cause an error. Also, this parameter is not taken into consideration if the backup name is specified. More information about `maxNumberOfBackups` can be found in the section <<index-replication.adoc#configuring-the-replicationhandler,Configuring the ReplicationHandler>>.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The number of backups to keep.+If `maxNumberOfBackups` has been specified on the replication handler in `solrconfig.xml`, `maxNumberOfBackups` is always used and attempts to use `numberToKeep` will cause an error.+Also, this parameter is not taken into consideration if the backup name is specified.+More information about `maxNumberOfBackups` can be found in the section <<user-managed-index-replication.adoc#configuring-the-replicationhandler,Configuring the ReplicationHandler>>.`repository`::-The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The name of the repository to be used for the backup.+If no repository is specified then the local filesystem repository will be used automatically.`commitName`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++The name of the commit which was used while taking a snapshot using the CREATESNAPSHOT command.=== Backup Status@@ -113,20 +162,44 @@ You can restore from a backup with a command like this:http://localhost:8983/solr/gettingstarted/replication?command=restore&name=backup_name-----This will restore the named index snapshot into the current core. Searches will start reflecting the snapshot data once the restore is complete.+This will restore the named index snapshot into the current core.+Searches will start reflecting the snapshot data once the restore is complete.The `restore` request can take these additional parameters:`location`::-The location of the backup snapshot file. If not specified, it looks for backups in Solr's data directory.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The location of the backup snapshot file.+If not specified, it looks for backups in Solr's data directory.`name`::-The name of the backup index snapshot to be restored. If the name is not provided it looks for backups with `snapshot.<timestamp>` format in the location directory. It picks the latest timestamp backup in that case.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The name of the backup index snapshot to be restored.+If the name is not provided it looks for backups with `snapshot.<timestamp>` format in the location directory.+It picks the latest timestamp backup in that case.`repository`::-The name of the repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The name of the repository to be used for the backup.+If no repository is specified then the local filesystem repository will be used automatically.-The `restore` command is an asynchronous call. Once the restore is complete the data reflected will be of the backed up index which was restored.+The `restore` command is an asynchronous call.+Once the restore is complete the data reflected will be of the backed up index which was restored.Only one `restore` call can can be made against a core at one point in time. While an ongoing restore operation is happening subsequent calls for restoring will throw an exception.@@ -159,7 +232,8 @@ The status value can be "In Progress", "success" or "failed". If it failed then=== Create Snapshot API-The snapshot functionality is different from the backup functionality as the index files aren't copied anywhere. The index files are snapshotted in the same index directory and can be referenced while taking backups.+The snapshot functionality is different from the backup functionality as the index files aren't copied anywhere.+The index files are snapshotted in the same index directory and can be referenced while taking backups.You can trigger a snapshot command with an HTTP command like this (replace "techproducts" with the name of the core you are working with):@@ -172,11 +246,31 @@ http://localhost:8983/solr/admin/cores?action=CREATESNAPSHOT&core=techproducts&cThe `CREATESNAPSHOT` request parameters are:`commitName`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++The name to store the snapshot as.-`core`:: The name of the core to perform the snapshot on.+`core`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The name of the core to perform the snapshot on.-`async`:: Request ID to track this action which will be processed asynchronously.+`async`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Request ID to track this action which will be processed asynchronously.=== List Snapshot API@@ -193,9 +287,21 @@ http://localhost:8983/solr/admin/cores?action=LISTSNAPSHOTS&core=techproducts&coThe list snapshot request parameters are:`core`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++The name of the core to whose snapshots we want to list.`async`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++Request ID to track this action which will be processed asynchronously.=== Delete Snapshot API@@ -213,12 +319,30 @@ http://localhost:8983/solr/admin/cores?action=DELETESNAPSHOT&core=techproducts&cThe delete snapshot request parameters are:`commitName`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++Specify the commit name to be deleted.`core`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++The name of the core whose snapshot we want to delete.`async`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===++Request ID to track this action which will be processed asynchronously.== Backup/Restore Storage Repositoriesdiff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adocindex fd7c72404eb..5751f67e366 100644--- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc@@ -16,19 +16,20 @@// specific language governing permissions and limitations// under the License.-Solr can support Basic authentication for users with the use of the BasicAuthPlugin.+Solr can support Basic authentication for users with the use of the `BasicAuthPlugin`.-An authorization plugin is also available to configure Solr with permissions to perform various activities in the system. The authorization plugin is described in the section <<rule-based-authorization-plugin.adoc#,Rule-Based Authorization Plugin>>.+This plugin only provides user authentication.+To control user permissions, you may need to configure an authorization plugin as described in the section <<rule-based-authorization-plugin.adoc#,Rule-Based Authorization Plugin>>.== Enable Basic Authentication-To use Basic authentication, you must first create a `security.json` file. This file and where to put it is described in detail in the section <<authentication-and-authorization-plugins.adoc#enable-plugins-with-security-json,Enable Plugins with security.json>>.+To use Basic authentication, you must first create a `security.json` file.+This file and where to put it is described in detail in the section <<authentication-and-authorization-plugins.adoc#configuring-security-json,Configuring security.json>>.-For Basic authentication, the `security.json` file must have an `authentication` part which defines the class being used for authentication. Usernames and passwords (as a sha256(password+salt) hash) could be added when the file is created, or can be added later with the Basic authentication API, described below.+For Basic authentication, `security.json` must have an `authentication` block which defines the class being used for authentication.+Usernames and passwords (as a sha256(password+salt) hash) could be added when the file is created, or can be added later with the Authentication API, described below.-The `authorization` part is not related to Basic authentication, but is a separate authorization plugin designed to support fine-grained user access control. For more information, see the section <<rule-based-authorization-plugin.adoc#,Rule-Based Authorization Plugin>>.--An example `security.json` showing both sections is shown below to show how these plugins can work together:+An example `security.json` showing `authentication` and `authorization` blocks is shown below to show how authentication and authorization plugins can work together:[source,json]----@@ -43,51 +44,62 @@ An example `security.json` showing both sections is shown below to show how thes"authorization":{"class":"solr.RuleBasedAuthorizationPlugin","permissions":[{"name":"security-edit",- "role":"admin"}], <6>- "user-role":{"solr":"admin"} <7>+ "role":"admin"}],+ "user-role":{"solr":"admin"}}}-----There are several things defined in this file:+There are several options defined in this example:-<1> Basic authentication and rule-based authorization plugins are enabled.+<1> The first block defines the authentication plugin to be used and its parameters.<2> The parameter `"blockUnknown":true` means that unauthenticated requests are not allowed to pass through.<3> A user called 'solr', with a password `'SolrRocks'` has been defined.<4> We override the `realm` property to display another text on the login prompt.<5> The parameter `"forwardCredentials":false` means we let Solr's PKI authenticaion handle distributed request instead of forwarding the Basic Auth header.-<6> The 'admin' role has been defined, and it has permission to edit security settings.-<7> The 'solr' user has been defined to the 'admin' role.-Save your settings to a file called `security.json` locally. If you are using Solr in standalone mode, you should put this file in `$SOLR_HOME`.+Save your settings to a file called `security.json` locally.+If you are using Solr in single-node installation, you should put this file in `$SOLR_HOME`.-If `blockUnknown` does not appear in the `security.json` file, it will default to `true`. This has the effect of requiring authentication for HTTP access to Solr. In some cases, you may not want authentication after enabling the plugin; for example, if you want to have `security.json` in place but aren't ready to enable authentication. However, you will want to ensure that `blockUnknown` is set to `true` or omitted entirely in order for authentication to be enforced for all requests to your system.+If `blockUnknown` is not defined in the `security.json` file, it will default to `true`.+This has the effect of requiring authentication for HTTP access to Solr.+In some cases, you may not want authentication after enabling the plugin; for example, if you want to have `security.json` in place but aren't ready to enable authentication.+However, you will want to ensure that `blockUnknown` is set to `true` or omitted entirely in order for authentication to be enforced for all requests to your system.If `realm` is not defined, it will default to `solr`.-If you are using SolrCloud, you must upload `security.json` to ZooKeeper. An example command and more information about securing your setup can be found at <<authentication-and-authorization-plugins#in-solrcloud-mode,Authentication and Authorization Plugins In SolrCloud Mode>>.+If you are using SolrCloud, you must upload `security.json` to ZooKeeper.+An example command and more information about securing your setup can be found at <<authentication-and-authorization-plugins#in-a-solrcloud-cluster,Authentication and Authorization Plugins In a SolrCloud Cluster>>.=== CaveatsThere are a few things to keep in mind when using the Basic authentication plugin.-* Credentials are sent in plain text by default. It's recommended to use SSL for communication when Basic authentication is enabled, as described in the section <<enabling-ssl.adoc#,Enabling SSL>>.-* A user who has access to write permissions to `security.json` will be able to modify all the permissions and how users have been assigned permissions. Special care should be taken to only grant access to editing security to appropriate users.-* Your network should, of course, be secure. Even with Basic authentication enabled, you should not unnecessarily expose Solr to the outside world.+* Credentials are sent in plain text by default.+It's recommended to use SSL for communication when Basic authentication is enabled, as described in the section <<enabling-ssl.adoc#,Enabling SSL>>.++* A user who has access to write permissions to `security.json` will be able to modify all permissions and user permission assignments.+Special care should be taken to only grant access to editing security to appropriate users.++* Your network should, of course, be secure.+Even with Basic authentication enabled, you should not unnecessarily expose Solr to the outside world.== Editing Basic Authentication Plugin Configuration-An Authentication API allows modifying user IDs and passwords. The API provides an endpoint with specific commands to set user details or delete a user.+An Authentication API allows modifying user IDs and passwords.+The API provides an endpoint with specific commands to set user details or delete a user.=== API Entry Point* v1: `\http://localhost:8983/solr/admin/authentication`* v2: `\http://localhost:8983/api/cluster/security/authentication`-This endpoint is not collection-specific, so users are created for the entire Solr cluster. If users need to be restricted to a specific collection, that can be done with the authorization rules.+This endpoint is not collection-specific, so users are created for the entire Solr cluster.+If users need to be restricted to a specific collection, that can be done with the authorization rules.=== Add a User or Edit a Password-The `set-user` command allows you to add users and change their passwords. For example, the following defines two users and their passwords:+The `set-user` command allows you to add users and change their passwords.+For example, the following defines two users and their passwords:[.dynamic-tabs]--@@ -113,7 +125,9 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica=== Delete a User-The `delete-user` command allows you to remove a user. The user password does not need to be sent to remove a user. In the following example, we've asked that user IDs 'tom' and 'harry' be removed from the system.+The `delete-user` command allows you to remove a user.+The user password does not need to be sent to remove a user.+In the following example, we've asked that user IDs 'tom' and 'harry' be removed from the system.[.dynamic-tabs]--@@ -138,7 +152,8 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica=== Set a Property-Set properties for the authentication plugin. The currently supported properties for the Basic Authentication plugin are `blockUnknown`, `realm` and `forwardCredentials`.+Set properties for the authentication plugin.+The currently supported properties for the Basic Authentication plugin are `blockUnknown`, `realm`, and `forwardCredentials`.[.dynamic-tabs]--@@ -163,7 +178,8 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica====---The authentication realm defaults to `solr` and is displayed in the `WWW-Authenticate` HTTP header and in the Admin UI login page. To change the realm, set the `realm` property:+The authentication realm defaults to `solr` and is displayed in the `WWW-Authenticate` HTTP header and in the Admin UI login page.+To change the realm, set the `realm` property:[.dynamic-tabs]--@@ -211,10 +227,12 @@ req.setBasicAuthCredentials(userName, password);QueryResponse rsp = req.process(solrClient);-----While this is method is simple, it can often be inconvenient to ensure the credentials are provided everywhere they're needed. It also doesn't work with the many `SolrClient` methods which don't consume `SolrRequest` objects.+While this is method is simple, it can often be inconvenient to ensure the credentials are provided everywhere they're needed.+It also doesn't work with the many `SolrClient` methods which don't consume `SolrRequest` objects.=== Per-Client Credentials-Http2SolrClient supports setting the credentials at the client level when building it. This will ensure all requests issued with this particular client get the Basic Authentication headers set.+Http2SolrClient supports setting the credentials at the client level when building it.+This will ensure all requests issued with this particular client get the Basic Authentication headers set.[source,java]----@@ -234,12 +252,15 @@ QueryResponse rsp = req.process(client);----=== Global (JVM) Basic Auth Credentials+Alternatively, users can use SolrJ's `PreemptiveBasicAuthClientBuilderFactory` to add basic authentication credentials to _all_ requests automatically.To enable this feature, users should set the following system property `-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory`.`PreemptiveBasicAuthClientBuilderFactory` allows applications to provide credentials in two different ways:-. The `basicauth` system property can be passed, containing the credentials directly (e.g., `-Dbasicauth=username:password`). This option is straightforward, but may expose the credentials in the command line, depending on how they're set.-. The `solr.httpclient.config` system property can be passed, containing a path to a properties file holding the credentials. Inside this file the username and password can be specified as `httpBasicAuthUser` and `httpBasicAuthPassword`, respectively.+. The `basicauth` system property can be passed, containing the credentials directly (e.g., `-Dbasicauth=username:password`).+This option is straightforward, but may expose the credentials in the command line, depending on how they're set.+. The `solr.httpclient.config` system property can be passed, containing a path to a properties file holding the credentials.+Inside this file the username and password can be specified as `httpBasicAuthUser` and `httpBasicAuthPassword`, respectively.+[source,bash]----@@ -249,10 +270,19 @@ httpBasicAuthPassword=secretPassword== Using the Solr Control Script with Basic Auth-Add the following line to the `solr.in.sh` or `solr.in.cmd` file. This example tells the `bin/solr` command line to to use "basic" as the type of authentication, and to pass credentials with the user-name "solr" and password "SolrRocks":+Once Basic authentication is enabled, all requests to the Solr Control Script (`bin/solr`) must contain user credentials.+To ensure this, add the following line to the `solr.in.sh` or `solr.in.cmd` file.++This example tells the `bin/solr` command line to to use "basic" as the type of authentication, and to pass credentials with the user-name "solr" and password "SolrRocks":[source,bash]----SOLR_AUTH_TYPE="basic"SOLR_AUTHENTICATION_OPTS="-Dbasicauth=solr:SolrRocks"----++Alternatively, the `SOLR_AUTHENTICATION_OPTS` can take a path to a file, as in:++[source,bash]+SOLR_AUTH_TYPE="basic"+SOLR_AUTHENTICATION_OPTS="-Dsolr.httpclient.config=/path/to/solr-{solr-docs-version}.0/server/solr/basicAuth.conf"diff --git a/solr/solr-ref-guide/src/block-join-query-parser.adoc b/solr/solr-ref-guide/src/block-join-query-parser.adocnew file mode 100644index 00000000000..47ab4bf0ea5--- /dev/null+++ b/solr/solr-ref-guide/src/block-join-query-parser.adoc@@ -0,0 +1,210 @@+= Block Join Query Parser+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License++There are two query parsers that support block joins.+These parsers allow indexing and searching for relational content that has been <<indexing-nested-documents.adoc#,indexed as Nested Documents>>.++The example usage of the query parsers below assumes the following documents have been indexed:++[source,xml]+----+<add>+ <doc>+ <field name="id">1</field>+ <field name="content_type">parent</field>+ <field name="title">Solr has block join support</field>+ <doc>+ <field name="id">2</field>+ <field name="content_type">child</field>+ <field name="comments">SolrCloud supports it too!</field>+ </doc>+ </doc>+ <doc>+ <field name="id">3</field>+ <field name="content_type">parent</field>+ <field name="title">New Lucene and Solr release</field>+ <doc>+ <field name="id">4</field>+ <field name="content_type">child</field>+ <field name="comments">Lots of new features</field>+ </doc>+ </doc>+</add>+----++== Block Join Children Query Parser++This parser wraps a query that matches some parent documents and returns the children of those documents.++The syntax for this parser is: `q={!child of=<blockMask>}<someParents>`.++* The inner subordinate query string (`someParents`) must be a query that will match some parent documents+* The `of` parameter must be a query string to use as a <<#block-mask,Block Mask>> -- typically a query that matches the set of all possible parent documents++The resulting query will match all documents which do _not_ match the `<blockMask>` query and are children (or descendents) of the documents matched by `<someParents>`.++Using the example documents above, we can construct a query such as `q={!child of="content_type:parent"}title:lucene`.+We only get one document in response:++[source,xml]+----+<result name="response" numFound="1" start="0">+ <doc>+ <str name="id">4</str>+ <arr name="content_type"><str>child</str></arr>+ <str name="comments">Lots of new features</str>+ </doc>+</result>+----++[CAUTION]+====+The query for `someParents` *MUST* match a strict subset of the documents matched by the <<#block-mask,Block Mask>> or your query may result in an Error:++[literal]+Parent query must not match any docs besides parent filter.+Combine them as must (+) and must-not (-) clauses to find a problem doc.++You can search for `q=+(someParents) -(blockMask)` to find a cause if you encounter this type of error.+====++=== Filtering and Tagging++`{!child}` also supports `filters` and `excludeTags` local params like the following:++[source,text]+?q={!child of=<blockMask> filters=$parentfq excludeTags=certain}<someParents>+&parentfq=BRAND:Foo+&parentfq=NAME:Bar+&parentfq={!tag=certain}CATEGORY:Baz++This is equivalent to:++[source,text]+q={!child of=<blockMask>}+<someParents> +BRAND:Foo +NAME:Bar++Notice "$" syntax in `filters` for referencing queries; comma-separated tags `excludeTags` allows to exclude certain queries by tagging.+Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>. Note, that filtering is applied to the subordinate clause (`<someParents>`), and the intersection result is joined to the children.++=== All Children Syntax++When subordinate clause (`<someParents>`) is omitted, it's parsed as a _segmented_ and _cached_ filter for children documents.+More precisely, `q={!child of=<blockMask>}` is equivalent to `q=\*:* -<blockMask>`.++== Block Join Parent Query Parser++This parser takes a query that matches child documents and returns their parents.++The syntax for this parser is similar to the `child` parser: `q={!parent which=<blockMask>}<someChildren>`.++* The inner subordinate query string (`someChildren`) must be a query that will match some child documents+* The `which` parameter must be a query string to use as a <<#block-mask,Block Mask>> -- typically a query that matches the set of all possible parent documents++The resulting query will match all documents which _do_ match the `<blockMask>` query and are parents (or ancestors) of the documents matched by `<someChildren>`.++Again using the example documents above, we can construct a query such as `q={!parent which="content_type:parent"}comments:SolrCloud`.+We get this document in response:++[source,xml]+----+<result name="response" numFound="1" start="0">+ <doc>+ <str name="id">1</str>+ <arr name="content_type"><str>parent</str></arr>+ <arr name="title"><str>Solr has block join support</str></arr>+ </doc>+</result>+----+++[CAUTION]+====+The query for `someChildren` *MUST NOT* match any documents matched by the <<#block-mask,Block Mask>> or your query may result in an Error:++[literal]+Child query must not match same docs with parent filter.+Combine them as must clauses (+) to find a problem doc.++You can search for `q=+(blockMask) +(someChildren)` to find a cause.+====+++=== Filtering and Tagging++The `{!parent}` query supports `filters` and `excludeTags` local params like the following:++[source,text]+?q={!parent which=<blockMask> filters=$childfq excludeTags=certain}<someChildren>+&childfq=COLOR:Red+&childfq=SIZE:XL+&childfq={!tag=certain}PRINT:Hatched++This is equivalent to:++[source,text]+q={!parent which=<blockMask>}+<someChildren> +COLOR:Red +SIZE:XL++Notice the "$" syntax in `filters` for referencing queries.+Comma-separated tags in `excludeTags` allow excluding certain queries by tagging.+Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>. Note that filtering is applied to the subordinate clause (`<someChildren>`) first, and the intersection result is joined to the parents.++=== Scoring with the Block Join Parent Query Parser++You can optionally use the `score` local parameter to return scores of the subordinate query.+The values to use for this parameter define the type of aggregation, which are `avg` (average), `max` (maximum), `min` (minimum), `total (sum)`. Implicit default is `none` which returns `0.0`.++=== All Parents Syntax++When subordinate clause (`<someChildren>`) is omitted, it's parsed as a _segmented_ and _cached_ filter for all parent documents, or more precisely `q={!parent which=<blockMask>}` is equivalent to `q=<blockMask>`.++[#block-mask]+== Block Masks: The `of` and `which` local params++The purpose of the "Block Mask" query specified as either an `of` or `which` param (depending on the parser used) is to identy the set of all documents in the index which should be treated as "parents" _(or their ancestors)_ and which documents should be treated as "children".+This is important because in the "on disk" index, the relationships are flattened into "blocks" of documents, so the `of` / `which` params are needed to serve as a "mask" against the flat document blocks to identify the boundaries of every hierarchical relationship.++In the example queries above, we were able to use a very simple Block Mask of `doc_type:parent` because our data is very simple: every document is either a `parent` or a `child`.+So this query string easily distinguishes _all_ of our documents.++A common mistake is to try and use a `which` parameter that is more restrictive then the set of all parent documents, in order to filter the parents that are matched, as in this bad example:++----+// BAD! DO NOT USE!+q={!parent which="title:join"}comments:support+----++This type of query will frequently not work the way you might expect.+Since the `which` param only identifies _some_ of the "parent" documents, the resulting query can match "parent" documents it should not, because it will mistakenly identify all documents which do _not_ match the `which="title:join"` Block Mask as children of the next "parent" document in the index (that does match this Mask).++A similar problematic situation can arise when mixing parent/child documents with "simple" documents that have no children _and do not match the query used to identify 'parent' documents_. For example, if we add the following document to our existing parent/child example documents:++[source,xml]+----+<add>+ <doc>+ <field name="id">0</field>+ <field name="content_type">plain</field>+ <field name="title">Lucene and Solr are cool</field>+ </doc>+</add>+----++...then our simple `doc_type:parent` Block Mask would no longer be adequate.+ We would instead need to use `\*:* -doc_type:child` or `doc_type:(simple parent)` to prevent our "simple" document from mistakenly being treated as a "child" of an adjacent "parent" document.++The <<searching-nested-documents#searching-nested-documents,Searching Nested Documents>> section contains more detailed examples of specifing Block Mask queries with non trivial hierarchicies of documents.diff --git a/solr/solr-ref-guide/src/caches-warming.adoc b/solr/solr-ref-guide/src/caches-warming.adocnew file mode 100644index 00000000000..7013c18b794--- /dev/null+++ b/solr/solr-ref-guide/src/caches-warming.adoc@@ -0,0 +1,326 @@+= Caches and Query Warming+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr's caches provide an essential way to improve query performance.+Caches can store documents, filters used in queries, and results from previous queries.++Caches are cleared after a <<commits-transaction-logs.adoc#commits,commit>> and usually need to be re-populated before their benefit can be seen again.+To counteract this, caches can be "warmed" before a new searcher is considered opened by automatically populating the new cache with values from the old cache.++Cache management is critical to a successful Solr implementation, so it should be noted that caches will need to be fine-tuned as your application grows.++== <query> in solrconfig.xml++The settings in this section affect the way that Solr will process and respond to queries.++These settings are all configured in child elements of the `<query>` element in `solrconfig.xml`.++[source,xml]+----+<config>+ <query>+ ...+ </query>+</config>+----++== Caches++Solr caches are associated with a specific instance of an Index Searcher, a specific view of an index that doesn't change during the lifetime of that searcher.+As long as that Index Searcher is being used, any items in its cache will be valid and available for reuse.+By default cached Solr objects do not expire after a time interval; instead, they remain valid for the lifetime of the Index Searcher.+Idle time-based expiration can be enabled by using `maxIdleTime` option.++When a new searcher is opened, the current searcher continues servicing requests while the new one auto-warms its cache.+The new searcher uses the current searcher's cache to pre-populate its own.+When the new searcher is ready, it is registered as the current searcher and begins handling all new search requests.+The old searcher will be closed once it has finished servicing all its requests.++=== Cache Implementations++Solr comes with a default `SolrCache` implementation that is used for different types of caches.++The `CaffeineCache` is an implementation backed by the https://github.com/ben-manes/caffeine[Caffeine caching library].+By default it uses a Window TinyLFU (W-TinyLFU) eviction policy, which allows the eviction based on both frequency and recency of use in O(1) time with a small footprint.+Generally this cache usually offers lower memory footprint, higher hit ratio, and better multi-threaded performance over legacy caches.++`CaffeineCache` uses an auto-warm count that supports both integers and percentages which get evaluated relative to the current size of the cache when warming happens.++The <<plugins-stats-screen.adoc#,Plugins/Stats>> page in the Solr Admin UI will display information about the performance of all the active caches.+This information can help you fine-tune the sizes of the various caches appropriately for your particular application.+When a Searcher terminates, a summary of its cache usage is also written to the log.++=== Cache Parameters++Each cache has settings to define its initial size (`initialSize`), maximum size (`size`), and number of items to use for during warming (`autowarmCount`).+For `autowarmCount` this can be also expressed as a percentage instead of an absolute value.++A `maxIdleTime` attribute controls the automatic eviction of entries that haven't been used for a while.+This attribute is expressed in seconds, with the default value of `0` meaning no entries are automatically evicted due to exceeded idle time.+Smaller values of this attribute will cause older entries to be evicted quickly, which will reduce cache memory usage but may instead cause thrashing due to a repeating eviction-lookup-miss-insertion cycle of the same entries.+Larger values will cause entries to stay around longer, waiting to be reused, at the cost of increased memory usage.+Reasonable values, depending on the query volume and patterns, may lie somewhere between 60-3600.++The `maxRamMB` attribute limits the maximum amount of memory a cache may consume.+When both `size` and `maxRamMB` limits are specified the `maxRamMB` limit will take precedence and the `size` limit will be ignored.++All caches can be disabled using the parameter `enabled` with a value of `false`.+Caches can also be disabled on a query-by-query basis with the `cache` parameter, as described in the section <<common-query-parameters.adoc#cache-local-parameter,cache Local Parameter>>.++Details of each cache are described below.++=== Filter Cache++This cache holds parsed queries paired with an unordered set of all documents that match it.+Unless such a set is trivially small, the set implementation is a bitset.++The most typical way Solr uses the `filterCache` is to cache results of each `fq` search parameter, though there are some other cases as well.+Subsequent queries using the same parameter filter query result in cache hits and rapid returns of results.+See <<common-query-parameters.adoc#fq-filter-query-parameter,fq (Filter Query) Parameter>> for a detailed discussion of `fq`.+Use of this cache can be disabled for a `fq` using the <<common-query-parameters.adoc#cache-local-parameter,`cache` local parameter>>.++Another Solr feature using this cache is the `filter(...)` syntax in the default Lucene query parser.++Solr also uses this cache for faceting when the configuration parameter `facet.method` is set to `fc`.+For a discussion of faceting parameters, see <<faceting.adoc#field-value-faceting-parameters,Field-Value Faceting Parameters>>.++[source,xml]+----+<filterCache class="solr.CaffeineCache"+ size="512"+ initialSize="512"+ autowarmCount="128"/>+----++The cache supports a `maxRamMB` parameter which restricts the maximum amount of heap used by this cache.+The `CaffeineCache` only supports evictions by either heap usage or size, but not both.+Therefore, the `size` parameter is ignored if `maxRamMB` is specified.++[source,xml]+----+<filterCache class="solr.CaffeineCache"+ maxRamMB="1000"+ autowarmCount="128"/>+----++=== Query Result Cache++The `queryResultCache` holds the results of previous searches: ordered lists of document IDs (DocList) based on a query, a sort, and the range of documents requested.++The `queryResultCache` has an optional setting to limit the maximum amount of RAM used (`maxRamMB`).+This lets you specify the maximum heap size, in megabytes, used by the contents of this cache.+When the cache grows beyond this size, oldest accessed queries will be evicted until the heap usage of the cache decreases below the specified limit.+If a `size` is specified in addition to `maxRamMB` then only the heap usage limit is respected.++Use of this cache can be disabled on a query-by-query basis in `q` using the <<common-query-parameters.adoc#cache-local-parameter,cache local parameter>>.++[source,xml]+----+<queryResultCache class="solr.CaffeineCache"+ size="512"+ initialSize="512"+ autowarmCount="128"/>+----++=== Document Cache++The `documentCache` holds Lucene Document objects (the stored fields for each document).+Since Lucene internal document IDs are transient, this cache is not auto-warmed.++The size for the `documentCache` should always be greater than `max_results` times the `max_concurrent_queries`, to ensure that Solr does not need to refetch a document during a request.+The more fields you store in your documents, the higher the memory usage of this cache will be.++[source,xml]+----+<documentCache class="solr.CaffeineCache"+ size="512"+ initialSize="512"+ autowarmCount="0"/>+----++=== User Defined Caches++You can also define named caches for your own application code to use.+You can locate and use your cache object by name by calling the `SolrIndexSearcher` methods `getCache()`, `cacheLookup()` and `cacheInsert()`.++[source,xml]+----+<cache name="myUserCache" class="solr.CaffeineCache"+ size="4096"+ initialSize="1024"+ autowarmCount="1024"+ regenerator="org.mycompany.mypackage.MyRegenerator" />+----++If you want auto-warming of your cache, include a `regenerator` attribute with the fully qualified name of a class that implements {solr-javadocs}/core/org/apache/solr/search/CacheRegenerator.html[`solr.search.CacheRegenerator`].+You can also use the `NoOpRegenerator`, which simply repopulates the cache with old items.+Define it with the `regenerator` parameter as `regenerator="solr.NoOpRegenerator"`.++=== Monitoring Cache Sizes and Usage++The section <<performance-statistics-reference.adoc#cache-statistics,Cache Statistics>> describes the metrics available for each cache.+The metrics can be accessed in the <<plugins-stats-screen.adoc#,Solr Admin UI>> or in the <<metrics-reporting.adoc#metrics-api,Metrics API>>.++The most important metrics to review when assessing caches are the size and the hit ratio.++The size indicates how many items are in the cache.+Some caches support setting the maximum cache size in MB of RAM.++The hit ratio is a percentage of queries served by the cache, shown as a number between 0 and 1.+Higher values indicate that the cache is being used often, while lower values would show that the cache isn't helping queries very much.+Ideally, this number should be as close to 1 as possible.++If you find that you have a low hit ratio but you've set your cache size high, you can optimize by reducing the cache size - there's no need to keep those objects in memory when they are not being used.++Another useful metric is the cache evictions, which measures the ojects removed from the cache.+A high rate of evictions can indicate that your cache is too small and increasing it may show a higher hit ratio.+Alternatively, if your hit ratio is high but your evictions are low, your cache might be too large and you may benefit from reducing the size.++A low hit ratio is not always a sign of a specific cache problem.+If your queries are not repeated often, a low hit ratio would be expected because it's less likely that cached objects will need to be reused.+In these cases, a smaller cache size may be ideal for your system.++== Query Sizing and Warming++Several elements are available to control the size of queries and how caches are warmed.++=== <maxBooleanClauses> Element++Sets the maximum number of clauses allowed when parsing a boolean query string.++This limit only impacts boolean queries specified by a user as part of a query string, and provides per-collection controls on how complex user specified boolean queries can be.+Query strings that specify more clauses than this will result in an error.++If this per-collection limit is greater than the <<configuring-solr-xml#global-maxbooleanclauses,global `maxBooleanClauses` limit>> specified in `solr.xml`, it will have no effect, as that setting also limits the size of user specified boolean queries.++In default configurations this property uses the value of the `solr.max.booleanClauses` system property if specified.+This is the same system property used in the <<configuring-solr-xml#global-maxbooleanclauses,global `maxBooleanClauses` setting>> in the default `solr.xml` making it easy for Solr administrators to increase both values (in all collections) without needing to search through and update the `solrconfig.xml` files in each collection.++[source,xml]+----+<maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>+----++=== <enableLazyFieldLoading> Element++When this parameter is set to `true`, fields that are not directly requested will be loaded only as needed.++This can boost performance if the most common queries only need a small subset of fields, especially if infrequently accessed fields are large in size.++[source,xml]+----+<enableLazyFieldLoading>true</enableLazyFieldLoading>+----++=== <useFilterForSortedQuery> Element++This parameter configures Solr to use a filter to satisfy a search.+If the requested sort does not include "score", the `filterCache` will be checked for a filter matching the query.+For most situations, this is only useful if the same search is requested often with different sort options and none of them ever use "score".++[source,xml]+----+<useFilterForSortedQuery>true</useFilterForSortedQuery>+----++=== <queryResultWindowSize> Element++Used with the `queryResultCache`, this will cache a superset of the requested number of document IDs.++For example, if a query requests documents 10 through 19, and `queryWindowSize` is 50, documents 0 through 49 will be cached.++[source,xml]+----+<queryResultWindowSize>20</queryResultWindowSize>+----++=== <queryResultMaxDocsCached> Element++This parameter sets the maximum number of documents to cache for any entry in the `queryResultCache`.++[source,xml]+----+<queryResultMaxDocsCached>200</queryResultMaxDocsCached>+----++=== <useColdSearcher> Element++This setting controls whether search requests for which there is not a currently registered searcher should wait for a new searcher to warm up (`false`) or proceed immediately (`true`).+When set to "false`, requests will block until the searcher has warmed its caches.++[source,xml]+----+<useColdSearcher>false</useColdSearcher>+----++=== <maxWarmingSearchers> Element++This parameter sets the maximum number of searchers that may be warming up in the background at any given time.+Exceeding this limit will raise an error.++For read-only followers, a value of `2` is reasonable.+Leaders should probably be set a little higher.++[source,xml]+----+<maxWarmingSearchers>2</maxWarmingSearchers>+----++== Query-Related Listeners++As described in the section on <<Caches>>, new Searchers are cached.+It's possible to use the triggers for listeners to perform query-related tasks.+The most common use of this is to define queries to further "warm" the Searchers while they are starting.+One benefit of this approach is that field caches are pre-populated for faster sorting.++Good query selection is key with this type of listener.+It's best to choose your most common and/or heaviest queries and include not just the keywords used, but any other parameters such as sorting or filtering requests.++There are two types of events that can trigger a listener.++. A `firstSearcher` event occurs when a new searcher is being prepared but there is no current registered searcher to handle requests or to gain auto-warming data from (i.e., on Solr startup).+. A `newSearcher` event is fired whenever a new searcher is being prepared, such as after a commit, and there is a current searcher handling requests.++The (commented out) examples below can be found in the `solrconfig.xml` file of the `sample_techproducts_configs` <<config-sets.adoc#,configset>> included with Solr, and demonstrate using the `solr.QuerySenderListener` class to warm a set of explicit queries:++[source,xml]+----+<listener event="newSearcher" class="solr.QuerySenderListener">+ <arr name="queries">+ <!--+ <lst><str name="q">solr</str><str name="sort">price asc</str></lst>+ <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst>+ -->+ </arr>+</listener>++<listener event="firstSearcher" class="solr.QuerySenderListener">+ <arr name="queries">+ <lst><str name="q">static firstSearcher warming in solrconfig.xml</str></lst>+ </arr>+</listener>+----++[IMPORTANT]+====+The above code comes from a _sample_ `solrconfig.xml`.++A key best practice is to modify these defaults before taking your application to production, but please note: while the sample queries are commented out in the section for the "newSearcher", the sample query is not commented out for the "firstSearcher" event.++There is no point in auto-warming your Searcher with the query string "static firstSearcher warming in solrconfig.xml" if that is not relevant to your search application.+====diff --git a/solr/solr-ref-guide/src/cert-authentication-plugin.adoc b/solr/solr-ref-guide/src/cert-authentication-plugin.adocindex 292ee5c8fa7..1c968bda55b 100644--- a/solr/solr-ref-guide/src/cert-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/cert-authentication-plugin.adoc@@ -16,11 +16,11 @@// specific language governing permissions and limitations// under the License.-Solr can support extracting the user principal out of the client's certificate with the use of the CertAuthPlugin.+Solr can support extracting the user principal out of the client's certificate with the use of the `CertAuthPlugin`.== Enable Certificate Authentication-For Certificate authentication, the `security.json` file must have an `authentication` part which defines the class being used for authentication.+For certificate authentication, the `security.json` file must have an `authentication` part which defines the class being used for authentication.An example `security.json` is shown below:@@ -50,7 +50,8 @@ Authorization plugins will need to accept and handle the full subject name, forCN=Solr User,OU=Engineering,O=Example Inc.,C=US-----A list of possible tags that can be present in the subject name is available in https://tools.ietf.org/html/rfc5280#section-4.1.2.4[RFC-5280, Section 4.1.2.4]. Values may have spaces, punctuation, and other characters.+A list of possible tags that can be present in the subject name is available in https://tools.ietf.org/html/rfc5280#section-4.1.2.4[RFC-5280, Section 4.1.2.4].+Values may have spaces, punctuation, and other characters.It is best practice to verify the actual contents of certificates issued by your trusted certificate authority before configuring authorization based on the contents.diff --git a/solr/solr-ref-guide/src/charfilterfactories.adoc b/solr/solr-ref-guide/src/charfilterfactories.adocindex 031706cdb18..ae06264adb3 100644--- a/solr/solr-ref-guide/src/charfilterfactories.adoc+++ b/solr/solr-ref-guide/src/charfilterfactories.adoc@@ -18,7 +18,8 @@CharFilter is a component that pre-processes input characters.-CharFilters can be chained like Token Filters and placed in front of a Tokenizer. CharFilters can add, change, or remove characters while preserving the original character offsets to support features like highlighting.+CharFilters can be chained like Token Filters and placed in front of a Tokenizer.+CharFilters can add, change, or remove characters while preserving the original character offsets to support features like highlighting.== solr.MappingCharFilterFactory@@ -65,9 +66,7 @@ Mapping file syntax:* The source string must contain at least one character, but the target string may be empty.* The following character escape sequences are recognized within source and target strings:+-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed-+-[cols="20,30,20,30",options="header"]+[%autowidth.stretch,options="header"]|===|Escape Sequence |Resulting Character (http://www.ecma-international.org/publications/standards/Ecma-048.htm[ECMA-48] alias) |Unicode Character |Example Mapping Line|`\\` |`\` |U+005C |`"\\" \=> "/"`@@ -83,7 +82,8 @@ Mapping file syntax:== solr.HTMLStripCharFilterFactory-This filter creates `org.apache.solr.analysis.HTMLStripCharFilter`. This CharFilter strips HTML from the input stream and passes the result to another CharFilter or a Tokenizer.+This filter creates `org.apache.solr.analysis.HTMLStripCharFilter`.+This CharFilter strips HTML from the input stream and passes the result to another CharFilter or a Tokenizer.This filter:@@ -103,7 +103,9 @@ This filter:* Inline tags, such as `<b>`, `<i>`, or `<span>` will be removed.* Uppercase character entities like `quot`, `gt`, `lt` and `amp` are recognized and handled as lowercase.-TIP: The input need not be an HTML document. The filter removes only constructs that look like HTML. If the input doesn't include anything that looks like HTML, the filter won't remove any input.+TIP: The input need not be an HTML document.+The filter removes only constructs that look like HTML.+If the input doesn't include anything that looks like HTML, the filter won't remove any input.The table below presents examples of HTML stripping.@@ -154,11 +156,16 @@ This filter performs pre-tokenization Unicode normalization using http://site.icArguments:-`form`:: A http://unicode.org/reports/tr15/[Unicode Normalization Form], one of `nfc`, `nfkc`, `nfkc_cf`. Default is `nfkc_cf`.+`form`:: A http://unicode.org/reports/tr15/[Unicode Normalization Form], one of `nfc`, `nfkc`, `nfkc_cf`.+Default is `nfkc_cf`.-`mode`:: Either `compose` or `decompose`. Default is `compose`. Use `decompose` with `name="nfc"` or `name="nfkc"` to get NFD or NFKD, respectively.+`mode`:: Either `compose` or `decompose`.+Default is `compose`.+Use `decompose` with `name="nfc"` or `name="nfkc"` to get NFD or NFKD, respectively.-`filter`:: A http://www.icu-project.org/apiref/icu4j/com/ibm/icu/text/UnicodeSet.html[UnicodeSet] pattern. Codepoints outside the set are always left unchanged. Default is `[]` (the null set, no filtering - all codepoints are subject to normalization).+`filter`:: A http://www.icu-project.org/apiref/icu4j/com/ibm/icu/text/UnicodeSet.html[UnicodeSet] pattern.+Codepoints outside the set are always left unchanged.+Default is `[]` (the null set, no filtering - all codepoints are subject to normalization).Example:@@ -234,9 +241,7 @@ You can configure this filter in `schema.xml` like this:The table below presents examples of regex-based pattern replacement:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="20,20,10,20,30",options="header"]+[%autowidth.stretch,options="header"]|===|Input |Pattern |Replacement |Output |Description|see-ing looking |`(\w+)(ing)` |`$1` |see-ing look |Removes "ing" from the end of word.diff --git a/solr/solr-ref-guide/src/choosing-an-output-format.adoc b/solr/solr-ref-guide/src/choosing-an-output-format.adocdeleted file mode 100644index 1b9eb7426d8..00000000000--- a/solr/solr-ref-guide/src/choosing-an-output-format.adoc+++ /dev/null@@ -1,23 +0,0 @@-= Choosing an Output Format-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Many programming environments are able to send HTTP requests and retrieve responses. Parsing the responses is a slightly more thorny problem. Fortunately, Solr makes it easy to choose an output format that will be easy to handle on the client side.--Specify a response format using the `wt` parameter in a query. The available response formats are documented in <<response-writers.adoc#,Response Writers>>.--Most client APIs hide this detail for you, so for many types of client applications, you won't ever have to specify a `wt` parameter. In JavaScript, however, the interface to Solr is a little closer to the metal, so you will need to add this parameter yourself.diff --git a/solr/solr-ref-guide/src/client-api-lineup.adoc b/solr/solr-ref-guide/src/client-api-lineup.adocdeleted file mode 100644index edcbea4c287..00000000000--- a/solr/solr-ref-guide/src/client-api-lineup.adoc+++ /dev/null@@ -1,19 +0,0 @@-= Other Clients-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Solr Wiki contains a list of client APIs at https://cwiki.apache.org/confluence/display/solr/IntegratingSolr.diff --git a/solr/solr-ref-guide/src/client-apis.adoc b/solr/solr-ref-guide/src/client-apis.adocindex 99d1cbb85e8..f47d9a1ab61 100644--- a/solr/solr-ref-guide/src/client-apis.adoc+++ b/solr/solr-ref-guide/src/client-apis.adoc@@ -1,5 +1,8 @@= Client APIs-:page-children: introduction-to-client-apis, choosing-an-output-format, using-solrj, using-javascript, using-python, using-solr-from-ruby, client-api-lineup+:page-children: solrj, \+ javascript, \+ python, \+ ruby// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -17,18 +20,54 @@// specific language governing permissions and limitations// under the License.-This section discusses the available client APIs for Solr. It covers the following topics:+At its heart, Solr is a Web application, but because it is built on open protocols, any type of client application can use Solr.-<<introduction-to-client-apis.adoc#,Introduction to Client APIs>>: A conceptual overview of Solr client APIs.+Solr offers documentation on the following client integrations:-<<choosing-an-output-format.adoc#,Choosing an Output Format>>: Information about choosing a response format in Solr.+****+// This tags the below list so it can be used in the parent page section list+// tag::client-sections[]+[width=100%,cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<solrj.adoc#,SolrJ>>: SolrJ, an API for working with Java applications.+| <<javascript.adoc#,JavaScript>>: JavaScript clients.+| <<python.adoc#,Python>>: Python and JSON responses.+| <<ruby.adoc#,Ruby>>: Solr with Ruby applications.+|===+//end::client-sections[]+****-<<using-solrj.adoc#,Using SolrJ>>: Detailed information about SolrJ, an API for working with Java applications.+The Solr Wiki contains a list of client APIs at https://cwiki.apache.org/confluence/display/solr/IntegratingSolr.-<<using-javascript.adoc#,Using JavaScript>>: Explains why a client API is not needed for JavaScript responses.+== Introduction to Client APIs-<<using-python.adoc#,Using Python>>: Information about Python and JSON responses.+HTTP is the fundamental protocol used between client applications and Solr.+The client makes a request and Solr does some work and provides a response.+Clients use requests to ask Solr to do things like perform queries or index documents.-<<using-solr-from-ruby.adoc#,Using Solr From Ruby>>: Detailed information about using Solr with Ruby applications.+Client applications can reach Solr by creating HTTP requests and parsing the HTTP responses.+Client APIs encapsulate much of the work of sending requests and parsing responses, which makes it much easier to write client applications.-<<client-api-lineup.adoc#,Other Clients>>: How to find links to 3rd-party client libraries.+Clients use Solr's five fundamental operations to work with Solr.+The operations are query, index, delete, commit, and optimize.++Queries are executed by creating a URL that contains all the query parameters.+Solr examines the request URL, performs the query, and returns the results.+The other operations are similar, although in certain cases the HTTP request is a POST operation and contains information beyond whatever is included in the request URL.+An index operation, for example, may contain a document in the body of the request.++Solr also features an EmbeddedSolrServer that offers a Java API without requiring an HTTP connection.+For details, see <<solrj.adoc#,SolrJ>>.+++== Choosing an Output Format++Many programming environments are able to send HTTP requests and retrieve responses.+Parsing the responses is a slightly more thorny problem.+Fortunately, Solr makes it easy to choose an output format that will be easy to handle on the client side.++Specify a response format using the `wt` parameter in a query.+The available response formats are documented in <<response-writers.adoc#,Response Writers>>.++Most client APIs hide this detail for you, so for many types of client applications, you won't ever have to specify a `wt` parameter.+In JavaScript, however, the interface to Solr is a little closer to the metal, so you will need to add this parameter yourself.diff --git a/solr/solr-ref-guide/src/cloud-screens.adoc b/solr/solr-ref-guide/src/cloud-screens.adocindex 0563bfee3d1..f8b256cbd1a 100644--- a/solr/solr-ref-guide/src/cloud-screens.adoc+++ b/solr/solr-ref-guide/src/cloud-screens.adoc@@ -16,41 +16,49 @@// specific language governing permissions and limitations// under the License.-When running in <<solrcloud.adoc#,SolrCloud>> mode, a "Cloud" option will appear in the Admin UI between <<logging.adoc#,Logging>> and <<collections-core-admin.adoc#,Collections>>.--This screen provides status information about each collection & node in your cluster, as well as access to the low level data being stored in <<using-zookeeper-to-manage-configuration-files.adoc#,ZooKeeper>>.+This screen provides status information about each collection & node in your cluster, as well as access to the low level data being stored in <<zookeeper-file-management.adoc#,ZooKeeper files>>..Only Visible When using SolrCloud[NOTE]====-The "Cloud" menu option is only available on Solr instances running in <<getting-started-with-solrcloud.adoc#,SolrCloud mode>>. Single node or leader/follower replication instances of Solr will not display this option.+The "Cloud" menu option is only available when Solr is running <<cluster-types.adoc#solrcloud-mode,SolrCloud>>.+User-managed clusters or single-node installations will not display this option.====-Click on the "Cloud" option in the left-hand navigation, and a small sub-menu appears with options called "Nodes", "Tree", "ZK Status" and "Graph". The sub-view selected by default is "Nodes".+Click on the "Cloud" option in the left-hand navigation, and a small sub-menu appears with options called "Nodes", "Tree", "ZK Status" and "Graph".+The sub-view selected by default is "Nodes".== Nodes ViewThe "Nodes" view shows a list of the hosts and nodes in the cluster along with key information for each: "CPU", "Heap", "Disk usage", "Requests", "Collections" and "Replicas".-The example below shows the default "cloud" example with some documents added to the "gettingstarted" collection. Details are expanded for node on port 7574, showing more metadata and more metrics details. The screen provides links to navigate to nodes, collections and replicas. The table supports paging and filtering on host/node names and collection names.+The example below shows the default "cloud" example with some documents added to the "gettingstarted" collection.+Details are expanded for node on port 7574, showing more metadata and more metrics details.+The screen provides links to navigate to nodes, collections and replicas.+The table supports paging and filtering on host/node names and collection names.image::images/cloud-screens/cloud-nodes.png[image,width=900,height=415]== Tree View-The "Tree" view shows a directory structure of the data in ZooKeeper, including cluster wide information regarding the `live_nodes` and `overseer` status, as well as collection specific information such as the `state.json`, current shard leaders, and configuration files in use. In this example, we see part of the `state.json` definition for the "tlog" collection:+The "Tree" view shows a directory structure of the data in ZooKeeper, including cluster wide information regarding the `live_nodes` and `overseer` status.+Collection-specific information such as the `state.json`, current shard leaders, and configuration files in use are also available.++In this example, we see part of the `state.json` definition for the "tlog" collection:image::images/cloud-screens/cloud-tree.png[image,width=487,height=250]As an aid to debugging, the data shown in the "Tree" view can be exported locally using the following command `bin/solr zk ls -r /`== ZK Status View-The "ZK Status" view gives an overview over the ZooKeeper servers or ensemble used by Solr. It lists whether running in `standalone` or `ensemble` mode, shows how many ZooKeeper nodes are configured, and then displays a table listing detailed monitoring status for each node, including who is the leader, configuration parameters, and more.+The "ZK Status" view gives an overview over the ZooKeeper servers or ensemble used by Solr.+It lists whether running in `standalone` or `ensemble` mode, shows how many ZooKeeper nodes are configured, and then displays a table listing detailed monitoring status for each node, including who is the leader, configuration parameters, and more.image::images/cloud-screens/cloud-zkstatus.png[image,width=512,height=509]== Graph View-The "Graph" view shows a graph of each collection, the shards that make up those collections, and the addresses and type ("NRT", "TLOG" or "PULL") of each replica for each shard.+The "Graph" view shows a graph of each collection, its shards, and the addresses and type ("NRT", "TLOG", or "PULL") of each replica for each shard.-This example shows a simple cluster. In addition to the 2 shard, 2 replica "gettingstarted" collection, there is an additional "tlog" collection consisting of mixed TLOG and PULL replica types.+This example shows a simple cluster.+In addition to the 2 shard, 2 replica "gettingstarted" collection, there is an additional "tlog" collection consisting of mixed TLOG and PULL replica types.image::images/cloud-screens/cloud-graph.png[image,width=512,height=250]diff --git a/solr/solr-ref-guide/src/cluster-node-management.adoc b/solr/solr-ref-guide/src/cluster-node-management.adocindex eb0ad53aa9a..2f37b5ac57b 100644--- a/solr/solr-ref-guide/src/cluster-node-management.adoc+++ b/solr/solr-ref-guide/src/cluster-node-management.adoc@@ -27,7 +27,8 @@ These API commands work with a SolrCloud cluster at the entire cluster level, orFetch the cluster status including collections, shards, replicas, configuration name as well as collection aliases and cluster properties.Additionally, this command reports a `health` status of each collection and shard, in-order to make it easier to monitor the operational state of the collections. The+order to make it easier to monitor the operational state of the collections.+Thefollowing health state values are defined, ordered from the best to worst, based onthe percentage of active replicas (`active`):@@ -40,9 +41,7 @@ the percentage of active replicas (`active`):`RED`::No active replicas *OR* there's no shard leader.-The collection health state is reported as the worst state of any shard, e.g., for a-collection with all shards GREEN except for one YELLOW the collection health will be-reported as YELLOW.+The collection health state is reported as the worst state of any shard, e.g., for a collection with all shards GREEN except for one YELLOW the collection health will be reported as YELLOW.[.dynamic-tabs]--@@ -69,10 +68,13 @@ We do not currently have a V2 equivalent.=== CLUSTERSTATUS Parameters`collection`::-The collection or alias name for which information is requested. If omitted, information on all collections in the cluster will be returned. If an alias is supplied, information on the collections in the alias will be returned.+The collection or alias name for which information is requested.+If omitted, information on all collections in the cluster will be returned.+If an alias is supplied, information on the collections in the alias will be returned.`shard`::-The shard(s) for which information is requested. Multiple shard names can be specified as a comma-separated list.+The shard(s) for which information is requested.+Multiple shard names can be specified as a comma-separated list.`\_route_`::This can be used if you need the details of the shard where a particular document belongs to and you don't know which shard it falls under.@@ -201,18 +203,21 @@ curl -X POST http://localhost:8983/api/cluster -H 'Content-Type: application/jso=== CLUSTERPROP Parameters`name`::-The name of the property. Supported properties names are `location`, `maxCoresPerNode`, `urlScheme`, and `defaultShardPreferences`.-If the <<solr-tracing.adoc#,Jaeger tracing contrib>> has been enabled, the property `samplePercentage` is also available.+The name of the property.+Supported properties names are `location`, `maxCoresPerNode`, `urlScheme`, and `defaultShardPreferences`.+If the <<distributed-tracing.adoc#,Jaeger tracing contrib>> has been enabled, the property `samplePercentage` is also available.+Other properties can be set (for example, if you need them for custom plugins) but they must begin with the prefix `ext.`.Unknown properties that don't begin with `ext.` will be rejected.`val`::-The value of the property. If the value is empty or null, the property is unset.+The value of the property.+If the value is empty or null, the property is unset.=== CLUSTERPROP Response-The response will include the status of the request and the properties that were updated or removed. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request and the properties that were updated or removed.+If the status is anything other than "0", an error message will explain why the request failed.=== Examples using CLUSTERPROP@@ -301,13 +306,15 @@ curl -X POST -H 'Content-type:application/json' --data-binary '----NOTE: Until Solr 7.5, cluster properties supported a `collectionDefaults` key which is now deprecated and-replaced with `defaults`. Using the `collectionDefaults` parameter in Solr 7.4 or 7.5 will continue to work+replaced with `defaults`.+Using the `collectionDefaults` parameter in Solr 7.4 or 7.5 will continue to workbut the format of the properties will automatically be converted to the new nested structure.Support for the "collectionDefaults" key will be removed in Solr 9.=== Default Shard Preferences-Using the `defaultShardPreferences` parameter, you can implement rack or availability zone awareness. First, make sure to "label" your nodes using a <<configuring-solrconfig-xml.adoc#jvm-system-properties,system property>> (e.g., `-Drack=rack1`). Then, set the value of `defaultShardPreferences` to `node.sysprop:sysprop.YOUR_PROPERTY_NAME` like this:+Using the `defaultShardPreferences` parameter, you can implement rack or availability zone awareness.+First, make sure to "label" your nodes using a <<property-substitution.adoc#jvm-system-properties,system property>> (e.g., `-Drack=rack1`). Then, set the value of `defaultShardPreferences` to `node.sysprop:sysprop.YOUR_PROPERTY_NAME` like this:[source,bash]----@@ -325,7 +332,9 @@ At this point, if you run a query on a node having e.g., `rack=rack1`, Solr will[[balanceshardunique]]== BALANCESHARDUNIQUE: Balance a Property Across Nodes-Insures that a particular property is distributed evenly amongst the physical nodes that make up a collection. If the property already exists on a replica, every effort is made to leave it there. If the property is *not* on any replica on a shard, one is chosen and the property is added.+Insures that a particular property is distributed evenly amongst the physical nodes that make up a collection.+If the property already exists on a replica, every effort is made to leave it there.+If the property is *not* on any replica on a shard, one is chosen and the property is added.[.dynamic-tabs]--@@ -360,20 +369,28 @@ curl -X POST http://localhost:8983/api/collections/techproducts -H 'Content-Type=== BALANCESHARDUNIQUE Parameters`collection`::-The name of the collection to balance the property in. This parameter is required.+The name of the collection to balance the property in.+This parameter is required.`property`::-The property to balance. The literal `property.` is prepended to this property if not specified explicitly. This parameter is required.+The property to balance.+The literal `property.` is prepended to this property if not specified explicitly.+This parameter is required.`onlyactivenodes`::-Defaults to `true`. Normally, the property is instantiated on active nodes only. If this parameter is specified as `false`, then inactive nodes are also included for distribution.+Defaults to `true`.+Normally, the property is instantiated on active nodes only.+If this parameter is specified as `false`, then inactive nodes are also included for distribution.`shardUnique`::-Something of a safety valve. There is one pre-defined property (`preferredLeader`) that defaults this value to `true`. For all other properties that are balanced, this must be set to `true` or an error message will be returned.+Something of a safety valve.+There is one pre-defined property (`preferredLeader`) that defaults this value to `true`.+For all other properties that are balanced, this must be set to `true` or an error message will be returned.=== BALANCESHARDUNIQUE Response-The response will include the status of the request. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request.+If the status is anything other than "0", an error message will explain why the request failed.=== Examples using BALANCESHARDUNIQUE@@ -434,29 +451,36 @@ We do not currently have a V2 equivalent.=== REPLACENODE Parameters`sourceNode`::-The source node from which the replicas need to be copied from. This parameter is required.+The source node from which the replicas need to be copied from.+This parameter is required.`targetNode`::-The target node where replicas will be copied. If this parameter is not provided, Solr will identify nodes automatically based on policies or number of cores in each node.+The target node where replicas will be copied.+If this parameter is not provided, Solr will identify nodes automatically based on policies or number of cores in each node.`parallel`::-If this flag is set to `true`, all replicas are created in separate threads. Keep in mind that this can lead to very high network and disk I/O if the replicas have very large indices. The default is `false`.+If this flag is set to `true`, all replicas are created in separate threads.+Keep in mind that this can lead to very high network and disk I/O if the replicas have very large indices.+The default is `false`.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.`timeout`::-Time in seconds to wait until new replicas are created, and until leader replicas are fully recovered. The default is `300`, or 5 minutes.+Time in seconds to wait until new replicas are created, and until leader replicas are fully recovered.+The default is `300`, or 5 minutes.[IMPORTANT]====-This operation does not hold necessary locks on the replicas that belong to on the source node. So don't perform other collection operations in this period.+This operation does not hold necessary locks on the replicas that belong to on the source node.+So don't perform other collection operations in this period.====[[deletenode]]== DELETENODE: Delete Replicas in a Node-Deletes all replicas of all collections in that node. Please note that the node itself will remain as a live node after this operation.+Deletes all replicas of all collections in that node.+Please note that the node itself will remain as a live node after this operation.[.dynamic-tabs]--[example.tab-pane#v1deletenode]@@ -482,7 +506,8 @@ We do not currently have a V2 equivalent.=== DELETENODE Parameters`node`::-The node to be removed. This parameter is required.+The node to be removed.+This parameter is required.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -490,11 +515,14 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou[[addrole]]== ADDROLE: Add a Role+Assigns a role to a given node in the cluster.+The only supported role is `overseer`.--Assigns a role to a given node in the cluster. The only supported role is `overseer`.--Use this command to dedicate a particular node as Overseer. Invoke it multiple times to add more nodes. This is useful in large clusters where an Overseer is likely to get overloaded. If available, one among the list of nodes which are assigned the 'overseer' role would become the overseer. The system would assign the role to any other node if none of the designated nodes are up and running.+Use this command to dedicate a particular node as Overseer.+Invoke it multiple times to add more nodes.+This is useful in large clusters where an Overseer is likely to get overloaded.+If available, one among the list of nodes which are assigned the 'overseer' role would become the overseer.+The system would assign the role to any other node if none of the designated nodes are up and running.[.dynamic-tabs]--@@ -530,14 +558,19 @@ curl -X POST http://localhost:8983/api/cluster -H 'Content-Type: application/jso=== ADDROLE Parameters`role`::-The name of the role. The only supported role as of now is `overseer`. This parameter is required.+The name of the role.+The only supported role as of now is `overseer`.+This parameter is required.`node`::-The name of the node that will be assigned the role. It is possible to assign a role even before that node is started. This parameter is started.+The name of the node that will be assigned the role.+It is possible to assign a role even before that node is started.+This parameter is started.=== ADDROLE Response-The response will include the status of the request and the properties that were updated or removed. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request and the properties that were updated or removed.+If the status is anything other than "0", an error message will explain why the request failed.=== Examples using ADDROLE@@ -563,7 +596,8 @@ http://localhost:8983/solr/admin/collections?action=ADDROLE&role=overseer&node=1[[removerole]]== REMOVEROLE: Remove Role-Remove an assigned role. This API is used to undo the roles assigned using ADDROLE operation+Remove an assigned role.+This API is used to undo the roles assigned using ADDROLE operation[.dynamic-tabs]--@@ -599,7 +633,9 @@ curl -X POST http://localhost:8983/api/cluster -H 'Content-Type: application/jso=== REMOVEROLE Parameters`role`::-The name of the role. The only supported role as of now is `overseer`. This parameter is required.+The name of the role.+The only supported role as of now is `overseer`.+This parameter is required.`node`::The name of the node where the role should be removed.@@ -607,7 +643,8 @@ The name of the node where the role should be removed.=== REMOVEROLE Response-The response will include the status of the request and the properties that were updated or removed. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request and the properties that were updated or removed.+If the status is anything other than "0", an error message will explain why the request failed.=== Examples using REMOVEROLEdiff --git a/solr/solr-ref-guide/src/cluster-types.adoc b/solr/solr-ref-guide/src/cluster-types.adocnew file mode 100644index 00000000000..edde3f59373--- /dev/null+++ b/solr/solr-ref-guide/src/cluster-types.adoc@@ -0,0 +1,111 @@+= Solr Cluster Types+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++A Solr cluster is a group of servers (_nodes_) that each run Solr.++There are two general modes of operating a cluster of Solr nodes.+One mode provides central coordination of the Solr nodes (<<SolrCloud Mode>>), while the other allows you to operate a cluster without this central coordination (<<User-Managed Mode>>).++Both modes share general concepts, but ultimately differ in how those concepts are reflected in functionality and features.++First let's cover a few general concepts and then outline the differences between the two modes.++== Cluster Concepts++=== Shards++In both cluster modes, a single logical index can be split across nodes as _shards_.+Each shard contains a subset of overall index.++The number of shards dictates the theoretical limit to the number of documents that can be indexed to Solr.+It also determines the amount of parallelization possible for an individual search request.++=== Replicas++In order to provide some failover for each shard, each shard can be copied as a _replica_.+A replica has the same configuration as the shard and any other replicas for the same index.++It's possible to have replicas without having created shards.+In this case, each replica would be a full copy of the entire index, instead of being only a copy of a part of the entire index.++The number of replicas determines the level of fault tolerance the entire cluster has in the event of a node failure.+It also dictates the theoretical limit on the number of concurrent search requests that can be processed under heavy load.++=== Leaders++Once replicas have been created, a _leader_ must be identified.+The responsibility of the leader is to be a source-of-truth for each replica.+When updates are made to the index, they are first processed by the leader and then by each replica (the exact mechanism for how this happens varies).++The replicas which are not leaders are _followers_.++=== Cores++Each replica, whether it is a leader or a follower, is called a _core_.+Multiple cores can be hosted on any one node.++== SolrCloud Mode++SolrCloud mode (also called "SolrCloud") uses Apache ZooKeeper to provide the centralized cluster management that is its main feature.+ZooKeeper tracks each node of the cluster and the state of each core on each node.++In this mode, configuration files are stored in ZooKeeper and not on the file system of each node.+When configuration changes are made, they must be uploaded to ZooKeeper, which in turn makes sure each node knows changes have been made.++SolrCloud introduces an additional concept, a _collection_.+A collection is the entire group of cores that represent an index: the logical shards and the physical replicas for each shard.+Collections all share the same configurations (schema, `solrconfig.xml`, etc.).+This is an additional centralization of the cluster management, as operations can be performed on the entire collection at one time.++When changes are made to configurations, a single command to reload the collection would automatically reload each individual core that is a member of the collection.++Sharding is handled automatically, simply by telling Solr during collection creation how many shards you'd like the collection to have.+Index updates are then generally balanced between each shard automatically.+Some degree of control over what documents are stored in which shards is also available, if needed.++ZooKeeper also handles load balancing and failover.+Incoming requests, either to index documents or for user queries, can be sent to any node of the cluster and ZooKeeper will route the request to an appropriate replica of each shard.++In SolrCloud, the leader is flexible, with built-in mechanisms for automatic leader election in case of failure in the leader.+This means another core can become the leader, and from that point forward it is the source-of-truth for all replicas.++As long as one replica of each relevant shard is available, a user query or indexing request can still be satisfied when running in SolrCloud mode.++== User-Managed Mode++Solr's user-managed mode requires that cluster coordination activities that SolrCloud uses ZooKeeper for must performed manually or with local scripts.++If the corpus of documents is too large for a single-sharded index, the logic to create shards is entirely left to the user.+There are no automated or programmatic ways for Solr to create shards during indexing.++Routing documents to shards are handled manually, either with a simple hashing system or a simple round-robin list of shards that sends each document to a different shard.+Document updates must be sent to the right shard or duplicate documents could result.++In user-managed mode, the concept of leader and follower becomes critical.+Identifying which node will host the leader replica and which host(s) will be replicas dictate how each node is configured.+In this mode, all index updates are sent to the leader only.+Once the leader has completed indexing, the replica will request the index updates and copy them from the leader.++Load balancing is achieved with an external tool or process, unless request traffic can be managed by the leader or one of its replicas alone.++If the leader goes down, there is no built-in failover mechanism.+A replica could continue to serve queries if the queries were specifically directed to it.+Changing a replica to serve as the leader would require changing `solrconfig.xml` configurations on all replicas and reloading each core.++User-managed mode has no concept of a collection, so for all intents and purposes each Solr node is distinct from other nodes.+Only some configuration parameters keep each node from behaving as independent entities.diff --git a/solr/solr-ref-guide/src/collapse-and-expand-results.adoc b/solr/solr-ref-guide/src/collapse-and-expand-results.adocindex 6f32487a42c..ce39c1cd968 100644--- a/solr/solr-ref-guide/src/collapse-and-expand-results.adoc+++ b/solr/solr-ref-guide/src/collapse-and-expand-results.adoc@@ -22,7 +22,7 @@ The Collapsing query parser groups documents (collapsing the result set) accordi[IMPORTANT]====-In order to use these features with SolrCloud, the documents must be located on the same shard. To ensure document co-location, you can define the `router.name` parameter as `compositeId` when creating the collection. For more information on this option, see the section <<shards-and-indexing-data-in-solrcloud.adoc#document-routing,Document Routing>>.+In order to use these features with SolrCloud, the documents must be located on the same shard. To ensure document co-location, you can define the `router.name` parameter as `compositeId` when creating the collection. For more information on this option, see the section <<solrcloud-shards-indexing.adoc#document-routing,Document Routing>>.====== Collapsing Query Parser@@ -33,7 +33,7 @@ The CollapsingQParserPlugin fully supports the QueryElevationComponent.=== Collapsing Query Parser Options-The CollapsingQParser accepts the following local parameters:+The CollapsingQParser accepts the following local params:`field`::The field that is being collapsed on. The field must be a single valued String, Int or Float-type of field.@@ -73,7 +73,6 @@ The `hint=top_fc` hint is only available when collapsing on String fields. `top_+The default is none.-`size`::Sets the initial size of the collapse data structures when collapsing on a *numeric field only*.+diff --git a/solr/solr-ref-guide/src/collection-management.adoc b/solr/solr-ref-guide/src/collection-management.adocindex 22c1f4c74d2..788e0d5e129 100644--- a/solr/solr-ref-guide/src/collection-management.adoc+++ b/solr/solr-ref-guide/src/collection-management.adoc@@ -70,41 +70,59 @@ curl -X POST http://localhost:8983/api/collections -H 'Content-Type: applicationThe CREATE action allows the following parameters:`name`::-The name of the collection to be created. This parameter is required.+The name of the collection to be created.+This parameter is required.`router.name`::-The router name that will be used. The router defines how documents will be distributed among the shards. Possible values are `implicit` or `compositeId`, which is the default.+The router name that will be used.+The router defines how documents will be distributed among the shards.+Possible values are `implicit` or `compositeId`, which is the default.+-The `implicit` router does not automatically route documents to different shards. Whichever shard you indicate on the indexing request (or within each document) will be used as the destination for those documents.+The `implicit` router does not automatically route documents to different shards.+Whichever shard you indicate on the indexing request (or within each document) will be used as the destination for those documents.+The `compositeId` router hashes the value in the uniqueKey field and looks up that hash in the collection's clusterstate to determine which shard will receive the document, with the additional ability to manually direct the routing.+-When using the `implicit` router, the `shards` parameter is required. When using the `compositeId` router, the `numShards` parameter is required.+When using the `implicit` router, the `shards` parameter is required.+When using the `compositeId` router, the `numShards` parameter is required.+-For more information, see also the section <<shards-and-indexing-data-in-solrcloud.adoc#document-routing,Document Routing>>.+For more information, see also the section <<solrcloud-shards-indexing.adoc#document-routing,Document Routing>>.`numShards`::-The number of shards to be created as part of the collection. This is a required parameter when the `router.name` is `compositeId`.+The number of shards to be created as part of the collection.+This is a required parameter when the `router.name` is `compositeId`.`shards`::-A comma separated list of shard names, e.g., `shard-x,shard-y,shard-z`. This is a required parameter when the `router.name` is `implicit`.+A comma separated list of shard names, e.g., `shard-x,shard-y,shard-z`.+This is a required parameter when the `router.name` is `implicit`.`replicationFactor`::-The number of replicas to be created for each shard. The default is `1`.+The number of replicas to be created for each shard.+The default is `1`.+-This will create a NRT type of replica. If you want another type of replica, see the `tlogReplicas` and `pullReplica` parameters below. See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.+This will create a NRT type of replica.+If you want another type of replica, see the `tlogReplicas` and `pullReplica` parameters below.+See the section <<solrcloud-shards-indexing.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.`nrtReplicas`::-The number of NRT (Near-Real-Time) replicas to create for this collection. This type of replica maintains a transaction log and updates its index locally. If you want all of your replicas to be of this type, you can simply use `replicationFactor` instead.+The number of NRT (Near-Real-Time) replicas to create for this collection.+This type of replica maintains a transaction log and updates its index locally.+If you want all of your replicas to be of this type, you can simply use `replicationFactor` instead.`tlogReplicas`::-The number of TLOG replicas to create for this collection. This type of replica maintains a transaction log but only updates its index via replication from a leader. See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.+The number of TLOG replicas to create for this collection.+This type of replica maintains a transaction log but only updates its index via replication from a leader.+See the section <<solrcloud-shards-indexing.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.`pullReplicas`::-The number of PULL replicas to create for this collection. This type of replica does not maintain a transaction log and only updates its index via replication from a leader. This type is not eligible to become a leader and should not be the only type of replicas in the collection. See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.+The number of PULL replicas to create for this collection.+This type of replica does not maintain a transaction log and only updates its index via replication from a leader.+This type is not eligible to become a leader and should not be the only type of replicas in the collection.+See the section <<solrcloud-shards-indexing.adoc#types-of-replicas,Types of Replicas>> for more information about replica types.`createNodeSet` (v1), `nodeSet` (v2)::-Allows defining the nodes to spread the new collection across. The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`.+Allows defining the nodes to spread the new collection across.+The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`.+If not provided, the CREATE operation will create shard-replicas spread across all live Solr nodes.+@@ -113,7 +131,8 @@ Alternatively, use the special value of `EMPTY` to initially create no shard-rep`createNodeSet.shuffle` (v1), `shuffleNodes` (v2)::Controls whether or not the shard-replicas created for this collection will be assigned to the nodes specified by the `createNodeSet` in a sequential manner, or if the list of nodes should be shuffled prior to creating individual replicas.+-A `false` value makes the results of a collection creation predictable and gives more exact control over the location of the individual shard-replicas, but `true` can be a better choice for ensuring replicas are distributed evenly across nodes. The default is `true`.+A `false` value makes the results of a collection creation predictable and gives more exact control over the location of the individual shard-replicas, but `true` can be a better choice for ensuring replicas are distributed evenly across nodes.+The default is `true`.+This parameter is ignored if `createNodeSet` is not also specified.@@ -124,7 +143,8 @@ If not provided, Solr will use the configuration of `_default` configset to creaWhen such a collection is deleted, its autocreated configset will be deleted by default when it is not in use by any other collection.`router.field` (v1), `router` (v2)::-If this parameter is specified, the router will look at the value of the field in an input document to compute the hash and identify a shard instead of looking at the `uniqueKey` field. If the field specified is null in the document, the document will be rejected.+If this parameter is specified, the router will look at the value of the field in an input document to compute the hash and identify a shard instead of looking at the `uniqueKey` field.+If the field specified is null in the document, the document will be rejected.+Please note that <<realtime-get.adoc#,RealTime Get>> or retrieval by document ID would also require the parameter `\_route_` (or `shard.keys`) to avoid a distributed search.@@ -132,37 +152,43 @@ Please note that <<realtime-get.adoc#,RealTime Get>> or retrieval by document IDIf `true` the states of individual replicas will be maintained as individual child of the `state.json`. The default is `false`.`property._name_=_value_`::-Set core property _name_ to _value_. See the section <<defining-core-properties.adoc#,Defining core.properties>> for details on supported properties and values.+Set core property _name_ to _value_. See the section <<core-discovery.adoc#,Core Discovery>> for details on supported properties and values.[WARNING]====-The entries in each core.properties file are vital for Solr to function correctly. Overriding entries can result in unusable collections. Altering these entries by specifying `property._name_=_value_` is an expert-level option and should only be used if you have a thorough understanding of the consequences.+The entries in each core.properties file are vital for Solr to function correctly.+Overriding entries can result in unusable collections.+Altering these entries by specifying `property._name_=_value_` is an expert-level option and should only be used if you have a thorough understanding of the consequences.====`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.`waitForFinalState`::-If `true`, the request will complete only when all affected replicas become active. The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.+If `true`, the request will complete only when all affected replicas become active.+The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.`alias`::Starting with version 8.1 when a collection is created additionally an alias can be created-that points to this collection. This parameter allows specifying the name of this alias, effectively combining-this operation with <<collection-aliasing.adoc#createalias,CREATEALIAS>>+that points to this collection.+This parameter allows specifying the name of this alias, effectively combining+this operation with <<alias-management.adoc#createalias,CREATEALIAS>>Collections are first created in read-write mode but can be put in `readOnly`mode using the <<collection-management.adoc#modifycollection,MODIFYCOLLECTION>> action.=== CREATE Response-The response will include the status of the request and the new core names. If the status is anything other than "success", an error message will explain why the request failed.+The response will include the status of the request and the new core names.+If the status is anything other than "success", an error message will explain why the request failed.[[reload]]== RELOAD: Reload a CollectionThe RELOAD action is used when you have changed a configuration file in ZooKeeper, like uploading a new `schema.xml`.Solr automatically reloads collections when certain files, monitored via a watch in ZooKeeper are changed,-such as `security.json`. However, for changes to files in configsets, like uploading a new `schema.xml`, you+such as `security.json`.+ However, for changes to files in configsets, like uploading a new `schema.xml`, youwill need to manually trigger the RELOAD.[.dynamic-tabs]@@ -213,21 +239,26 @@ curl -X POST http://localhost:8983/api/collections/techproducts_v2 -H 'Content-T=== RELOAD Parameters`name`::-The name of the collection to reload. This parameter is required by the V1 API.+The name of the collection to reload.+This parameter is required by the V1 API.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.=== RELOAD Response-The response will include the status of the request and the cores that were reloaded. If the status is anything other than "success", an error message will explain why the request failed.+The response will include the status of the request and the cores that were reloaded.+If the status is anything other than "success", an error message will explain why the request failed.[[modifycollection]]== MODIFYCOLLECTION: Modify Attributes of a Collection-It's possible to edit multiple attributes at a time. Changing these values only updates the znode on ZooKeeper, they do not change the topology of the collection. For instance, increasing `replicationFactor` will _not_ automatically add more replicas to the collection but _will_ allow more ADDREPLICA commands to succeed.+It's possible to edit multiple attributes at a time.+Changing these values only updates the znode on ZooKeeper, they do not change the topology of the collection.+For instance, increasing `replicationFactor` will _not_ automatically add more replicas to the collection but _will_ allow more ADDREPLICA commands to succeed.-An attribute can be deleted by passing an empty value. For example, `yet_another_attribute_name=` (with no value) will delete the `yet_another_attribute_name` parameter from the collection.+An attribute can be deleted by passing an empty value.+For example, `yet_another_attribute_name=` (with no value) will delete the `yet_another_attribute_name` parameter from the collection.[.dynamic-tabs]--@@ -268,7 +299,8 @@ curl -X POST http://localhost:8983/api/collections/techproducts_v2 -H 'Content-T=== MODIFYCOLLECTION Parameters`collection`::-The name of the collection to be modified. This parameter is required.+The name of the collection to be modified.+This parameter is required.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -290,7 +322,8 @@ See the <<create,CREATE action>> section above for details on these attributes.[[readonlymode]]==== Read-Only ModeSetting the `readOnly` attribute to `true` puts the collection in read-only mode,-in which any index update requests are rejected. Other collection-level actions (e.g., adding /+in which any index update requests are rejected.+Other collection-level actions (e.g., adding /removing / moving replicas) are still available in this mode.The transition from the (default) read-write to read-only mode consists of the following steps:@@ -361,7 +394,8 @@ using the new name.This command does NOT actually rename the underlying Solr collection - it sets up a new one-to-one aliasusing the new name, or renames the existing alias so that it uses the new name, while still referring to-the same underlying Solr collection. However, from the user's point of view the collection can now be+the same underlying Solr collection.+However, from the user's point of view the collection can now beaccessed using the new name, and the new name can be also referred to in other aliases.The following limitations apply:@@ -400,9 +434,11 @@ Name of the existing SolrCloud collection or an alias that refers to exactly onea Routed Alias.`target`::-Target name of the collection. This will be the new alias that refers to the underlying SolrCloud collection.+Target name of the collection.+This will be the new alias that refers to the underlying SolrCloud collection.The original name (or alias) of the collection will be replaced also in the existing aliases so that they-also refer to the new name. Target name must not be an existing alias.+also refer to the new name.+Target name must not be an existing alias.=== Examples using RENAMEAssuming there are two actual SolrCloud collections named `collection1` and `collection2`,@@ -470,14 +506,16 @@ curl -X DELETE http://localhost:8983/api/collections/techproducts_v2?async=aaaa=== DELETE Parameters`name`::-The name of the collection to delete. This parameter is required.+The name of the collection to delete.+This parameter is required.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.=== DELETE Response-The response will include the status of the request and the cores that were deleted. If the status is anything other than "success", an error message will explain why the request failed.+The response will include the status of the request and the cores that were deleted.+If the status is anything other than "success", an error message will explain why the request failed.*Output*@@ -552,16 +590,20 @@ The name of the collection for which the property would be set.The name of the property.`propertyValue` (v1), `value` (v2)::-The value of the property. When not provided, the property is deleted.+The value of the property.+When not provided, the property is deleted.=== COLLECTIONPROP Response-The response will include the status of the request and the properties that were updated or removed. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request and the properties that were updated or removed.+If the status is anything other than "0", an error message will explain why the request failed.[[migrate]]== MIGRATE: Migrate Documents to Another Collection-The MIGRATE command is used to migrate all documents having a given routing key to another collection. The source collection will continue to have the same data as-is but it will start re-routing write requests to the target collection for the number of seconds specified by the `forward.timeout` parameter. It is the responsibility of the user to switch to the target collection for reads and writes after the MIGRATE action completes.+The MIGRATE command is used to migrate all documents having a given routing key to another collection.+The source collection will continue to have the same data as-is but it will start re-routing write requests to the target collection for the number of seconds specified by the `forward.timeout` parameter.+It is the responsibility of the user to switch to the target collection for reads and writes after the MIGRATE action completes.[.dynamic-tabs]--@@ -595,30 +637,40 @@ curl -X POST http://localhost:8983/api/collections/techproducts_v2 -H 'Content-T---The routing key specified by the `split.key` parameter may span multiple shards on both the source and the target collections. The migration is performed shard-by-shard in a single thread. One or more temporary collections may be created by this command during the ‘migrate’ process but they are cleaned up at the end automatically.+The routing key specified by the `split.key` parameter may span multiple shards on both the source and the target collections.+The migration is performed shard-by-shard in a single thread.+One or more temporary collections may be created by this command during the ‘migrate’ process but they are cleaned up at the end automatically.-This is a long running operation and therefore using the `async` parameter is highly recommended. If the `async` parameter is not specified then the operation is synchronous by default and keeping a large read timeout on the invocation is advised. Even with a large read timeout, the request may still timeout but that doesn’t necessarily mean that the operation has failed. Users should check logs, cluster state, source and target collections before invoking the operation again.+This is a long running operation and therefore using the `async` parameter is highly recommended.+If the `async` parameter is not specified then the operation is synchronous by default and keeping a large read timeout on the invocation is advised.+Even with a large read timeout, the request may still timeout but that doesn’t necessarily mean that the operation has failed.+Users should check logs, cluster state, source and target collections before invoking the operation again.-This command works only with collections using the compositeId router. The target collection must not receive any writes during the time the MIGRATE command is running otherwise some writes may be lost.+This command works only with collections using the compositeId router.+The target collection must not receive any writes during the time the MIGRATE command is running otherwise some writes may be lost.Please note that the MIGRATE API does not perform any de-duplication on the documents so if the target collection contains documents with the same uniqueKey as the documents being migrated then the target collection will end up with duplicate documents.=== MIGRATE Parameters`collection`::-The name of the source collection from which documents will be split. This parameter is required.+The name of the source collection from which documents will be split.+This parameter is required.`target.collection` (v1), `target` (v2)::-The name of the target collection to which documents will be migrated. This parameter is required.+The name of the target collection to which documents will be migrated.+This parameter is required.`split.key` (v1), `splitKey` (v2)::-The routing key prefix. For example, if the uniqueKey of a document is "a!123", then you would use `split.key=a!`. This parameter is required.+The routing key prefix.+For example, if the uniqueKey of a document is "a!123", then you would use `split.key=a!`. This parameter is required.`forward.timeout` (v1), `forwardTimeout` (v2)::-The timeout, in seconds, until which write requests made to the source collection for the given `split.key` will be forwarded to the target shard. The default is 60 seconds.+The timeout, in seconds, until which write requests made to the source collection for the given `split.key` will be forwarded to the target shard.+The default is 60 seconds.`property._name_=_value_`::-Set core property _name_ to _value_. See the section <<defining-core-properties.adoc#,Defining core.properties>> for details on supported properties and values.+Set core property _name_ to _value_. See the section <<core-discovery.adoc#,Core Discovery>> for details on supported properties and values.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -661,73 +713,72 @@ with caution, evaluating the potential impact by using different source and targcollection names first, and preserving the source collection until the evaluation iscomplete.-The target collection must not exist (and may not be an alias). If the target-collection name is the same as the source collection then first a unique sequential name-will be generated for the target collection, and then after reindexing is done an alias-will be created that points from the source name to the actual sequentially-named target collection.+The target collection must not exist (and may not be an alias).+If the target collection name is the same as the source collection then first a unique sequential name will be generated for the target collection, and then after reindexing is done an alias will be created that points from the source name to the actual sequentially-named target collection.-When reindexing is started the source collection is put in <<readonlymode,read-only mode>> to ensure that-all source documents are properly processed.+When reindexing is started the source collection is put in <<readonlymode,read-only mode>> to ensure that all source documents are properly processed.-Using optional parameters a different index schema, collection shape (number of shards and replicas)-or routing parameters can be requested for the target collection.+Using optional parameters a different index schema, collection shape (number of shards and replicas) or routing parameters can be requested for the target collection.-Reindexing is executed as a streaming expression daemon, which runs on one of the-source collection's replicas. It is usually a time-consuming operation so it's recommended to execute-it as an asynchronous request in order to avoid request time outs. Only one reindexing operation may-execute concurrently for a given source collection. Long-running, erroneous or crashed reindexing-operations may be terminated by using the `abort` option, which also removes partial results.+Reindexing is executed as a streaming expression daemon, which runs on one of the source collection's replicas.+It is usually a time-consuming operation so it's recommended to execute it as an asynchronous request in order to avoid request time outs.+Only one reindexing operation may execute concurrently for a given source collection.+Long-running, erroneous or crashed reindexing operations may be terminated by using the `abort` option, which also removes partial results.=== REINDEXCOLLECTION Parameters`name`::-Source collection name, may be an alias. This parameter is required.+Source collection name, may be an alias.+This parameter is required.`cmd`::-Optional command. Default command is `start`. Currently supported commands are:+Optional command.+Default command is `start`.+Currently supported commands are:* `start` - default, starts processing if not already running,* `abort` - aborts an already running reindexing (or clears a left-over status after a crash),and deletes partial results,* `status` - returns detailed status of a running reindexing command.`target`::-Target collection name, optional. If not specified a unique name will be generated and-after all documents have been copied an alias will be created that points from the source-collection name to the unique sequentially-named collection, effectively "hiding"+Target collection name, optional.+If not specified a unique name will be generated and after all documents have been copied an alias will be created that points from the source collection name to the unique sequentially-named collection, effectively "hiding"the original source collection from regular update and search operations.`q`::-Optional query to select documents for reindexing. Default value is `\*:*`.+Optional query to select documents for reindexing.+Default value is `\*:*`.`fl`::-Optional list of fields to reindex. Default value is `*`.+Optional list of fields to reindex.+Default value is `*`.`rows`::-Documents are transferred in batches. Depending on the average size of the document large-batch sizes may cause memory issues. Default value is 100.+Documents are transferred in batches.+Depending on the average size of the document large batch sizes may cause memory issues.+Default value is 100.`configName`::`collection.configName`::-Optional name of the configset for the target collection. Default is the same as the-source collection.+Optional name of the configset for the target collection.+Default is the same as the source collection.-There's a number of optional parameters that determine the target collection layout. If they-are not specified in the request then their values are copied from the source collection.+There's a number of optional parameters that determine the target collection layout.+If they are not specified in the request then their values are copied from the source collection.The following parameters are currently supported (described in detail in the <<create,CREATE collection>> section):`numShards`, `replicationFactor`, `nrtReplicas`, `tlogReplicas`, `pullReplicas`,`shards`, `policy`, `createNodeSet`, `createNodeSet.shuffle`, `router.*`.`removeSource`::-Optional boolean. If true then after the processing is successfully finished the source collection will-be deleted.+Optional boolean.+If true then after the processing is successfully finished the source collection will be deleted.`async`::Optional request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.When the reindexing process has completed the target collection is marked using`property.rx: "finished"`, and the source collection state is updated to become read-write.-On any errors the command will delete any temporary and target collections and also reset the-state of the source collection's read-only flag.+On any errors the command will delete any temporary and target collections and also reset the state of the source collection's read-only flag.=== Examples using REINDEXCOLLECTION@@ -737,11 +788,9 @@ state of the source collection's read-only flag.----http://localhost:8983/solr/admin/collections?action=REINDEXCOLLECTION&name=techproducts_v2&numShards=3&configName=conf2&q=id:aa*&fl=id,string_s-----This request specifies a different schema for the target collection, copies only some of the fields, selects only the documents-matching a query, and also potentially re-shapes the collection by explicitly specifying 3 shards. Since the target collection-hasn't been specified in the parameters, a collection with a unique name, e.g., `.rx_techproducts_v2_2`, will be created and on success-an alias pointing from `techproducts_v2` to `.rx_techproducts_v2_2` will be created, effectively replacing the source collection-for the purpose of indexing and searching. The source collection is assumed to be small so a synchronous request was made.+This request specifies a different schema for the target collection, copies only some of the fields, selects only the documents matching a query, and also potentially re-shapes the collection by explicitly specifying 3 shards.+Since the target collection hasn't been specified in the parameters, a collection with a unique name, e.g., `.rx_techproducts_v2_2`, will be created and on success an alias pointing from `techproducts_v2` to `.rx_techproducts_v2_2` will be created, effectively replacing the source collection for the purpose of indexing and searching.+The source collection is assumed to be small so a synchronous request was made.*Output*@@ -762,15 +811,14 @@ for the purpose of indexing and searching. The source collection is assumed to b}}-----As a result a new collection `.rx_techproducts_v2_2` has been created, with selected documents reindexed to 3 shards, and-with an alias pointing from `techproducts_v2` to this one. The status also shows that the source collection-was already an alias to `.rx_techproducts_v2_1`, which was likely a result of a previous reindexing.+As a result a new collection `.rx_techproducts_v2_2` has been created, with selected documents reindexed to 3 shards, and with an alias pointing from `techproducts_v2` to this one.+The status also shows that the source collection was already an alias to `.rx_techproducts_v2_1`, which was likely a result of a previous reindexing.[[colstatus]]== COLSTATUS: Detailed Status of a Collection's Indexes-The COLSTATUS command provides a detailed description of the collection status, including low-level index-information about segments and field data. There isn't a good equivalent V2 API that supports all the parameters below.+The COLSTATUS command provides a detailed description of the collection status, including low-level index information about segments and field data.+There isn't a good equivalent V2 API that supports all the parameters below.[.dynamic-tabs]--@@ -797,13 +845,8 @@ curl -X GET http://localhost:8983/api/collections/techproducts_v2====---This command also checks the compliance of Lucene index field types with the current Solr collection-schema and indicates the names of non-compliant fields, i.e., Lucene fields with field types incompatible-(or different) from the corresponding Solr field types declared in the current schema. Such incompatibilities may-result from incompatible schema changes or after migration of-data to a different major Solr release.--+This command also checks the compliance of Lucene index field types with the current Solr collection schema and indicates the names of non-compliant fields, i.e., Lucene fields with field types incompatible (or different) from the corresponding Solr field types declared in the current schema.+Such incompatibilities may result from incompatible schema changes or after migration of data to a different major Solr release.=== COLSTATUS Parameters@@ -811,62 +854,64 @@ data to a different major Solr release.Collection name (optional). If missing then it means all collections.`coreInfo`::-Optional boolean. If true then additional information will be provided about+Optional boolean.+If true then additional information will be provided aboutSolrCore of shard leaders.`segments`::-Optional boolean. If true then segment information will be provided.+Optional boolean.+If true then segment information will be provided.`fieldInfo`::-Optional boolean. If true then detailed Lucene field information will be provided+Optional boolean.+If true then detailed Lucene field information will be providedand their corresponding Solr schema types.`sizeInfo`::-Optional boolean. If true then additional information about the index files+Optional boolean.+If true then additional information about the index filessize and their RAM usage will be provided.==== Index Size Analysis Tool-The `COLSTATUS` command also provides a tool for analyzing and estimating the composition of raw index data. Please note that-this tool should be used with care because it generates a significant IO load on all shard leaders of the-analyzed collections. A sampling threshold and a sampling percent parameters can be adjusted to reduce this-load to some degree.+The `COLSTATUS` command also provides a tool for analyzing and estimating the composition of raw index data.+Please note that this tool should be used with care because it generates a significant IO load on all shard leaders of the analyzed collections.+A sampling threshold and a sampling percent parameters can be adjusted to reduce this load to some degree.-Size estimates produced by this tool are only approximate and represent the aggregated size of uncompressed-index data. In reality these values would never occur, because Lucene (and Solr) always stores data in a-compressed format - still, these values help to understand what occupies most of the space and the relative size-of each type of data and each field in the index.+Size estimates produced by this tool are only approximate and represent the aggregated size of uncompressed index data.+In reality these values would never occur, because Lucene (and Solr) always stores data in a compressed format - still, these values help to understand what occupies most of the space and the relative size of each type of data and each field in the index.-In the following sections whenever "size" is mentioned it means an estimated aggregated size of-uncompressed (raw) data.+In the following sections whenever "size" is mentioned it means an estimated aggregated size of uncompressed (raw) data.The following parameters are specific to this tool:`rawSize`::-Optional boolean. If true then run the raw index data analysis tool (other boolean options below imply-this option if any of them are true). Command response will include sections that show estimated breakdown of-data size per field and per data type.+Optional boolean.+If `true` then run the raw index data analysis tool (other boolean options below imply this option if any of them are true).+Command response will include sections that show estimated breakdown of data size per field and per data type.`rawSizeSummary`::-Optional boolean. If true then include also a more detailed breakdown of data size per field and per type.+Optional boolean.+If `true` then include also a more detailed breakdown of data size per field and per type.`rawSizeDetails`::-Optional boolean. If true then provide exhaustive details that include statistical distribution of items per-field and per type as well as top 20 largest items per field.+Optional boolean.+If `true` then provide exhaustive details that include statistical distribution of items per field and per type as well as top 20 largest items per field.`rawSizeSamplingPercent`::-Optional float. When the index is larger than a certain threshold (100k documents per shard) only a part of-data is actually retrieved and analyzed in order to reduce the IO load, and then the final results are extrapolated.-Values must be greater than 0 and less or equal to 100.0. Default value is 5.0. Very small values (between 0.0 and 1.0)-may introduce significant estimation errors. Also, values that would result in less than 10 documents being sampled-are rejected with an exception.+Optional float.+When the index is larger than a certain threshold (100k documents per shard) only a part of data is actually retrieved and analyzed in order to reduce the IO load, and then the final results are extrapolated.+Values must be greater than 0 and less or equal to 100.0.+Default value is `5.0`.+Very small values (between 0.0 and 1.0) may introduce significant estimation errors.+Also, values that would result in less than 10 documents being sampled are rejected with an exception.Response for this command always contains two sections:-* `fieldsBySize` is a map where field names are keys and values are estimated sizes of raw (uncompressed) data-that belongs to the field. The map is sorted by size so that it's easy to see what field occupies most space.+* `fieldsBySize` is a map where field names are keys and values are estimated sizes of raw (uncompressed) data that belongs to the field.+The map is sorted by size so that it's easy to see what field occupies most space.-* `typesBySize` is a map where data types are the keys and values are estimates sizes of raw (uncompressed) data-of particular type. This map is also sorted by size.+* `typesBySize` is a map where data types are the keys and values are estimates sizes of raw (uncompressed) data of particular type.+This map is also sorted by size.Optional sections include:@@ -877,19 +922,19 @@ This section also shows `topN` values by size from each field.Data types shown in the response can be roughly divided into the following groups:-* `storedFields` - represents the raw uncompressed data in stored fields. For example, for UTF-8 strings this represents-the aggregated sum of the number of bytes in the strings' UTF-8 representation, for long numbers this is 8 bytes per value, etc.+* `storedFields` - represents the raw uncompressed data in stored fields.+For example, for UTF-8 strings this represents the aggregated sum of the number of bytes in the strings' UTF-8 representation, for long numbers this is 8 bytes per value, etc.-* `terms_terms` - represents the aggregated size of the term dictionary. The size of this data is affected by the-the number and length of unique terms, which in turn depends on the field size and the analysis chain.+* `terms_terms` - represents the aggregated size of the term dictionary.+The size of this data is affected by the the number and length of unique terms, which in turn depends on the field size and the analysis chain.* `terms_postings` - represents the aggregated size of all term position and offset information, if present.This information may be absent if position-based searching, such as phrase queries, is not needed.* `terms_payloads` - represents the aggregated size of all per-term payload data, if present.-* `norms` - represents the aggregated size of field norm information. This information may be omitted if a field-has an `omitNorms` flag in the schema, which is common for fields that don't need weighting or scoring by field length.+* `norms` - represents the aggregated size of field norm information.+This information may be omitted if a field has an `omitNorms` flag in the schema, which is common for fields that don't need weighting or scoring by field length.* `termVectors` - represents the aggregated size of term vectors.@@ -1250,7 +1295,9 @@ curl -X POST http://localhost:8983/api/collections -H 'Content-Type: application====---The BACKUP command will backup Solr indexes and configurations for a specified collection. The BACKUP command <<making-and-restoring-backups.adoc#,takes one copy from each shard for the indexes>>. For configurations, it backs up the configset that was associated with the collection and metadata.+The BACKUP command will backup Solr indexes and configurations for a specified collection.+The BACKUP command <<backup-restore.adoc#,takes one copy from each shard for the indexes>>.+For configurations, it backs up the configset that was associated with the collection and metadata.Backup data is stored in the repository based on the provided `name` and `location`.Each backup location can hold multiple backups for the same collection, allowing users to later restore from any of these "backup points" as desired.@@ -1266,24 +1313,30 @@ See the `incremental` parameter below for more information.=== BACKUP Parameters`collection`::-The name of the collection to be backed up. This parameter is required.+The name of the collection to be backed up.+This parameter is required.`name`::-What to name the backup that is created. This is checked to make sure it doesn't already exist, and otherwise an error message is raised. This parameter is required.+What to name the backup that is created.+This is checked to make sure it doesn't already exist, and otherwise an error message is raised.+This parameter is required.`location`::-The location on a shared drive for the backup command to write to. This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+The location on a shared drive for the backup command to write to.+This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+If the location path is on a mounted drive, the mount must be available on the node that serves as the overseer, even if the overseer node does not host a replica of the collection being backed up.Since any node can take the overseer role at any time, a best practice to avoid possible backup failures is to ensure the mount point is available on all nodes of the cluster.+-Each backup location can only hold a backup for one collection, however the same location can be used for repeated backups of the same collection. Repeated backups of the same collection are done incrementally, so that files unchanged since the last backup are not duplicated in the backup repository.+Each backup location can only hold a backup for one collection, however the same location can be used for repeated backups of the same collection.+Repeated backups of the same collection are done incrementally, so that files unchanged since the last backup are not duplicated in the backup repository.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.`repository`::-The name of a repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.+The name of a repository to be used for the backup.+If no repository is specified then the local filesystem repository will be used automatically.`maxNumBackupPoints`::The upper-bound on how many backups should be retained at the backup location.@@ -1320,7 +1373,8 @@ The backup name usually corresponds to the collection-name, but isn't required tThis parameter is required.`location`::-The repository location to list backups from. This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+The repository location to list backups from.+This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+If the location path is on a mounted drive, the mount must be available on the node that serves as the overseer, even if the overseer node does not host a replica of the collection being backed up.Since any node can take the overseer role at any time, a best practice to avoid possible backup failures is to ensure the mount point is available on all nodes of the cluster.@@ -1442,33 +1496,40 @@ The RESTORE operation will replace the content of a collection with files from tIf the provided `collection` value matches an existing collection, Solr will use it for restoration, assuming it is compatible (same number of shards, etc.) with the stored backup files.If the provided `collection` value doesn't exist, a new collection with that name is created in a way compatible with the stored backup files.-The collection created will be have the same number of shards and replicas as the original collection, preserving routing information, etc. Optionally, you can override some parameters documented below.+The collection created will be have the same number of shards and replicas as the original collection, preserving routing information, etc.+Optionally, you can override some parameters documented below.While restoring, if a configset with the same name exists in ZooKeeper then Solr will reuse that, or else it will upload the backed up configset in ZooKeeper and use that.-You can use the collection <<collection-aliasing.adoc#createalias,CREATEALIAS>> command to make sure clients don't need to change the endpoint to query or index against the newly restored collection.+You can use the collection <<alias-management.adoc#createalias,CREATEALIAS>> command to make sure clients don't need to change the endpoint to query or index against the newly restored collection.=== RESTORE Parameters`collection`::-The collection where the indexes will be restored into. This parameter is required.+The collection where the indexes will be restored into.+This parameter is required.`name`::-The name of the existing backup that you want to restore. This parameter is required.+The name of the existing backup that you want to restore.+This parameter is required.`location`::-The location on a shared drive for the RESTORE command to read from. Alternately it can be set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+The location on a shared drive for the RESTORE command to read from.+Alternately it can be set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.`repository`::-The name of a repository to be used for the backup. If no repository is specified then the local filesystem repository will be used automatically.+The name of a repository to be used for the backup.+If no repository is specified then the local filesystem repository will be used automatically.`backupId`::The ID of a specific backup point to restore from.+-Backup locations can hold multiple backups of the same collection. This parameter allows users to choose which of those backups should be used to restore from. If not specified the most recent backup point is used.+Backup locations can hold multiple backups of the same collection.+This parameter allows users to choose which of those backups should be used to restore from.+If not specified the most recent backup point is used.There are also optional parameters that determine the target collection layout.The following parameters are currently supported (described in detail in the <<create,CREATE collection>> section):@@ -1556,16 +1617,19 @@ curl -X POST http://localhost:8983/v2/collections/backups -H 'Content-Type: appl=== DELETEBACKUP Parameters`name`::-The backup name to delete backup files from. This parameter is required.+The backup name to delete backup files from.+This parameter is required.`location`::-The repository location to delete backups from. This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+The repository location to delete backups from.+This parameter is required, unless a default location is defined on the repository configuration, or set as a <<cluster-node-management.adoc#clusterprop,cluster property>>.+If the location path is on a mounted drive, the mount must be available on the node that serves as the overseer, even if the overseer node does not host a replica of the collection being backed up.Since any node can take the overseer role at any time, a best practice to avoid possible backup failures is to ensure the mount point is available on all nodes of the cluster.`repository`::-The name of a repository to be used for deleting backup files. If no repository is specified then the local filesystem repository will be used automatically.+The name of a repository to be used for deleting backup files.+If no repository is specified then the local filesystem repository will be used automatically.`backupId`::Explicitly specify a single backup-ID to delete.@@ -1621,28 +1685,35 @@ curl -X POST http://localhost:8983/api/collections/techproducts -H 'Content-Type====---Leaders are assigned in a collection according to the `preferredLeader` property on active nodes. This command should be run after the preferredLeader property has been assigned via the BALANCESHARDUNIQUE or ADDREPLICAPROP commands.+Leaders are assigned in a collection according to the `preferredLeader` property on active nodes.+This command should be run after the preferredLeader property has been assigned via the BALANCESHARDUNIQUE or ADDREPLICAPROP commands.-NOTE: It is not _required_ that all shards in a collection have a `preferredLeader` property. Rebalancing will only attempt to reassign leadership to those replicas that have the `preferredLeader` property set to `true` _and_ are not currently the shard leader _and_ are currently active.+NOTE: It is not _required_ that all shards in a collection have a `preferredLeader` property.+Rebalancing will only attempt to reassign leadership to those replicas that have the `preferredLeader` property set to `true` _and_ are not currently the shard leader _and_ are currently active.=== REBALANCELEADERS Parameters`collection`::-The name of the collection to rebalance `preferredLeaders` on. This parameter is required.+The name of the collection to rebalance `preferredLeaders` on.+This parameter is required.`maxAtOnce`::-The maximum number of reassignments to have queue up at once. Values \<=0 are use the default value Integer.MAX_VALUE.+The maximum number of reassignments to have queue up at once.+Values \<=0 are use the default value Integer.MAX_VALUE.+When this number is reached, the process waits for one or more leaders to be successfully assigned before adding more to the queue.`maxWaitSeconds`::-Defaults to `60`. This is the timeout value when waiting for leaders to be reassigned. If `maxAtOnce` is less than the number of reassignments that will take place, this is the maximum interval that any _single_ wait for at least one reassignment.+Defaults to `60`. This is the timeout value when waiting for leaders to be reassigned.+If `maxAtOnce` is less than the number of reassignments that will take place, this is the maximum interval that any _single_ wait for at least one reassignment.+For example, if 10 reassignments are to take place and `maxAtOnce` is `1` and `maxWaitSeconds` is `60`, the upper bound on the time that the command may wait is 10 minutes.=== REBALANCELEADERS Response-The response will include the status of the request. A status of "0" indicates the request was _processed_, not that all assignments were successful. Examine the "Summary" section for that information.+The response will include the status of the request.+A status of "0" indicates the request was _processed_, not that all assignments were successful.+Examine the "Summary" section for that information.=== Examples using REBALANCELEADERS@@ -1662,10 +1733,12 @@ http://localhost:8983/solr/admin/collections?action=REBALANCELEADERS&collection=In this example:* In the "alreadyLeaders" section, core_node5 was already the leader, so there were no changes in leadership for shard1.-* In the "inactivePreferreds" section, core_node57 had the preferredLeader property set, but the node was not active, the leader for shard7 was not changed. This is considered successful.+* In the "inactivePreferreds" section, core_node57 had the preferredLeader property set, but the node was not active, the leader for shard7 was not changed.+This is considered successful.* In the "successes" section, core_node23 was _not_ the leader for shard3, so leadership was assigned to that replica.-The "Summary" section with the "Success" tag indicates that the command rebalanced all _active_ replicas with the preferredLeader property set as requried. If a replica cannot be made leader due to not being healthy (for example, it is on a Solr instance that is not running), it's also considered success.+The "Summary" section with the "Success" tag indicates that the command rebalanced all _active_ replicas with the preferredLeader property set as requried.+If a replica cannot be made leader due to not being healthy (for example, it is on a Solr instance that is not running), it's also considered success.[source,json]----@@ -1691,6 +1764,8 @@ The "Summary" section with the "Success" tag indicates that the command rebalancExamining the clusterstate after issuing this call should show that every active replica that has the `preferredLeader` property should also have the "leader" property set to _true_.-NOTE: The added work done by an NRT leader is quite small and only present when indexing. The primary use-case is to redistribute the leader role if there are a large number of leaders concentrated on a small number of nodes. Rebalancing will likely not improve performance unless the imbalance of leadership roles is measured in multiples of 10.+NOTE: The added work done by an NRT leader is quite small and only present when indexing.+The primary use-case is to redistribute the leader role if there are a large number of leaders concentrated on a small number of nodes.+Rebalancing will likely not improve performance unless the imbalance of leadership roles is measured in multiples of 10.NOTE: The BALANCESHARDUNIQUE command that distributes the preferredLeader property does not guarantee perfect distribution and in some collection topologies it is impossible to make that guarantee.diff --git a/solr/solr-ref-guide/src/collection-specific-tools.adoc b/solr/solr-ref-guide/src/collection-specific-tools.adocdeleted file mode 100644index 3e09c927e4a..00000000000--- a/solr/solr-ref-guide/src/collection-specific-tools.adoc+++ /dev/null@@ -1,43 +0,0 @@-= Collection-Specific Tools-:page-children: analysis-screen, documents-screen, files-screen, query-screen, stream-screen, schema-browser-screen-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--In the left-hand navigation bar, you will see a pull-down menu titled "Collection Selector" that can be used to access collection specific administration screens.--.Only Visible When Using SolrCloud-[NOTE]-====-The "Collection Selector" pull-down menu is only available on Solr instances running in <<solrcloud.adoc#,SolrCloud mode>>.--Single node or leader/follower replication instances of Solr will not display this menu, instead the Collection specific UI pages described in this section will be available in the <<core-specific-tools.adoc#,Core Selector pull-down menu>>.-====--Clicking on the Collection Selector pull-down menu will show a list of the collections in your Solr cluster, with a search box that can be used to find a specific collection by name. When you select a collection from the pull-down, the main display of the page will display some basic metadata about the collection, and a secondary menu will appear in the left nav with links to additional collection specific administration screens.--image::images/collection-specific-tools/collection_dashboard.png[image,width=482,height=250]--The collection-specific UI screens are listed below, with a link to the section of this guide to find out more:--// TODO: SOLR-10655 BEGIN: refactor this into a 'collection-screens-list.include.adoc' file for reuse-* <<analysis-screen.adoc#,Analysis>> - lets you analyze the data found in specific fields.-* <<documents-screen.adoc#,Documents>> - provides a simple form allowing you to execute various Solr indexing commands directly from the browser.-* <<files-screen.adoc#,Files>> - shows the current core configuration files such as `solrconfig.xml`.-* <<query-screen.adoc#,Query>> - lets you submit a structured query about various elements of a core.-* <<stream-screen.adoc#,Stream>> - allows you to submit streaming expressions and see results and parsing explanations.-* <<schema-browser-screen.adoc#,Schema Browser>> - displays schema data in a browser window.-// TODO: SOLR-10655 ENDdiff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adocindex ab5e9492b39..03c778c50fe 100644--- a/solr/solr-ref-guide/src/collections-api.adoc+++ b/solr/solr-ref-guide/src/collections-api.adoc@@ -1,5 +1,4 @@= Collections API-:page-children: cluster-node-management, collection-management, collection-aliasing, shard-management, replica-management:page-show-toc: false// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file@@ -26,7 +25,7 @@ Because this API has a large number of commands and options, we've grouped the c*<<collection-management.adoc#,Collection Management>>*: Create, list, reload and delete collections; set collection properties; migrate documents to another collection; rebalance leaders; backup and restore collections.-*<<collection-aliasing.adoc#,Collection Aliasing>>*: Create, list or delete collection aliases; set alias properties.+*<<alias-management.adoc#,Alias Management>>*: Create, list or delete collection aliases; set alias properties.*<<shard-management.adoc#,Shard Management>>*: Create and delete a shard; split a shard into two or more additional shards; force a shard leader.diff --git a/solr/solr-ref-guide/src/collections-core-admin.adoc b/solr/solr-ref-guide/src/collections-core-admin.adocindex ffc228d31d9..292da8a4fab 100644--- a/solr/solr-ref-guide/src/collections-core-admin.adoc+++ b/solr/solr-ref-guide/src/collections-core-admin.adoc@@ -20,12 +20,13 @@ The Collections screen provides some basic functionality for managing your Colle[NOTE]====-If you are running a single node Solr instance, you will not see a Collections option in the left nav menu of the Admin UI.+If you are running a user-managed cluster or a single-node installation, you will not see a Collections option in the left nav menu of the Admin UI.You will instead see a "Core Admin" screen that supports some comparable Core level information & manipulation via the <<coreadmin-api.adoc#,CoreAdmin API>> instead.====-The main display of this page provides a list of collections that exist in your cluster. Clicking on a collection name provides some basic metadata about how the collection is defined, and its current shards & replicas, with options for adding and deleting individual replicas.+The main display of this page provides a list of collections that exist in your cluster.+Clicking on a collection name provides some basic metadata about how the collection is defined, and its current shards & replicas, with options for adding and deleting individual replicas.The buttons at the top of the screen let you make various collection level changes to your cluster, from add new collections or aliases to reloading or deleting a single collection.diff --git a/solr/solr-ref-guide/src/combining-distribution-and-replication.adoc b/solr/solr-ref-guide/src/combining-distribution-and-replication.adocdeleted file mode 100644index a45e59fbf37..00000000000--- a/solr/solr-ref-guide/src/combining-distribution-and-replication.adoc+++ /dev/null@@ -1,33 +0,0 @@-= Combining Distribution and Replication-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--When your index is too large for a single machine and you have a query volume that single shards cannot keep up with, it's time to replicate each shard in your distributed search setup.--The idea is to combine distributed search with replication. As shown in the figure below, a combined distributed-replication configuration features a leader server for each shard and then 1-_n_ followers that are replicated from the leader. As in a standard replicated configuration, the leader server handles updates and optimizations without adversely affecting query handling performance.--Query requests should be load balanced across each of the shard followers. This gives you both increased query handling capacity and fail-over backup if a server goes down.--.A Solr configuration combining both replication and leader-follower distribution.-image::images/combining-distribution-and-replication/distributed-replication.png[image,width=312,height=344]---None of the leader shards in this configuration know about each other. You index to each leader, the index is replicated to each follower, and then searches are distributed across the followers, using one follower from each leader/follower shard.--For high availability you can use a load balancer to set up a virtual IP for each shard's set of followers. If you are new to load balancing, HAProxy (http://haproxy.1wt.eu/) is a good open source software load-balancer. If a follower server goes down, a good load-balancer will detect the failure using some technique (generally a heartbeat system), and forward all requests to the remaining live followers that served with the failed follower. A single virtual IP should then be set up so that requests can hit a single IP, and get load balanced to each of the virtual IPs for the search followers.--With this configuration you will have a fully load balanced, search-side fault-tolerant system (Solr does not yet support fault-tolerant indexing). Incoming searches will be handed off to one of the functioning followers, then the follower will distribute the search request across a follower for each of the shards in your configuration. The follower will issue a request to each of the virtual IPs for each shard, and the load balancer will choose one of the available followers. Finally, the results will be combined into a single results set and returned. If any of the followers go down, they will be taken out of rotation and the remaining followers will be used. If a shard leader goes down, searches can still be served from the followers until you have corrected the problem and put the leader back into production.diff --git a/solr/solr-ref-guide/src/commits-transaction-logs.adoc b/solr/solr-ref-guide/src/commits-transaction-logs.adocnew file mode 100644index 00000000000..cf5903cb5dd--- /dev/null+++ b/solr/solr-ref-guide/src/commits-transaction-logs.adoc@@ -0,0 +1,321 @@+= Commits and Transaction Logs+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++In Solr, documents are not available for searching until a "commit" updates the Lucene index files.+Your commit strategy will determine when document additions, deletes, or changes are available for searching.+A transaction log records document updates that have been received since the last "hard" commit point.++== <updateHandler> in solrconfig.xml++The settings in this section are configured in the `<updateHandler>` element in `solrconfig.xml` and may affect the performance of index updates.+These settings affect how updates are done internally.++The `<updateHandler>` element takes a class parameter, which must be `solr.DirectUpdateHandler2`.+The `_default` configset included with Solr has this section defined already, but the values for many parameters discussed below likely need to be customized for your application.++[source,xml]+----+<config>+ <updateHandler class="solr.DirectUpdateHandler2">+ ...+ </updateHandler>+</config>+----++Note that `<updateHandler>` configurations do not affect the higher level configuration of <<requesthandlers-searchcomponents.adoc#,request handlers>> that process client update requests.++== Commits++Data sent to Solr is not searchable until it has been _committed_ to the index.+The reason for this is that in some cases commits can be slow and they should be done in isolation from other possible commit requests to avoid overwriting data.++=== Hard Commits vs. Soft Commits++Solr supports two types of commits: hard commits and soft commits.++A *hard commit* calls `fsync` on the index files to ensure they have been flushed to stable storage.+The current transaction log is closed and a new one is opened.+See the section <<Transaction Log>> below for how data is recovered in the absence of a hard commit.+Optionally a hard commit can also make documents visible for search, but this may not be ideal in some use cases as it is more expensive than a soft commit.+By default commit actions result in a hard commit of all the Lucene index files to stable storage (disk).++A *soft commit* is faster since it only makes index changes visible and does not `fsync` index files, start a new segment, nor start a new transaction log.+Search collections that have NRT requirements will want to soft commit often enough to satisfy the visibility requirements of the application.+A softCommit may be "less expensive" than a hard commit (`openSearcher=true`), but it is not free.+It is recommended that this be set for as long as is reasonable given the application requirements.++A hard commit means that, if a server crashes, Solr will know exactly where your data was stored; a soft commit means that the data is stored, but the location information isn't yet stored.+The tradeoff is that a soft commit gives you faster visibility because it's not waiting for background merges to finish.++=== Explicit Commits++When a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds and deletes on an update are written to disk as soon as index updates are completed.++If an additional parameter `softCommit=true` is specified, then Solr performs a soft commit.+This is an implementation of Near Real Time storage, a feature that boosts document visibility, since you don't have to wait for background merges and storage (to ZooKeeper, if using SolrCloud) to finish before moving on to something else.++Details about using explicit commit requests during indexing are in the section <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>.++For more information about Near Real Time operations, see <<solrcloud-distributed-requests.adoc#near-real-time-nrt-use-cases,Near Real Time Use Cases>>.++=== Automatic Commits++To avoid sending explicit commit commands during indexing and to provide control over when commits happen, it's possible to configure `autoCommit` parameters in `solrconfig.xml`.++This is preferable to sending explicit commits from the indexing client as it offers much more control over your commit strategy.+Note that defaults are provided in `solrconfig.xml`, but they are very likely not tuned to your needs and may introduce performance problems if not tuned effectively.++These settings control how often pending updates will be automatically pushed to the index.++`maxDocs`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The number of updates that have occurred since the last commit.++`maxTime`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The number of milliseconds since the oldest uncommitted update.+When sending a large batch of documents, this parameter is preferred over `maxDocs`.++`maxSize`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The maximum size of the transaction log (tlog) on disk, after which a hard commit is triggered.+This is useful when the size of documents is unknown and the intention is to restrict the size of the transaction log to reasonable size.+++Valid values can be bytes (default with no suffix), kilobytes (if defined with a `k` suffix, as in `25k`), megabytes (`m`) or gigabytes (`g`).++`openSearcher`::+++[%autowidth,frame=none]+|===+|Optional |Default: `true`+|===+++Whether to open a new searcher when performing a commit.+If this is `false`, the commit will flush recent index changes to stable storage, but does not cause a new searcher to be opened to make those changes visible.++If any of the `maxDocs`, `maxTime`, or `maxSize` limits are reached, Solr automatically performs a commit operation.+The first of these thresholds to be reached will trigger the commit.++If the `autoCommit` tag is missing from `solrconfig.xml`, then only explicit commits will update the index.+The decision whether to use autoCommit or not depends on the needs of your application.++[source,xml]+----+<autoCommit>+ <maxDocs>10000</maxDocs>+ <maxTime>30000</maxTime>+ <maxSize>512m</maxSize>+ <openSearcher>false</openSearcher>+</autoCommit>+----++You can also specify 'soft' autoCommits with the `autoSoftCommit` tag.++[source,xml]+----+<autoSoftCommit>+ <maxTime>60000</maxTime>+</autoSoftCommit>+----++=== AutoCommit Best Practices++Determining the best `autoCommit` settings is a tradeoff between performance and accuracy.+Settings that cause frequent updates will improve the accuracy of searches because new content will be searchable more quickly, but performance may suffer because of the frequent updates.+Less frequent updates may improve performance but it will take longer for updates to show up in queries.++Here is an example NRT configuration for the two flavors of commit, a hard commit every 60 seconds and a soft commit every 30 seconds.+Note that these are _not_ the values in the examples shipped with Solr!++[source,xml]+----+<autoCommit>+ <maxTime>${solr.autoCommit.maxTime:60000}</maxTime>+ <openSearcher>false</openSearcher>+</autoCommit>++<autoSoftCommit>+ <maxTime>${solr.autoSoftCommit.maxTime:30000}</maxTime>+ </autoSoftCommit>+----++TIP: These parameters can be overridden at run time by defining Java "system variables", for example specifying ``-Dsolr.autoCommit.maxTime=15000` would override the hard commit interval with a value of 15 seconds.++The choices for `autoCommit` (with `openSearcher=false`) and `autoSoftCommit` have different consequences.+In the event of un-graceful shutdown, it can take up to the time specified in `autoCommit` for Solr to replay the uncommitted documents from the transaction log.++The time chosen for `autoSoftCommit` determines the maximum time after a document is sent to Solr before it becomes searchable and does not affect the transaction log.++Choose as long an interval as your application can tolerate for this value, often 15-60 seconds is reasonable, or even longer depending on the requirements.+In situations where the the time is set to a very short interval (say 1 second), consider disabling your caches (queryResultCache and filterCache especially) as they will have little utility.++TIP: For extremely high bulk indexing, especially for the initial load if there is no searching, consider turning off `autoSoftCommit` by specifying a value of `-1` for the maxTime parameter.++=== Commit Within a Time Period++An alternative to `autoCommit` is to use `commitWithin`, which can be defined when making the update request to Solr (i.e., when pushing documents), or in an update request handler.++The `commitWithin` settings allow forcing document commits to happen in a defined time period.+This is used most frequently with <<solrcloud-distributed-requests.adoc#near-real-time-nrt-use-cases,Near Real Time use cases>>, and for that reason the default is to perform a soft commit.+This does not, however, replicate new documents to follower servers in a user-managed cluster.+If that's a requirement for your implementation, you can force a hard commit by adding a parameter, as in this example:++[source,xml]+----+<commitWithin>+ <softCommit>false</softCommit>+</commitWithin>+----++With this configuration, when you call `commitWithin` as part of your update message, it will automatically perform a hard commit every time.+++== Transaction Log++Transaction logs (tlogs) are a "rolling window" of updates since the last hard commit.+The current transaction log is closed and a new one opened each time any variety of hard commit occurs.+Soft commits have no effect on the transaction log.++When tlogs are enabled, documents being added to the index are written to the tlog before the indexing call returns to the client.+In the event of an un-graceful shutdown (power loss, JVM crash, `kill -9`, etc.) any documents written to the tlog but not yet committed with a hard commit when Solr was stopped are replayed on startup.+Therefore the data is not lost.++When Solr is shut down gracefully (using the `bin/solr stop` command) Solr will close the tlog file and index segments so no replay will be necessary on startup.++One point of confusion is how much data is contained in a transaction log.+A tlog does not contain all documents, only the ones since the last hard commit.+Older transaction log files are deleted when no longer needed.++WARNING: Implicit in the above is that transaction logs will grow forever if hard commits are disabled. Therefore it is important that hard commits be enabled when indexing.++=== Transaction Log Configuration++Transaction logs are required for all SolrCloud clusters, as well as the <<realtime-get.adoc#,RealTime Get>> feature.+It is configured in the `updateHandler` section of `solrconfig.xml`.++Transaction logs are configured in `solrconfig.xml`, in a section like the following:++[source,xml]+----+<updateLog>+ <str name="dir">${solr.ulog.dir:}</str>+</updateLog>+----++The only required parameter is:++`dir`::+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The location of the transaction log.+In Solr's default `solrconfig.xml` files, this is defined as `${solr.ulog.dir:}`.+++As shown in the default value, the location of the transaction log can be anywhere as long as it is defined in `solrconfig.xml` and write- and read-able by Solr.++There are three additional expert-level configuration settings which affect indexing performance and how far a replica can fall behind on updates before it must enter into full recovery.+These settings would primarily impact SolrCloud cluster configurations:++`numRecordsToKeep`::+++[%autowidth,frame=none]+|===+|Optional |Default: `100`+|===+++The number of update records to keep per log.++`maxNumLogsToKeep`::+++[%autowidth,frame=none]+|===+|Optional |Default: `10`+|===+++The maximum number of logs keep. The default is `10`.++`numVersionBuckets`::+++[%autowidth,frame=none]+|===+|Optional |Default: `65336`+|===+++The number of buckets used to keep track of maximum version values when checking for re-ordered updates.+Increase this value to reduce the cost of synchronizing access to version buckets during high-volume indexing.+This requires `(8 bytes (long) * numVersionBuckets)` of heap space per Solr core.++An example, to be included under `<updateHandler>` in `solrconfig.xml`, employing the above advanced settings:++[source,xml]+----+<updateLog>+ <str name="dir">${solr.ulog.dir:}</str>+ <int name="numRecordsToKeep">500</int>+ <int name="maxNumLogsToKeep">20</int>+ <int name="numVersionBuckets">65536</int>+</updateLog>+----++== Event Listeners++The UpdateHandler section is also where update-related event listeners can be configured.+These can be triggered to occur after any commit (`event="postCommit"`) or only after optimize commands (`event="postOptimize"`).++Users can write custom update event listener classes in Solr plugins.+As of Solr 7.1, `RunExecutableListener` was removed for security reasons.++== Other <updateHandler> Options++In some cases complex updates (such as spatial/shape) may take very long time to complete.+In the default configuration other updates that fall into the same internal version bucket will wait indefinitely.+Eventually these outstanding requests may pile up and lead to thread exhaustion and possibly to OutOfMemory errors.++The parameter `versionBucketLockTimeoutMs` helps to prevent that by specifying a timeout for long-running update requests.+If this limit is reached the update will fail but it won't block forever all other updates.++There is a memory cost associated with this setting.+Values greater than the default of `0` (unlimited timeout) cause Solr to use a different internal implementation of the version bucket, which increases memory consumption from ~1.5MB to ~6.8MB per Solr core.++An example of specifying this option under `<config>` section of `solrconfig.xml`:++[source,xml]+----+<updateHandler class="solr.DirectUpdateHandler2">+ ...+ <int name="versionBucketLockTimeoutMs">10000</int>+</updateHandler>+----diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adocindex 08db623c366..62627a4db73 100644--- a/solr/solr-ref-guide/src/common-query-parameters.adoc+++ b/solr/solr-ref-guide/src/common-query-parameters.adoc@@ -18,19 +18,23 @@Several query parsers share supported query parameters.-The following sections describe Solr's common query parameters, which are supported by the <<requesthandlers-and-searchcomponents-in-solrconfig#search-handlers,Search RequestHandlers>>.+The following sections describe Solr's common query parameters, which are supported by the <<requesthandlers-searchcomponents#search-handlers,Search RequestHandlers>>.== defType Parameter-The defType parameter selects the query parser that Solr should use to process the main query parameter (`q`) in the request. For example:+The defType parameter selects the query parser that Solr should use to process the main query parameter (`q`) in the request.+For example:`defType=dismax`-If no `defType` parameter is specified, then by default, the <<the-standard-query-parser.adoc#,The Standard Query Parser>> is used. (e.g., `defType=lucene`)+If no `defType` parameter is specified, then by default, the <<standard-query-parser.adoc#,Standard Query Parser>> is used.+(e.g., `defType=lucene`)== sort Parameter-The `sort` parameter arranges search results in either ascending (`asc`) or descending (`desc`) order. The parameter can be used with either numerical or alphabetical content. The directions can be entered in either all lowercase or all uppercase letters (i.e., both `asc` and `ASC` are accepted).+The `sort` parameter arranges search results in either ascending (`asc`) or descending (`desc`) order.+The parameter can be used with either numerical or alphabetical content.+The directions can be entered in either all lowercase or all uppercase letters (i.e., both `asc` and `ASC` are accepted).Solr can sort query responses according to:@@ -38,16 +42,17 @@ Solr can sort query responses according to:* <<function-queries.adoc#sort-by-function,Function results>>* The value of any primitive field (numerics, string, boolean, dates, etc.) which has `docValues="true"` (or `multiValued="false"` and `indexed="true"`, in which case the indexed terms will used to build DocValue like structures on the fly at runtime)* A SortableTextField which implicitly uses `docValues="true"` by default to allow sorting on the original input string regardless of the analyzers used for Searching.-* A single-valued TextField that uses an analyzer (such as the KeywordTokenizer) that produces only a single term per document. TextField does not support `docValues="true"`, but a DocValue-like structure will be built on the fly at runtime.-** *NOTE:* If you want to be able to sort on a field whose contents you want to tokenize to facilitate searching, <<copying-fields.adoc#,use a `copyField` directive>> in the the Schema to clone the field. Then search on the field and sort on its clone.+* A single-valued TextField that uses an analyzer (such as the KeywordTokenizer) that produces only a single term per document.+TextField does not support `docValues="true"`, but a DocValue-like structure will be built on the fly at runtime.+** *NOTE:* If you want to be able to sort on a field whose contents you want to tokenize to facilitate searching, <<copy-fields.adoc#,use a `copyField` directive>> in the the Schema to clone the field.+Then search on the field and sort on its clone.-In the case of primitive fields, or SortableTextFields, that are `multiValued="true"` the representative value used for each doc when sorting depends on the sort direction: The minimum value in each document is used for ascending (`asc`) sorting, while the maximal value in each document is used for descending (`desc`) sorting. This default behavior is equivalent to explicitly sorting using the 2 argument `<<function-queries.adoc#field-function,field()>>` function: `sort=field(name,min) asc` and `sort=field(name,max) desc`+In the case of primitive fields, or SortableTextFields, that are `multiValued="true"` the representative value used for each doc when sorting depends on the sort direction: The minimum value in each document is used for ascending (`asc`) sorting, while the maximal value in each document is used for descending (`desc`) sorting.+This default behavior is equivalent to explicitly sorting using the 2 argument `<<function-queries.adoc#field-function,field()>>` function: `sort=field(name,min) asc` and `sort=field(name,max) desc`The table below explains how Solr responds to various settings of the `sort` parameter.-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="30,70",options="header"]+[%autowidth.stretch,options="header"]|===|Example |Result| |If the sort parameter is omitted, sorting is performed as though the parameter were set to `score desc`.@@ -63,7 +68,9 @@ Regarding the sort parameter's arguments:* A sort ordering must include a field name (or `score` as a pseudo field), followed by whitespace (escaped as + or `%20` in URL strings), followed by a sort direction (`asc` or `desc`).* Multiple sort orderings can be separated by a comma, using this syntax: `sort=<field name>+<direction>,<field name>+<direction>],...`-** When more than one sort criteria is provided, the second entry will only be used if the first entry results in a tie. If there is a third entry, it will only be used if the first AND second entries are tied. And so on.+** When more than one sort criteria is provided, the second entry will only be used if the first entry results in a tie.+If there is a third entry, it will only be used if the first AND second entries are tied.+And so on.** If documents tie in all of the explicit sort criteria, Solr uses each document's Lucene document ID as the final tie-breaker.This internal property is subject to change during segment merges and document updates, which can lead to unexpected result ordering changes.Users looking to avoid this behavior can add an additional sort criteria on a unique or rarely-shared field such as `id` to prevent ties from occurring (e.g., `price desc,id asc`).@@ -76,11 +83,13 @@ The default value is `0`. In other words, by default, Solr returns results withoSetting the `start` parameter to some other number, such as `3`, causes Solr to skip over the preceding records and start at the document identified by the offset.-You can use the `start` parameter this way for paging. For example, if the `rows` parameter is set to 10, you could display three successive pages of results by setting start to 0, then re-issuing the same query and setting start to 10, then issuing the query again and setting start to 20.+You can use the `start` parameter this way for paging.+For example, if the `rows` parameter is set to 10, you could display three successive pages of results by setting start to 0, then re-issuing the same query and setting start to 10, then issuing the query again and setting start to 20.== rows Parameter-You can use the `rows` parameter to paginate results from a query. The parameter specifies the maximum number of documents from the complete result set that Solr should return to the client at one time.+You can use the `rows` parameter to paginate results from a query.+The parameter specifies the maximum number of documents from the complete result set that Solr should return to the client at one time.The default value is `10`. That is, by default, Solr returns 10 documents at a time in response to a query.@@ -91,42 +100,54 @@ This parameter defines if this query is cancellable during execution using the== queryUUID Parameter-For cancellable queries, this allows specifying a custom UUID to identify the query with. If `canCancel` is specified and `queryUUID` is not set, an auto generated UUID will be assigned to the query.+For cancellable queries, this allows specifying a custom UUID to identify the query with.+If `canCancel` is specified and `queryUUID` is not set, an auto generated UUID will be assigned to the query.-If `queryUUID` is specified, this UUID will be used for identifying the query. Note that if using `queryUUID`, the responsibility of ensuring uniqueness of the UUID lies with the caller. If a query UUID-is reused while the original query UUID is still active, it will cause an exception to be throws for the second query.+If `queryUUID` is specified, this UUID will be used for identifying the query.+Note that if using `queryUUID`, the responsibility of ensuring uniqueness of the UUID lies with the caller.+If a query UUID is reused while the original query UUID is still active, it will cause an exception to be throws for the second query.-It is recommended that the user either uses all custom UUIDs or depends completely on the system to generate UUID. Mixing the two can lead to conflict of UUIDs.+It is recommended that the user either uses all custom UUIDs or depends completely on the system to generate UUID.+Mixing the two can lead to conflict of UUIDs.== fq (Filter Query) Parameter-The `fq` parameter defines a query that can be used to restrict the superset of documents that can be returned, without influencing score. It can be very useful for speeding up complex queries, since the queries specified with `fq` are cached independently of the main query. When a later query uses the same filter, there's a cache hit, and filter results are returned quickly from the cache.+The `fq` parameter defines a query that can be used to restrict the superset of documents that can be returned, without influencing score.+It can be very useful for speeding up complex queries, since the queries specified with `fq` are cached independently of the main query.+When a later query uses the same filter, there's a cache hit, and filter results are returned quickly from the cache.When using the `fq` parameter, keep in mind the following:-* The `fq` parameter can be specified multiple times in a query. Documents will only be included in the result if they are in the intersection of the document sets resulting from each instance of the parameter. In the example below, only documents which have a popularity greater then 10 and have a section of 0 will match.+* The `fq` parameter can be specified multiple times in a query.+Documents will only be included in the result if they are in the intersection of the document sets resulting from each instance of the parameter.+In the example below, only documents which have a popularity greater then 10 and have a section of 0 will match.+[source,text]----fq=popularity:[10 TO *]&fq=section:0-----* Filter queries can involve complicated Boolean queries. The above example could also be written as a single `fq` with two mandatory clauses like so:+* Filter queries can involve complicated Boolean queries.+The above example could also be written as a single `fq` with two mandatory clauses like so:+[source,text]----fq=+popularity:[10 TO *] +section:0-----* The document sets from each filter query are cached independently. Thus, concerning the previous examples: use a single `fq` containing two mandatory clauses if those clauses appear together often, and use two separate `fq` parameters if they are relatively independent. (To learn about tuning cache sizes and making sure a filter cache actually exists, see <<the-well-configured-solr-instance.adoc#,The Well-Configured Solr Instance>>.)-* It is also possible to use <<the-standard-query-parser.adoc#differences-between-lucenes-classic-query-parser-and-solrs-standard-query-parser,filter(condition) syntax>> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries.+* The document sets from each filter query are cached independently.+Thus, concerning the previous examples: use a single `fq` containing two mandatory clauses if those clauses appear together often, and use two separate `fq` parameters if they are relatively independent.+(To learn about tuning cache sizes and making sure a filter cache actually exists, see <<caches-warming.adoc#caches,Caches>>.)+* It is also possible to use <<standard-query-parser.adoc#differences-between-lucenes-classic-query-parser-and-solrs-standard-query-parser,filter(condition) syntax>> inside the `fq` to cache clauses individually and - among other things - to achieve union of cached filter queries.-* As with all parameters: special characters in an URL need to be properly escaped and encoded as hex values. Online tools are available to help you with URL-encoding. For example: http://meyerweb.com/eric/tools/dencoder/.+* As with all parameters: special characters in an URL need to be properly escaped and encoded as hex values.+Online tools are available to help you with URL-encoding.+For example: http://meyerweb.com/eric/tools/dencoder/.=== cache Local Parameter-Solr caches the results of filter queries by default in the <<query-settings-in-solrconfig.adoc#filtercache,filter cache>>.-To disable it, use the boolean `cache` <<local-parameters-in-queries.adoc#,local parameter>>, such as `fq={!geofilt cache=false}...`.+Solr caches the results of filter queries by default in the <<caches-warming.adoc#filter-cache,filter cache>>.+To disable it, use the boolean `cache` <<local-params.adoc#,local param>>, such as `fq={!geofilt cache=false}...`.Do this when you think a query is unlikely to be repeated.Non-cached filter queries also support the `cost` local parameter to provide a _hint_ as to the order in which they are evaluated.@@ -157,15 +178,17 @@ fq={!frange cache=false cost=200 l=0}pow(mul(sum(1, query('tag:smartphone')), di== fl (Field List) Parameter-The `fl` parameter limits the information included in a query response to a specified list of fields. The fields must be either `stored="true"` or `docValues="true"``.`+The `fl` parameter limits the information included in a query response to a specified list of fields.+The fields must be either `stored="true"` or `docValues="true"``.`-The field list can be specified as a space-separated or comma-separated list of field names. The string "score" can be used to indicate that the score of each document for the particular query should be returned as a field. The wildcard character `*` selects all the fields in the document which are either `stored="true"` or `docValues="true"` and `useDocValuesAsStored="true"` (which is the default when docValues are enabled). You can also add pseudo-fields, functions and transformers to the field list request.+The field list can be specified as a space-separated or comma-separated list of field names.+The string "score" can be used to indicate that the score of each document for the particular query should be returned as a field.+The wildcard character `*` selects all the fields in the document which are either `stored="true"` or `docValues="true"` and `useDocValuesAsStored="true"` (which is the default when docValues are enabled).+You can also add pseudo-fields, functions and transformers to the field list request.This table shows some basic examples of how to use `fl`:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="30,70",options="header"]+[%autowidth.stretch,options="header"]|===|Field List |Result|id name price |Return only the id, name, and price fields.@@ -188,7 +211,7 @@ fl=id,title,product(price,popularity)=== Document Transformers with fl-<<transforming-result-documents.adoc#,Document Transformers>> can be used to modify the information returned about each documents in the results of a query:+<<document-transformers.adoc#,Document Transformers>> can be used to modify the information returned about each documents in the results of a query:[source,text]----@@ -231,15 +254,18 @@ The `debug` parameter can be specified multiple times and supports the following* `debug=timing`: return debug information about how long the query took to process.* `debug=results`: return debug information about the score results (also known as "explain").** By default, score explanations are returned as large string values, using newlines and tab indenting for structure & readability, but an additional `debug.explain.structured=true` parameter may be specified to return this information as nested data structures native to the response format requested by `wt`.-* `debug=all`: return all available debug information about the request request. (alternatively usage: `debug=true`)+* `debug=all`: return all available debug information about the request request.+An alternative usage is `debug=true`.-For backwards compatibility with older versions of Solr, `debugQuery=true` may instead be specified as an alternative way to indicate `debug=all`+For backwards compatibility with older versions of Solr, `debugQuery=true` may instead be specified as an alternative way to indicate `debug=all`.The default behavior is not to include debugging information.== explainOther Parameter-The `explainOther` parameter specifies a Lucene query in order to identify a set of documents. If this parameter is included and is set to a non-blank value, the query will return debugging information, along with the "explain info" of each document that matches the Lucene query, relative to the main query (which is specified by the `q` parameter). For example:+The `explainOther` parameter specifies a Lucene query in order to identify a set of documents.+If this parameter is included and is set to a non-blank value, the query will return debugging information, along with the "explain info" of each document that matches the Lucene query, relative to the main query (which is specified by the `q` parameter).+For example:[source,text]----@@ -252,7 +278,11 @@ The default value of this parameter is blank, which causes no extra "explain inf== timeAllowed Parameter-This parameter specifies the amount of time, in milliseconds, allowed for a search to complete. If this time expires before the search is complete, any partial results will be returned, but values such as `numFound`, <<faceting.adoc#,facet>> counts, and result <<the-stats-component.adoc#,stats>> may not be accurate for the entire result set. In case of expiration, if `omitHeader` isn't set to `true` the response header contains a special flag called `partialResults`. When using `timeAllowed` in combination with <<pagination-of-results.adoc#using-cursors,`cursorMark`>>, and the `partialResults` flag is present, some matching documents may have been skipped in the result set. Additionally, if the `partialResults` flag is present, `cursorMark` can match `nextCursorMark` even if there may be more results+This parameter specifies the amount of time, in milliseconds, allowed for a search to complete.+If this time expires before the search is complete, any partial results will be returned, but values such as `numFound`, <<faceting.adoc#,facet>> counts, and result <<stats-component.adoc#,stats>> may not be accurate for the entire result set.+In case of expiration, if `omitHeader` isn't set to `true` the response header contains a special flag called `partialResults`.+When using `timeAllowed` in combination with <<pagination-of-results.adoc#using-cursors,`cursorMark`>>, and the `partialResults` flag is present, some matching documents may have been skipped in the result set.+Additionally, if the `partialResults` flag is present, `cursorMark` can match `nextCursorMark` even if there may be more results[source,json]----@@ -280,17 +310,19 @@ This value is only checked at the time of:. Document collection. Doc Values reading-As this check is periodically performed, the actual time for which a request can be processed before it is aborted would be marginally greater than or equal to the value of `timeAllowed`. If the request consumes more time in other stages, custom components, etc., this parameter is not expected to abort the request. Regular search, JSON Facet and the Analytics component abandon requests in accordance with this parameter.+As this check is periodically performed, the actual time for which a request can be processed before it is aborted would be marginally greater than or equal to the value of `timeAllowed`.+If the request consumes more time in other stages, custom components, etc., this parameter is not expected to abort the request.+Regular search, JSON Facet and the Analytics component abandon requests in accordance with this parameter.== segmentTerminateEarly ParameterThis parameter may be set to either `true` or `false`.-If set to `true`, and if <<indexconfig-in-solrconfig.adoc#mergepolicyfactory,the mergePolicyFactory>> for this collection is a {solr-javadocs}/core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] which uses a `sort` option compatible with <<sort Parameter,the sort parameter>> specified for this query, then Solr will be able to skip documents on a per-segment basis that are definitively not candidates for the current page of results.+If set to `true`, and if <<index-segments-merging.adoc#mergepolicyfactory,the mergePolicyFactory>> for this collection is a {solr-javadocs}/core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] which uses a `sort` option compatible with <<sort Parameter,the sort parameter>> specified for this query, then Solr will be able to skip documents on a per-segment basis that are definitively not candidates for the current page of results.If early termination is used, a `segmentTerminatedEarly` header will be included in the `responseHeader`.-Similar to using <<timeAllowed Parameter,the `timeAllowed` Parameter>>, when early segment termination happens values such as `numFound`, <<faceting.adoc#,Facet>> counts, and result <<the-stats-component.adoc#,Stats>> may not be accurate for the entire result set.+Similar to using <<timeAllowed Parameter,the `timeAllowed` Parameter>>, when early segment termination happens values such as `numFound`, <<faceting.adoc#,Facet>> counts, and result <<stats-component.adoc#,Stats>> may not be accurate for the entire result set.The default value of this parameter is `false`.@@ -298,17 +330,23 @@ The default value of this parameter is `false`.This parameter may be set to either `true` or `false`.-If set to `true`, this parameter excludes the header from the returned results. The header contains information about the request, such as the time it took to complete. The default value for this parameter is `false`. When using parameters such as <<common-query-parameters.adoc#timeallowed-parameter,`timeAllowed`>>, and <<solrcloud-query-routing-and-read-tolerance.adoc#shards-tolerant-parameter,`shards.tolerant`>>, which can lead to partial results, it is advisable to keep the header, so that the `partialResults` flag can be checked, and values such as `numFound`, `nextCursorMark`, <<faceting.adoc#,Facet>> counts, and result <<the-stats-component.adoc#,Stats>> can be interpreted in the context of partial results.+If set to `true`, this parameter excludes the header from the returned results.+The header contains information about the request, such as the time it took to complete.+The default value for this parameter is `false`.+When using parameters such as <<common-query-parameters.adoc#timeallowed-parameter,`timeAllowed`>>, and <<solrcloud-distributed-requests.adoc#shards-tolerant-parameter,`shards.tolerant`>>, which can lead to partial results, it is advisable to keep the header, so that the `partialResults` flag can be checked, and values such as `numFound`, `nextCursorMark`, <<faceting.adoc#,Facet>> counts, and result <<stats-component.adoc#,Stats>> can be interpreted in the context of partial results.== wt Parameter-The `wt` parameter selects the Response Writer that Solr should use to format the query's response. For detailed descriptions of Response Writers, see <<response-writers.adoc#,Response Writers>>.+The `wt` parameter selects the Response Writer that Solr should use to format the query's response.+For detailed descriptions of Response Writers, see <<response-writers.adoc#,Response Writers>>.If you do not define the `wt` parameter in your queries, JSON will be returned as the format of the response.== logParamsList Parameter-By default, Solr logs all parameters of requests. Set this parameter to restrict which parameters of a request are logged. This may help control logging to only those parameters considered important to your organization.+By default, Solr logs all parameters of requests.+Set this parameter to restrict which parameters of a request are logged.+This may help control logging to only those parameters considered important to your organization.For example, you could define this like:@@ -327,8 +365,11 @@ The `echoParams` parameter controls what information about request parameters isThe `echoParams` parameter accepts the following values:* `explicit`: Only parameters included in the actual request will be added to the `params` section of the response header.-* `all`: Include all request parameters that contributed to the query. This will include everything defined in the request handler definition found in `solrconfig.xml` as well as parameters included with the request, plus the `_` parameter. If a parameter is included in the request handler definition AND the request, it will appear multiple times in the response header.-* `none`: Entirely removes the `params` section of the response header. No information about the request parameters will be available in the response.+* `all`: Include all request parameters that contributed to the query.+This will include everything defined in the request handler definition found in `solrconfig.xml` as well as parameters included with the request, plus the `_` parameter.+If a parameter is included in the request handler definition AND the request, it will appear multiple times in the response header.+* `none`: Entirely removes the `params` section of the response header.+No information about the request parameters will be available in the response.The default value is `none`, though many `solrconfig.xml` handlers set default to be `explicit`.Here is an example of a JSON response where the echoParams parameter was set in that SearchHandler's default,@@ -382,14 +423,21 @@ This is what happens if a similar request is sent that adds `echoParams=all` to----== minExactCount Parameter-When this parameter is used, Solr will count the number of hits accurately at least until this value. After that, Solr can skip over documents that don't have a score high enough to enter in the top N. This can greatly improve performance of search queries. On the other hand, when this parameter is used, the `numFound` may not be exact, and may instead be an approximation.-The `numFoundExact` boolean attribute is included in all responses, indicating if the `numFound` value is exact or an approximation. If it's an approximation, the real number of hits for the query is guaranteed to be greater or equal `numFound`.+When this parameter is used, Solr will count the number of hits accurately at least until this value.+After that, Solr can skip over documents that don't have a score high enough to enter in the top N.+This can greatly improve performance of search queries.+On the other hand, when this parameter is used, the `numFound` may not be exact, and may instead be an approximation.+The `numFoundExact` boolean attribute is included in all responses, indicating if the `numFound` value is exact or an approximation.+If it's an approximation, the real number of hits for the query is guaranteed to be greater or equal `numFound`.More about approximate document counting and `minExactCount`:-* The documents returned in the response are guaranteed to be the docs with the top scores. This parameter will not make Solr skip documents that are to be returned in the response, it will only allow Solr to skip counting docs that, while they match the query, their score is low enough to not be in the top N.-* Providing `minExactCount` doesn't guarantee that Solr will use approximate hit counting (and thus, provide the speedup). Some types of queries, or other parameters (like if facets are requested) will require accurate counting.-* Approximate counting can only be used when sorting by `score desc` first (which is the default sort in Solr). Other fields can be used after `score desc`, but if any other type of sorting is used before score, then the approximation won't be applied.+* The documents returned in the response are guaranteed to be the docs with the top scores.+This parameter will not make Solr skip documents that are to be returned in the response, it will only allow Solr to skip counting docs that, while they match the query, their score is low enough to not be in the top N.+* Providing `minExactCount` doesn't guarantee that Solr will use approximate hit counting (and thus, provide the speedup).+Some types of queries, or other parameters (like if facets are requested) will require accurate counting.+* Approximate counting can only be used when sorting by `score desc` first (which is the default sort in Solr).+Other fields can be used after `score desc`, but if any other type of sorting is used before score, then the approximation won't be applied.* When doing distributed queries across multiple shards, each shard will accurately count hits until `minExactCount` (which means the query could be hitting `numShards * minExactCount` docs and `numFound` in the response would still be accurate)For example:@@ -419,4 +467,5 @@ q=quick brown fox&minExactCount=200&rows=10"docs": [{"doc1"}]}-----In this case we know that `163` is the exact number of hits for the query. Both queries must have returned the same number of documents in the top 10.+In this case we know that `163` is the exact number of hits for the query.+Both queries must have returned the same number of documents in the top 10.diff --git a/solr/solr-ref-guide/src/config-api.adoc b/solr/solr-ref-guide/src/config-api.adocindex 4c70a332f10..258f3084afc 100644--- a/solr/solr-ref-guide/src/config-api.adoc+++ b/solr/solr-ref-guide/src/config-api.adoc@@ -18,17 +18,22 @@The Config API enables manipulating various aspects of your `solrconfig.xml` using REST-like API calls.-This feature is enabled by default and works similarly in both SolrCloud and standalone mode. Many commonly edited properties (such as cache sizes and commit settings) and request handler definitions can be changed with this API.+This feature is enabled by default and works similarly in both SolrCloud, user-managed clusters, and single-node installations.+Many commonly edited properties (such as cache sizes and commit settings) and request handler definitions can be changed with this API.-When using this API, `solrconfig.xml` is not changed. Instead, all edited configuration is stored in a file called `configoverlay.json`. The values in `configoverlay.json` override the values in `solrconfig.xml`.+When using this API, `solrconfig.xml` is not changed.+Instead, all edited configuration is stored in a file called `configoverlay.json`.+The values in `configoverlay.json` override the values in `solrconfig.xml`.== Config API EndpointsAll Config API endpoints are collection-specific, meaning this API can inspect or modify the configuration for a single collection at a time.-* `_collection_/config`: retrieve the full effective config, or modify the config. Use GET to retrieve and POST for executing commands.+* `_collection_/config`: retrieve the full effective config, or modify the config.+Use GET to retrieve and POST for executing commands.* `_collection_/config/overlay`: retrieve the details in the `configoverlay.json` only, removing any options defined in `solrconfig.xml` directly or implicitly through defaults.-* `_collection_/config/params`: create parameter sets that can override or take the place of parameters defined in `solrconfig.xml`. See <<request-parameters-api.adoc#,Request Parameters API>> for more information about this endpoint.+* `_collection_/config/params`: create parameter sets that can override or take the place of parameters defined in `solrconfig.xml`.+See <<request-parameters-api.adoc#,Request Parameters API>> for more information about this endpoint.== Retrieving the Config@@ -60,7 +65,9 @@ http://localhost:8983/api/collections/techproducts/configThe response will be the Solr configuration resulting from merging settings in `configoverlay.json` with those in `solrconfig.xml`.-It's possible to restrict the returned configuration to a top-level section, such as, `query`, `requestHandler` or `updateHandler`. To do this, append the name of the section to the `config` endpoint. For example, to retrieve configuration for all request handlers:+It's possible to restrict the returned configuration to a top-level section, such as, `query`, `requestHandler` or `updateHandler`.+To do this, append the name of the section to the `config` endpoint.+For example, to retrieve configuration for all request handlers:[.dynamic-tabs]--@@ -85,7 +92,9 @@ http://localhost:8983/api/collections/techproducts/config/requestHandler====---The output will be details of each request handler defined in `solrconfig.xml`, all <<implicit-requesthandlers.adoc#,defined implicitly>> by Solr, and all defined with this Config API stored in `configoverlay.json`. To see the configuration for implicit request handlers, add `expandParams=true` to the request. See the documentation for the implicit request handlers for examples using this command.+The output will be details of each request handler defined in `solrconfig.xml`, <<implicit-requesthandlers.adoc#,defined implicitly>> by Solr, and defined with this Config API stored in `configoverlay.json`.+To see the configuration for implicit request handlers, add `expandParams=true` to the request.+See the documentation for implicit request handlers linked above for examples using this command.The available top-level sections that can be added as path parameters are: `query`, `requestHandler`, `searchComponent`, `updateHandler`, `queryResponseWriter`, `initParams`, `znodeVersion`, `listener`, `directoryFactory`, `indexConfig`, and `codecFactory`.@@ -135,9 +144,11 @@ The ability to restrict to objects within a top-level section is limited to requ== Commands to Modify the Config-This API uses specific commands with POST requests to tell Solr what property or type of property to add to or modify in `configoverlay.json`. The commands are passed with the data to add or modify the property or component.+This API uses specific commands with POST requests to tell Solr what property or type of property to add to or modify in `configoverlay.json`.+The commands are passed with the data to add or modify the property or component.-The Config API commands for modifications are categorized into 3 types, each of which manipulate specific data structures in `solrconfig.xml`. These types are:+The Config API commands for modifications are categorized into 3 types, each of which manipulate specific data structures in `solrconfig.xml`.+These types are:* `set-property` and `unset-property` for <<Commands for Common Properties,Common Properties>>* Component-specific `add-`, `update-`, and `delete-` commands for <<Commands for Handlers and Components,Custom Handlers and Local Components>>@@ -145,16 +156,20 @@ The Config API commands for modifications are categorized into 3 types, each of=== Commands for Common Properties-The common properties are those that are frequently customized in a Solr instance. They are manipulated with two commands:+The common properties are those that are frequently customized in a Solr instance.+They are manipulated with two commands:-* `set-property`: Set a well known property. The names of the properties are predefined and fixed. If the property has already been set, this command will overwrite the previous setting.+* `set-property`: Set a known property.+The names of the properties are predefined and fixed.+If the property has already been set, this command will overwrite the previous setting.* `unset-property`: Remove a property set using the `set-property` command.-The properties that can be configured with `set-property` and `unset-property` are predefined and listed below. The names of these properties are derived from their XML paths as found in `solrconfig.xml`.+The properties that can be configured with `set-property` and `unset-property` are predefined and listed below.+The names of these properties are derived from their XML paths as found in `solrconfig.xml`.*Update Handler Settings*-See <<updatehandlers-in-solrconfig.adoc#,UpdateHandlers in SolrConfig>> for defaults and acceptable values for these settings.+See <<commits-transaction-logs.adoc#,Commits and Transaction Logs>> for defaults and acceptable values for these settings.* `updateHandler.autoCommit.maxDocs`* `updateHandler.autoCommit.maxTime`@@ -166,7 +181,7 @@ See <<updatehandlers-in-solrconfig.adoc#,UpdateHandlers in SolrConfig>> for defa*Query Settings*-See <<query-settings-in-solrconfig.adoc#,Query Settings in SolrConfig>> for defaults and acceptable values for these settings.+See <<caches-warming.adoc#,Caches and Query Warming>> for defaults and acceptable values for these settings._Caches and Cache Sizes_@@ -210,7 +225,7 @@ See <<circuit-breakers.adoc#,Circuit Breakers in Solr>> for more details*RequestDispatcher Settings*-See <<requestdispatcher-in-solrconfig.adoc#,RequestDispatcher in SolrConfig>> for defaults and acceptable values for these settings.+See <<requestdispatcher.adoc#,RequestDispatcher>> for defaults and acceptable values for these settings.* `requestDispatcher.handleSelect`* `requestDispatcher.requestParsers.enableRemoteStreaming`@@ -321,7 +336,8 @@ curl -X POST -H 'Content-type: application/json' -d '{"unset-property": "updateHRequest handlers, search components, and other types of localized Solr components (such as query parsers, update processors, etc.) can be added, updated and deleted with specific commands for the type of component being modified.-The syntax is similar in each case: `add-<component-name>`, `update-_<component-name>_`, and `delete-<component-name>`. The command name is not case sensitive, so `Add-RequestHandler`, `ADD-REQUESTHANDLER` and `add-requesthandler` are equivalent.+The syntax is similar in each case: `add-<_component-name_>`, `update-<_component-name_>`, and `delete-<_component-name_>`.+The command name is not case sensitive, so `Add-RequestHandler`, `ADD-REQUESTHANDLER` and `add-requesthandler` are equivalent.In each case, `add-` commands add a new configuration to `configoverlay.json`, which will override any other settings for the component in `solrconfig.xml`.@@ -433,7 +449,7 @@ Make a call to the new request handler to check if it is registered:[source,bash]-----curl http://localhost:8983/solr/techproducts/mypath?omitHeader=true+$ curl http://localhost:8983/solr/techproducts/mypath?omitHeader=true----And you should see the following as output:@@ -561,14 +577,18 @@ curl -X POST -H 'Content-type:application/json' -d '{=== Commands for User-Defined Properties-Solr lets users templatize the `solrconfig.xml` using the place holder format `${variable_name:default_val}`. You could set the values using system properties, for example, `-Dvariable_name= my_customvalue`. The same can be achieved during runtime using these commands:+Solr lets users templatize the `solrconfig.xml` using the place holder format `${variable_name:default_val}`.+You could set the values using system properties, for example, `-Dvariable_name= my_customvalue`.+The same can be achieved during runtime using these commands:-* `set-user-property`: Set a user-defined property. If the property has already been set, this command will overwrite the previous setting.+* `set-user-property`: Set a user-defined property.+If the property has already been set, this command will overwrite the previous setting.* `unset-user-property`: Remove a user-defined property.-The structure of the request is similar to the structure of requests using other commands, in the format of `"command":{"variable_name": "property_value"}`. You can add more than one variable at a time if necessary.+The structure of the request is similar to the structure of requests using other commands, in the format of `"command":{"variable_name": "property_value"}`.+You can add more than one variable at a time if necessary.-For more information about user-defined properties, see the section <<configuring-solrconfig-xml.adoc#user-defined-properties-in-core-properties,User defined properties in core.properties>>.+For more information about user-defined properties, see the section <<property-substitution.adoc#user-defined-properties-in-core-properties,User defined properties in core.properties>>.See also the section <<Creating and Updating User-Defined Properties>> below for examples of how to use this type of command.@@ -660,9 +680,10 @@ curl -X POST -H 'Content-type:application/json' -d '{"unset-user-property": "var====---=== What about updateRequestProcessorChain?+=== updateRequestProcessorChain Elements-The Config API does not let you create or edit `updateRequestProcessorChain` elements. However, it is possible to create `updateProcessor` entries and use them by name to create a chain.+The Config API does not let you create or edit `updateRequestProcessorChain` elements.+However, it is possible to create `updateProcessor` entries and use them by name to create a chain.For example:@@ -701,7 +722,8 @@ You can use this directly in your request by adding a parameter in the `updateRe== How to Map solrconfig.xml Properties to JSON-By using this API, you will be generating JSON representations of properties defined in `solrconfig.xml`. To understand how properties should be represented with the API, let's take a look at a few examples.+By using this API, you will be generating JSON representations of properties defined in `solrconfig.xml`.+To understand how properties should be represented with the API, let's take a look at a few examples.Here is what a request handler looks like in `solrconfig.xml`:@@ -850,14 +872,22 @@ Define the same properties with the Config API:=== Name Components for the Config API-The Config API always allows changing the configuration of any component by name. However, some configurations such as `listener` or `initParams` do not require a name in `solrconfig.xml`. In order to be able to `update` and `delete` of the same item in `configoverlay.json`, the name attribute becomes mandatory.+The Config API always allows changing the configuration of any component by name.+However, some configurations such as `listener` or `initParams` do not require a name in `solrconfig.xml`.+In order to be able to `update` and `delete` of the same item in `configoverlay.json`, the name attribute becomes mandatory.== How the Config API Works-Every core watches the ZooKeeper directory for the configset being used with that core. In standalone mode, however, there is no watch (because ZooKeeper is not running). If there are multiple cores in the same node using the same configset, only one ZooKeeper watch is used.+When using SolrCloud, every core watches the ZooKeeper directory for the configset being used with that core.+If there are multiple cores in the same node using the same configset, only one ZooKeeper watch is used.-For instance, if the configset 'myconf' is used by a core, the node would watch `/configs/myconf`. Every write operation performed through the API would 'touch' the directory and all watchers are notified. Every core would check if the schema file, `solrconfig.xml`, or `configoverlay.json` has been modified by comparing the `znode` versions. If any have been modified, the core is reloaded.+TIP:: In a user-managed cluster or single-node installation, there is no watch (because ZooKeeper is not running).++For instance, if the configset 'myconf' is used by a core, the node would watch `/configs/myconf`.+Every write operation performed through the API would 'touch' the directory and all watchers are notified.+Every core would check if the schema file, `solrconfig.xml`, or `configoverlay.json` has been modified by comparing the `znode` versions.+If any have been modified, the core is reloaded.If `params.json` is modified, the params object is just updated without a core reload (see <<request-parameters-api.adoc#,Request Parameters API>> for more information about `params.json`).@@ -898,4 +928,6 @@ Any component can register a listener using:`SolrCore#addConfListener(Runnable listener)`-to get notified for configuration changes. This is not very useful if the files modified result in core reloads (i.e., `configoverlay.xml` or the schema). Components can use this to reload the files they are interested in.+to get notified of configuration changes.+This is not very useful if the files modified result in core reloads (i.e., `configoverlay.xml` or the schema).+Components can use this to reload the files they are interested in.diff --git a/solr/solr-ref-guide/src/config-sets.adoc b/solr/solr-ref-guide/src/config-sets.adocindex b3c7f1503e6..1c566272a4b 100644--- a/solr/solr-ref-guide/src/config-sets.adoc+++ b/solr/solr-ref-guide/src/config-sets.adoc@@ -18,13 +18,14 @@Configsets are a set of configuration files used in a Solr installation: `solrconfig.xml`, the schema, and then <<resource-loading.adoc#,resources>> like language files, `synonyms.txt`, and others.-Such configuration, _configsets_, can be named and then referenced by collections or cores, possibly with the intent to share them to avoid duplication.+Such configuration, _configsets_, can be named and then referenced by collections or cores, allowing you to share them to avoid duplication.-Solr ships with two example configsets located in `server/solr/configsets`, which can be used as a base for your own. These example configsets are named `_default` and `sample_techproducts_configs`.+Solr ships with two example configsets located in `server/solr/configsets`, which can be used as a base for your own.+These example configsets are named `_default` and `sample_techproducts_configs`.-== Configsets in Standalone Mode+== Configsets in User-Managed Clusters or Single-Node Installations-If you are using Solr in standalone mode, configsets are managed on the filesystem.+If you are using Solr in a user-managed cluster or a single-node installation, configsets are managed on the filesystem.Each Solr core can have it's very own configset located beneath it in a `<instance_dir>/conf/` dir.Here, it is not named or shared and the word _configset_ isn't found.@@ -48,9 +49,11 @@ The structure should look something like this:/solrconfig.xml-----The default base directory is `$SOLR_HOME/configsets`. This path can be configured in `solr.xml` (see <<format-of-solr-xml.adoc#,Format of solr.xml>> for details).+The default base directory is `$SOLR_HOME/configsets`.+This path can be configured with the `configSetBaseDir` parameter in `solr.xml` (see <<configuring-solr-xml.adoc#,Format of solr.xml>> for details).-To create a new core using a configset, pass `configSet` as one of the core properties. For example, if you do this via the CoreAdmin API:+To create a new core using a configset, pass `configSet` as one of the core properties.+For example, if you do this via the CoreAdmin API:[.dynamic-tabs]--@@ -81,18 +84,19 @@ curl -v -X POST -H 'Content-type: application/json' -d '{====---== Configsets in SolrCloud Mode+== Configsets in SolrCloud Clusters-In SolrCloud, it's critical to understand that configsets are fundamentally stored in ZooKeeper _and not_ the file system.+In SolrCloud, it's critical to understand that configsets are stored in ZooKeeper _and not_ the file system.Solr's `_default` configset is uploaded to ZooKeeper on initialization.-This and some demonstration ones remain on the file system but Solr does not use them whatsoever in this mode.+This and a couple of example configsets remain on the file system but Solr does not use them unless they are used with a new collection.-When you create a collection in SolrCloud, you can specify a named configset -- possibly shared.-If you don't, then the `_default` will be copied and given a unique name for use by this collection.+When you create a collection in SolrCloud, you can specify a named configset.+If you don't, then the `_default` will be copied and given a unique name for use by the new collection.A configset can be uploaded to ZooKeeper either via the <<configsets-api.adoc#,Configsets API>> or more directly via <<solr-control-script-reference.adoc#upload-a-configuration-set,`bin/solr zk upconfig`>>.The Configsets API has some other operations as well, and likewise, so does the CLI.To upload a file to a configset already stored on ZooKeeper, you can use <<solr-control-script-reference.adoc#copy-between-local-files-and-zookeeper-znodes,`bin/solr zk cp`>>.-CAUTION: By default, ZooKeeper's file size limit is 1MB. If your files are larger than this, you'll need to either <<setting-up-an-external-zookeeper-ensemble.adoc#increasing-the-file-size-limit,increase the ZooKeeper file size limit>> or store them instead <<libs.adoc#lib-directives-in-solrconfig,on the filesystem>>.+CAUTION: By default, ZooKeeper's file size limit is 1MB.+If your files are larger than this, you'll need to either <<zookeeper-ensemble.adoc#increasing-the-file-size-limit,increase the ZooKeeper file size limit>> or store them <<libs.adoc#lib-directives-in-solrconfig,on the filesystem>>.diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adocindex 141d6d629b1..c0d9a9ccc8b 100644--- a/solr/solr-ref-guide/src/configsets-api.adoc+++ b/solr/solr-ref-guide/src/configsets-api.adoc@@ -19,17 +19,22 @@The Configsets API enables you to upload new configsets to ZooKeeper, create, and delete configsets when Solr is running SolrCloud mode.-Configsets are a collection of configuration files such as `solrconfig.xml`, `synonyms.txt`, the schema, language-specific files, and other collection-level configuration files (everything that normally lives in the `conf` directory). Solr ships with two example configsets (`_default` and `sample_techproducts_configs`) which can be used when creating collections. Using the same concept, you can create your own configsets and make them available when creating collections.+Configsets are a collection of configuration files such as `solrconfig.xml`, `synonyms.txt`, the schema, language-specific files, and other collection-level configuration files (everything that normally lives in the `conf` directory). Solr ships with two example configsets (`_default` and `sample_techproducts_configs`) which can be used when creating collections.+Using the same concept, you can create your own configsets and make them available when creating collections.This API provides a way to upload configuration files to ZooKeeper and share the same set of configuration files between two or more collections.Once a configset has been uploaded to ZooKeeper, use the configset name when creating the collection with the <<collections-api.adoc#,Collections API>> and the collection will use your configuration files.-Configsets do not have to be shared between collections if they are uploaded with this API, but this API makes it easier to do so if you wish. An alternative to uploading your configsets in advance would be to put the configuration files into a directory under `server/solr/configsets` and using the directory name as the `-d` parameter when using `bin/solr create` to create a collection.+Configsets do not have to be shared between collections if they are uploaded with this API, but this API makes it easier to do so if you wish.+An alternative to uploading your configsets in advance would be to put the configuration files into a directory under `server/solr/configsets` and using the directory name as the `-d` parameter when using `bin/solr create` to create a collection.-NOTE: This API can only be used with Solr running in SolrCloud mode. If you are not running Solr in SolrCloud mode but would still like to use shared configurations, please see the section <<config-sets.adoc#,Configsets>>.+NOTE: This API can only be used with Solr running in SolrCloud mode.+If you are not running Solr in SolrCloud mode but would still like to use shared configurations, please see the section <<config-sets.adoc#,Configsets>>.-The API works by passing commands to the `configs` endpoint. The path to the endpoint varies depending on the API being used: the v1 API uses `solr/admin/configs`, while the v2 API uses `api/cluster/configs`. Examples of both types are provided below.+The API works by passing commands to the `configs` endpoint.+The path to the endpoint varies depending on the API being used: the v1 API uses `solr/admin/configs`, while the v2 API uses `api/cluster/configs`.+Examples of both types are provided below.[[configsets-list]]== List Configsets@@ -83,20 +88,25 @@ The output will look like:Upload a configset, which is sent as a zipped file.A single, non-zipped file can also be uploaded with the `filePath` parameter.-This functionality is enabled by default, but can be disabled via a runtime parameter `-Dconfigset.upload.enabled=false`. Disabling this feature is advisable if you want to expose Solr installation to untrusted users (even though you should never do that!).+This functionality is enabled by default, but can be disabled via a runtime parameter `-Dconfigset.upload.enabled=false`.+Disabling this feature is advisable if you want to expose Solr installation to untrusted users (even though you should never do that!).-A configset is uploaded in a "trusted" mode if authentication is enabled and the upload operation is performed as an authenticated request. Without authentication, a configset is uploaded in an "untrusted" mode. Upon creation of a collection using an "untrusted" configset, the following functionality will not work:+A configset is uploaded in a "trusted" mode if authentication is enabled and the upload operation is performed as an authenticated request.+Without authentication, a configset is uploaded in an "untrusted" mode.+Upon creation of a collection using an "untrusted" configset, the following functionality will not work:* The XSLT transformer (`tr` parameter) cannot be used at request processing time.* If specified in the configset, the ScriptUpdateProcessorFactory will not initialize.-* Collections won't initialize if <lib> directives are used in the configset. (Note: Libraries added to Solr's classpath don't need the <lib> directive)+* Collections won't initialize if <lib> directives are used in the configset.+(Note: Libraries added to Solr's classpath don't need the <lib> directive)If you use any of these parameters or features, you must have enabled security features in your Solr installation and you must upload the configset as an authenticated user.The `upload` command takes the following parameters:`name`::-The configset to be created when the upload is complete. This parameter is required.+The configset to be created when the upload is complete.+This parameter is required.`overwrite`::If set to `true`, Solr will overwrite an existing configset with the same name (if false, the request will fail).@@ -104,7 +114,8 @@ If `filePath` is provided, then this option specifies whether the specified fileDefault is `false` when using the v1 API, but `true` when using the v2 API.`cleanup`::-When overwriting an existing configset (`overwrite=true`), this parameter tells Solr to delete the files in ZooKeeper that existed in the old configset but not in the one being uploaded. Default is `false`.+When overwriting an existing configset (`overwrite=true`), this parameter tells Solr to delete the files in ZooKeeper that existed in the old configset but not in the one being uploaded.+Default is `false`.This parameter cannot be set to true when `filePath` is used.`filePath`::@@ -112,7 +123,8 @@ This parameter allows the uploading of a single, non-zipped file to the given paThis functionality respects the `overwrite` parameter, so a request will fail if the given file path already exists in the configset and overwrite is set to `false`.The `cleanup` parameter cannot be set to true when `filePath` is used.-If uploading an entire configset, the body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file).+If uploading an entire configset, the body of the request should be a zip file that contains the configset.+The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file).Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet":@@ -207,10 +219,12 @@ If you have not yet uploaded any configsets, see the <<Upload a Configset>> commThe following parameters are supported when creating a configset.`name`::-The configset to be created. This parameter is required.+The configset to be created.+This parameter is required.`baseConfigSet`::-The name of the configset to copy as a base. This defaults to `_default`+The name of the configset to copy as a base.+This defaults to `_default``configSetProp._property_=_value_`::A configset property from the base configset to override in the copied configset.@@ -278,10 +292,12 @@ curl -X POST -H 'Content-type: application/json' -d '{[[configsets-delete]]== Delete a Configset-The `delete` command removes a configset. It does not remove any collections that were created with the configset.+The `delete` command removes a configset.+It does not remove any collections that were created with the configset.`name`::-The configset to be deleted. This parameter is required.+The configset to be deleted.+This parameter is required.To delete a configset named "myConfigSet":@@ -291,7 +307,8 @@ To delete a configset named "myConfigSet":====[.tab-label]*V1 API*-With the v1 API, the `delete` command must be capitalized as `DELETE`. The name of the configset to delete is provided with the `name` parameter:+With the v1 API, the `delete` command must be capitalized as `DELETE`.+The name of the configset to delete is provided with the `name` parameter:[source,bash]----@@ -303,7 +320,8 @@ http://localhost:8983/solr/admin/configs?action=DELETE&name=myConfigSet&omitHead====[.tab-label]*V2 API*-With the v2 API, the `delete` command is provided as the request method, as in `-X DELETE`. The name of the configset to delete is provided as a path parameter:+With the v2 API, the `delete` command is provided as the request method, as in `-X DELETE`.+The name of the configset to delete is provided as a path parameter:[source,bash]----diff --git a/solr/solr-ref-guide/src/configuration-apis.adoc b/solr/solr-ref-guide/src/configuration-apis.adocindex d3a0eeb9eb1..ef11bcf51f4 100644--- a/solr/solr-ref-guide/src/configuration-apis.adoc+++ b/solr/solr-ref-guide/src/configuration-apis.adoc@@ -1,5 +1,11 @@= Configuration APIs-:page-children: config-api, request-parameters-api, managed-resources+:page-children: config-api, \+ request-parameters-api, \+ managed-resources, \+ collections-api, \+ configsets-api, \+ coreadmin-api, \+ v2-api// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -19,6 +25,19 @@Solr includes several APIs that can be used to modify settings in `solrconfig.xml`.-* <<config-api.adoc#,Config API>>-* <<request-parameters-api.adoc#,Request Parameters API>>-* <<managed-resources.adoc#,Managed Resources>>+****+// This tags the below list so it can be used in the parent page section list+// tag::configapi-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<config-api.adoc#,Config API>>: Configure `solrconfig.xml`.+| <<request-parameters-api.adoc#,Request Parameters API>>: Override parameters in `solrconfig.xml`.+| <<managed-resources.adoc#,Managed Resources>>: Programmatic control over resource files.+| <<collections-api.adoc#,Collections API>>: Manage SolrCloud from cores to nodes.+| <<configsets-api.adoc#,Configsets API>>: Manage configsets.+| <<coreadmin-api.adoc#,CoreAdmin API>>: Manage Cores.+| <<v2-api.adoc#,V2 API>>: The v2 API structure.+|+|===+// end::configapi-sections[]+****diff --git a/solr/solr-ref-guide/src/configuration-files.adoc b/solr/solr-ref-guide/src/configuration-files.adocnew file mode 100644index 00000000000..f0781c795f3--- /dev/null+++ b/solr/solr-ref-guide/src/configuration-files.adoc@@ -0,0 +1,107 @@+= Solr Configuration Files+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr has several configuration files that you will interact with during your implementation.++Many of these files are in XML format, although APIs that interact with configuration settings tend to accept JSON for programmatic access as needed.++== Solr Home+When Solr runs, it needs access to a home directory.++When you first install Solr, your home directory is `server/solr`.+However, some examples may change this location (such as, if you run `bin/solr start -e cloud`, your home directory will be `example/cloud`).++The home directory contains important configuration information and is the place where Solr will store its index.+The layout of the home directory will look a little different when you are running Solr in a user-managed cluster or single-node installation vs. when you are running a SolrCloud cluster.++The crucial parts of the Solr home directory are shown in these examples:++.User-Managed Cluster or Single-Node+[source,plain]+----+<solr-home-directory>/+ solr.xml+ core_name1/+ core.properties+ conf/+ solrconfig.xml+ managed-schema+ data/+ core_name2/+ core.properties+ conf/+ solrconfig.xml+ managed-schema+ data/+----++.SolrCloud+[source,plain]+----+<solr-home-directory>/+ solr.xml+ core_name1/+ core.properties+ data/+ core_name2/+ core.properties+ data/+----++You may see other files, but the main ones you need to know are discussed below.++== Solr's Configuration Files+Inside Solr's Home, you'll find these files:++* `solr.xml` specifies configuration options for your Solr server instance.+For more information on `solr.xml` see <<configuring-solr-xml.adoc#,Configuring solr.xml>>.+* Per Solr Core:+** `core.properties` defines specific properties for each core such as its name, the collection the core belongs to, the location of the schema, and other parameters.+For more details on `core.properties`, see the section <<core-discovery.adoc#,Core Discovery>>.+** `solrconfig.xml` controls high-level behavior.+You can, for example, specify an alternate location for the data directory.+For more information on `solrconfig.xml`, see <<configuring-solrconfig-xml.adoc#,Configuring solrconfig.xml>>.+** `managed-schema` (or `schema.xml`) describes the documents you will ask Solr to index.+The schema defines a document as a collection of fields.+You can define both the field types and the fields themselves.+Field type definitions are powerful and include information about how Solr processes incoming field values and query values.+For more information on Solr schemas, see <<solr-schema.adoc#,Solr Schema>>.+** `data/` contains index files.++Note that the SolrCloud example does not include a `conf` directory for each Solr Core (so there is no `solrconfig.xml` or schema file).+This is because the configuration files usually found in the `conf` directory are stored in ZooKeeper so they can be propagated across the cluster.++If you are using SolrCloud with the embedded ZooKeeper instance, you may also see `zoo.cfg` and `zoo.data` which are ZooKeeper configuration and data files.+However, if you are running your own ZooKeeper ensemble, you would supply your own ZooKeeper configuration file when you start it and the copies in Solr would be unused.++== Files Screen++The Files screen in the Admin UI lets you browse & view configuration files (such `solrconfig.xml` and the schema file) for the collection you selected.++.The Files Screen+image::images/configuration-files/files-screen.png[Files screen,height=400]++If you are using <<cluster-types.adoc#solrcloud-mode,SolrCloud>>, the files displayed are the configuration files for this collection stored in ZooKeeper.+In user-managed clusters or single-node installations, all files in the `conf` directory are displayed.++The configuration files shown may or may not be used by the collection as use of the file depends on how they are referenced in either `solrconfig.xml` or your schema.++Configuration files cannot be edited with this screen, so a text editor of some kind must be used.++This screen is related to the <<schema-browser-screen.adoc#,Schema Browser Screen>>, in that they both can display information from the schema.+However, the Schema Browser provides a way to drill into the analysis chain and displays linkages between field types, fields, and dynamic field rules.diff --git a/solr/solr-ref-guide/src/configuration-guide.adoc b/solr/solr-ref-guide/src/configuration-guide.adocnew file mode 100644index 00000000000..d0c4e129998--- /dev/null+++ b/solr/solr-ref-guide/src/configuration-guide.adoc@@ -0,0 +1,74 @@+= Configuration Guide+:page-children: configuration-files, \+ property-substitution, \+ core-discovery, \+ configuring-solr-xml, \+ configuring-solrconfig-xml, \+ configuration-apis, \+ config-sets, \+ resource-loading, \+ solr-plugins+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+This section covers configuration files and options for customizing your Solr installation.++****+[discrete]+=== Configuration Overview++[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<configuration-files.adoc#,Solr Configuration Files>>: Solr's major configuration files.+| <<property-substitution.adoc#,Property Substitution in Configuration Files>>: Provide property values at startup or in shared property files.+| <<core-discovery.adoc#,Core Discovery>>: Placement of `core.properties` and available property options.+| <<config-sets.adoc#,Configsets>>: Use configsets to avoid duplicating effort when defining a new core.+| <<resource-loading.adoc#,Resource Loading>>: Resolving word lists, model files, and related data.+| <<configuring-solr-xml.adoc#,Configuring solr.xml>>: Global configuration options.+|===+****++****+[discrete]+=== solrconfig.xml++<<configuring-solrconfig-xml.adoc#,*Configuring solrconfig.xml*>>++// This pulls the sub-section list from the child page to reduce errors+include::configuring-solrconfig-xml.adoc[tag=solrconfig-sections]+****++****+[discrete]+=== Configuration APIs++<<configuration-apis.adoc#,*Configuration APIs*>>++// This pulls the sub-section list from the child page to reduce errors+include::configuration-apis.adoc[tag=configapi-sections]+****++****+[discrete]+=== Solr Plugins++<<solr-plugins.adoc#,*Solr Plugins*>>++// This pulls the sub-section list from the child page to reduce errors+include::solr-plugins.adoc[tag=plugin-sections]+****diff --git a/solr/solr-ref-guide/src/configuring-logging.adoc b/solr/solr-ref-guide/src/configuring-logging.adocindex 8d50cb8d147..8607710c3e3 100644--- a/solr/solr-ref-guide/src/configuring-logging.adoc+++ b/solr/solr-ref-guide/src/configuring-logging.adoc@@ -16,50 +16,65 @@// specific language governing permissions and limitations// under the License.-Solr logs are a key way to know what's happening in the system. There are several ways to adjust the default logging configuration.--[IMPORTANT]-====-In addition to the logging options described below, there is a way to configure which request parameters (such as parameters sent as part of queries) are logged with an additional request parameter called `logParamsList`. See the section on <<common-query-parameters.adoc#logparamslist-parameter,Common Query Parameters>> for more information.-====+Solr logs are a key way to know what's happening in the system.+There are several ways to adjust the default logging configuration.== Temporary Logging Settings-You can control the amount of logging output in Solr by using the Admin Web interface. Select the *LOGGING* link. Note that this page only lets you change settings in the running system and is not saved for the next run. (For more information about the Admin Web interface, see <<using-the-solr-administration-user-interface.adoc#,Using the Solr Administration User Interface>>.)+There are several ways to temporarily change log levels when needed.++=== Logging Screen+You can temporarily change the amount of logging output by Solr using the Admin UI.+Select the *Logging*] link in the left-hand menu.++Note that log levels will also be reset on next Solr restart..The Logging Screen-image::images/logging/logging.png[image]+image::images/configuring-logging/logging.png[Logging Screen]-This part of the Admin Web interface allows you to set the logging level for many different log categories. Fortunately, any categories that are *unset* will have the logging level of its parent. This makes it possible to change many categories at once by adjusting the logging level of their parent.+This part of the Admin Web interface allows you to set the logging level for many different log categories.+Fortunately, any categories that are *unset* will have the logging level of its parent.+This makes it possible to change many categories at once by adjusting the logging level of their parent.-When you select **Level**, you see the following menu:+When you select menu:Logging[Level], you see the following menu:.The Log Level Menu-image::images/logging/level_menu.png[image,width=1159,height=577]+image::images/configuring-logging/level_menu.png[image,width=1159,height=577]++Solr classes are shown in the left column in a directory tree structure representing the classpath.+The current level is shown in the right column.++Directories are shown with their current logging levels.+A row highlighted in yellow indicates that the class currently has logging enabled.+The Log Level Menu floats over these.+To set a log level for a particular directory, click the current level in the right column and the Log Level Menu will appear.+Select the button next to your desired log level.-Directories are shown with their current logging levels. The Log Level Menu floats over these. To set a log level for a particular directory, select it and click the appropriate log level button.The log level change will be distributed to all nodes in the cluster.-Log levels settings are as follows:+The possible log levels are as follows:[width="100%",options="header",]|===|Level |Result-|FINEST |Reports everything.-|FINE |Reports everything but the least important messages.-|CONFIG |Reports configuration errors.+|ALL |Reports everything.+|TRACE |Reports everything but the least important messages.+|DEBUG |Reports configuration errors.|INFO |Reports everything but normal status.|WARN |Reports all warnings.-|SEVERE |Reports only the most severe warnings.+|ERROR |Reports only the most severe warnings.+|FATAL |Reports only fatal events.|OFF |Turns off logging.|UNSET |Removes the previous log setting.|===Multiple settings at one time are allowed.-=== Loglevel API+=== Log Level API++There is also a way of sending REST commands to the `admin/info/logging` endpoint to do the same.-There is also a way of sending REST commands to the logging endpoint to do the same. Example:+Example:[source,bash]----@@ -67,13 +82,16 @@ There is also a way of sending REST commands to the logging endpoint to do the scurl -s http://localhost:8983/solr/admin/info/logging --data-binary "set=root:WARN"-----== Choosing Log Level at Startup+=== Choosing Log Level at Startup-You can temporarily choose a different logging level as you start Solr. There are two ways:+You can temporarily choose a different logging level as you start Solr.+There are two ways:-The first way is to set the `SOLR_LOG_LEVEL` environment variable before you start Solr, or place the same variable in `bin/solr.in.sh` or `bin/solr.in.cmd`. The variable must contain an uppercase string with a supported log level (see above).+The first way is to set the `SOLR_LOG_LEVEL` environment variable before you start Solr, or place the same variable in `bin/solr.in.sh` or `bin/solr.in.cmd`.+The variable must contain an uppercase string with a supported log level (see above).-The second way is to start Solr with the -v or -q options, see <<solr-control-script-reference.adoc#,Solr Control Script Reference>> for details. Examples:+The second way is to start Solr with the -v or -q options, see <<solr-control-script-reference.adoc#,Solr Control Script Reference>> for details.+Examples:[source,bash]----@@ -85,11 +103,16 @@ bin/solr start -f -q== Permanent Logging Settings-Solr uses http://logging.apache.org/log4j/log4j-{ivy-log4j-version}/[Log4J version {ivy-log4j-version}] for logging which is configured using `server/resources/log4j2.xml`. Take a moment to inspect the contents of the `log4j2.xml` file so that you are familiar with its structure. By default, Solr log messages will be written to `SOLR_LOGS_DIR/solr.log`.+Solr uses http://logging.apache.org/log4j/log4j-{ivy-log4j-version}/[Log4J version {ivy-log4j-version}] for logging which is configured using `server/resources/log4j2.xml`.+Take a moment to inspect the contents of the `log4j2.xml` file so that you are familiar with its structure.+By default, Solr log messages will be written to `SOLR_LOGS_DIR/solr.log`.-When you're ready to deploy Solr in production, set the variable `SOLR_LOGS_DIR` to the location where you want Solr to write log files, such as `/var/solr/logs`. You may also want to tweak `log4j2.xml`. Note that if you installed Solr as a service using the instructions provided in <<taking-solr-to-production.adoc#,Taking Solr to Production>>, then see `/var/solr/log4j2.xml` instead of the default `server/resources` version.+When you're ready to deploy Solr in production, set the variable `SOLR_LOGS_DIR` to the location where you want Solr to write log files, such as `/var/solr/logs`.+You may also want to tweak `log4j2.xml`.+Note that if you installed Solr as a service using the instructions provided in <<taking-solr-to-production.adoc#,Taking Solr to Production>>, then see `/var/solr/log4j2.xml` instead of the default `server/resources` version.-When starting Solr in the foreground (`-f` option), all logs will be sent to the console, in addition to `solr.log`. When starting Solr in the background, it will write all `stdout` and `stderr` output to a log file in `solr-<port>-console.log`, and automatically disable the CONSOLE logger configured in `log4j2.xml`, having the same effect as if you removed the CONSOLE appender from the rootLogger manually.+When starting Solr in the foreground (`-f` option), all logs will be sent to the console, in addition to `solr.log`.+When starting Solr in the background, it will write all `stdout` and `stderr` output to a log file in `solr-<port>-console.log`, and automatically disable the CONSOLE logger configured in `log4j2.xml`, having the same effect as if you removed the CONSOLE appender from the rootLogger manually.Also, in `log4j2.xml` if the default log rotation size threshold of 32MB is too small for production servers then you should increase it to a larger value (such as 100MB or more).@@ -100,19 +123,29 @@ Also, in `log4j2.xml` if the default log rotation size threshold of 32MB is tooJava Garbage Collection logs are rotated by the JVM when size hits 20M, for a max of 9 generations.-On every startup or restart of Solr, log4j2 performs log rotation. If you choose to use another log framework that does not support rotation on startup, you may enable `SOLR_LOG_PRESTART_ROTATION` in `bin/solr.in.sh` or `bin/solr.in.cmd` to let the start script rotate the logs on startup.+On every startup or restart of Solr, log4j2 performs log rotation.+If you choose to use another log framework that does not support rotation on startup, you may enable `SOLR_LOG_PRESTART_ROTATION` in `bin/solr.in.sh` or `bin/solr.in.cmd` to let the start script rotate the logs on startup.== Logging Slow Queries-For high-volume search applications, logging every query can generate a large amount of logs and, depending on the volume, potentially impact performance. If you mine these logs for additional insights into your application, then logging every query request may be useful.+For high-volume search applications, logging every query can generate a large amount of logs and, depending on the volume, potentially impact performance.+If you mine these logs for additional insights into your application, then logging every query request may be useful.-On the other hand, if you're only concerned about warnings and error messages related to requests, then you can set the log verbosity to WARN. However, this poses a potential problem in that you won't know if any queries are slow, as slow queries are still logged at the INFO level.+On the other hand, if you're only concerned about warnings and error messages related to requests, then you can set the log verbosity to WARN.+However, this poses a potential problem in that you won't know if any queries are slow, as slow queries are still logged at the INFO level.-Solr provides a way to set your log verbosity threshold to WARN and be able to set a latency threshold above which a request is considered "slow" and log that request at the WARN level to help you identify slow queries in your application. To enable this behavior, configure the `<slowQueryThresholdMillis>` element in the *query* section of `solrconfig.xml`:+Solr provides a way to set your log verbosity threshold to WARN and be able to set a latency threshold above which a request is considered "slow" and log that request at the WARN level to help you identify slow queries in your application.+To enable this behavior, configure the `<slowQueryThresholdMillis>` element in the *query* section of `solrconfig.xml`:[source,xml]----<slowQueryThresholdMillis>1000</slowQueryThresholdMillis>-----Any queries that take longer than the specified threshold will be logged as "slow" queries at the WARN level. The log file under which you can find all these queries is called `solr_slow_requests.log` and will be found in your `SOLR_LOGS_DIR` (see <<Permanent Logging Settings>> for more about defining log locations).+Any queries that take longer than the specified threshold will be logged as "slow" queries at the WARN level.+The log file under which you can find all these queries is called `solr_slow_requests.log` and will be found in your `SOLR_LOGS_DIR` (see <<Permanent Logging Settings>> for more about defining log locations).++== Logging Select Request Parameters++In addition to the logging options described above, it's possible to log only a selected list of request parameters (such as those sent with queries) with an additional request parameter called `logParamsList`.+See the section on <<common-query-parameters.adoc#logparamslist-parameter,logParamsList Parameter>> for more information.diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/configuring-solr-xml.adocsimilarity index 65%rename from solr/solr-ref-guide/src/format-of-solr-xml.adocrename to solr/solr-ref-guide/src/configuring-solr-xml.adocindex 1d9a1aee61e..0e0f74a2972 100644--- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc+++ b/solr/solr-ref-guide/src/configuring-solr-xml.adoc@@ -1,4 +1,4 @@-= Format of solr.xml+= Configuring solr.xml// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,11 +18,13 @@The `solr.xml` file defines some global configuration options that apply to all or many cores.-This section will describe the default `solr.xml` file included with Solr and how to modify it for your needs. For details on how to configure `core.properties`, see the section <<defining-core-properties.adoc#,Defining core.properties>>.+This section will describe the default `solr.xml` file included with Solr and how to modify it for your needs.+For details on how to configure `core.properties`, see the section <<core-discovery.adoc#,Core Discovery>>.== Defining solr.xml-You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or `/var/solr/data`) or optionally in ZooKeeper when using SolrCloud. The default `solr.xml` file looks like this:+You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or `/var/solr/data`) or optionally in ZooKeeper when using SolrCloud.+The default `solr.xml` file looks like this:[source,xml]----@@ -55,13 +57,15 @@ You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or</solr>-----As you can see, the discovery Solr configuration is "SolrCloud friendly". However, the presence of the `<solrcloud>` element does _not_ mean that the Solr instance is running in SolrCloud mode. Unless the `-DzkHost` or `-DzkRun` are specified at startup time, this section is ignored.+As you can see, the discovery Solr configuration is "SolrCloud friendly". However, the presence of the `<solrcloud>` element does _not_ mean that the Solr instance is running in SolrCloud mode.+Unless the `-DzkHost` or `-DzkRun` are specified at startup time, this section is ignored.== Solr.xml Parameters=== The <solr> Element-There are no attributes that you can specify in the `<solr>` tag, which is the root element of `solr.xml`. The tables below list the child nodes of each XML element in `solr.xml`.+There are no attributes that you can specify in the `<solr>` tag, which is the root element of `solr.xml`.+The tables below list the child nodes of each XML element in `solr.xml`.`configSetService`::This attribute does not need to be set.@@ -74,7 +78,8 @@ If this attribute isn't set, Solr uses the default `configSetService`, with zook`adminHandler`::This attribute does not need to be set.+-If used, this attribute should be set to the FQN (Fully qualified name) of a class that inherits from CoreAdminHandler. For example, `<str name="adminHandler">com.myorg.MyAdminHandler</str>` would configure the custom admin handler (MyAdminHandler) to handle admin requests.+If used, this attribute should be set to the FQN (Fully qualified name) of a class that inherits from CoreAdminHandler.+For example, `<str name="adminHandler">com.myorg.MyAdminHandler</str>` would configure the custom admin handler (MyAdminHandler) to handle admin requests.+If this attribute isn't set, Solr uses the default admin handler, `org.apache.solr.handler.admin.CoreAdminHandler`.@@ -99,33 +104,52 @@ The root of the core discovery tree, defaults to `$SOLR_HOME` (by default, `servCurrently non-operational.`sharedLib`::-Specifies the path to a common library directory that will be shared across all cores. Any JAR files in this directory will be added to the search path for Solr plugins. If the specified path is not absolute, it will be relative to `$SOLR_HOME`. Custom handlers may be placed in this directory. Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path.+Specifies the path to a common library directory that will be shared across all cores.+Any JAR files in this directory will be added to the search path for Solr plugins.+If the specified path is not absolute, it will be relative to `$SOLR_HOME`.+Custom handlers may be placed in this directory.+Note that specifying `sharedLib` will not remove `$SOLR_HOME/lib` from Solr's class path.`allowPaths`::-Solr will normally only access folders relative to `$SOLR_HOME`, `$SOLR_DATA_HOME` or `coreRootDir`. If you need to e.g., create a core outside of these paths, you can explicitly allow the path with `allowPaths`. It is a comma separated string of file system paths to allow. The special value of `*` will allow any path on the system.+Solr will normally only access folders relative to `$SOLR_HOME`, `$SOLR_DATA_HOME` or `coreRootDir`.+If you need to e.g., create a core outside of these paths, you can explicitly allow the path with `allowPaths`.+It is a comma separated string of file system paths to allow.+The special value of `*` will allow any path on the system.-[#_allow_urls]+[#allow-urls]`allowUrls`::-Comma-separated list of Solr hosts to allow. The http/https protocol may be omitted, and only the host and port are checked, i.e., `10.0.0.1:8983/solr,10.0.0.1:8984/solr`.-When running Solr in non-cloud mode and if planning to do distributed search (using the "shards" parameter), the list of hosts needs to be allowed or Solr will forbid the request. In Solr cloud mode, this allow-list is added to the cluster's live nodes to determine which hosts are allowed. The allow-list can also be configured with the `solr.allowUrls` system property (see its use in `solr.in.sh`).+Comma-separated list of Solr hosts to allow.+++The HTTP/HTTPS protocol may be omitted, and only the host and port are checked, i.e., `10.0.0.1:8983/solr,10.0.0.1:8984/solr`.+++When running Solr as a user-managed cluster and using the `shards` parameter, a list of hosts needs to be specifically configured as allowed or Solr will forbid the request.+++In SolrCloud mode, the allow-list is automatically configured to include all live nodes in the cluster.+++The allow-list can also be configured with the `solr.allowUrls` system property in `solr.in.sh` / `solr.in.cmd`.If you need to disable this feature for backwards compatibility, you can set the system property `solr.disable.allowUrls=true`.`shareSchema`::-This attribute, when set to `true`, ensures that the multiple cores pointing to the same Schema resource file will be referring to the same IndexSchema Object. Sharing the IndexSchema Object makes loading the core faster. If you use this feature, make sure that no core-specific property is used in your Schema file.+This attribute, when set to `true`, ensures that the multiple cores pointing to the same Schema resource file will be referring to the same IndexSchema Object.+Sharing the IndexSchema Object makes loading the core faster.+If you use this feature, make sure that no core-specific property is used in your Schema file.`transientCacheSize`::Defines how many cores with `transient=true` that can be loaded before swapping the least recently used core for a new core.`configSetBaseDir`::-The directory under which configsets for Solr cores can be found. Defaults to `$SOLR_HOME/configsets`.+The directory under which configsets for Solr cores can be found.+Defaults to `$SOLR_HOME/configsets`.[[global-maxbooleanclauses]]`maxBooleanClauses`::Sets the maximum number of (nested) clauses allowed in any query.+-This global limit provides a safety constraint on the total number of clauses allowed in any query against any collection -- regardless of whether those clauses were explicitly specified in a query string, or were the result of query expansion/re-writing from a more complex type of query based on the terms in the index. This limit is enforced at multiple points in the Lucene code base, both to prevent primative query objects (mainly `BooleanQuery`) from being constructed with an excessive number of clauses in a way that may exhaust the JVM Heap, but also to ensure that no composite query (made up of multiple primative queries) can be executed with an excessive _total_ number of nested clauses in a way that may cause a search thread to use excessive CPU.+This global limit provides a safety constraint on the total number of clauses allowed in any query against any collection -- regardless of whether those clauses were explicitly specified in a query string, or were the result of query expansion/re-writing from a more complex type of query based on the terms in the index.+This limit is enforced at multiple points in Lucene, both to prevent primitive query objects (mainly `BooleanQuery`) from being constructed with an excessive number of clauses in a way that may exhaust the JVM heap, but also to ensure that no composite query (made up of multiple primitive queries) can be executed with an excessive _total_ number of nested clauses in a way that may cause a search thread to use excessive CPU.+-In default configurations this property uses the value of the `solr.max.booleanClauses` system property if specified. This is the same system property used in the `_default` configset for the <<query-settings-in-solrconfig#maxbooleanclauses,`<maxBooleanClauses>` setting of `solrconfig.xml`>> making it easy for Solr administrators to increase both values (in all collections) without needing to search through and update all of their configs.+In default configurations this property uses the value of the `solr.max.booleanClauses` system property if specified.+This is the same system property used in the `_default` configset for the <<caches-warming.adoc#maxbooleanclauses-element,`<maxBooleanClauses>` element of `solrconfig.xml`>> making it easy for Solr administrators to increase both values (in all collections) without needing to search through and update all of their configs.+[source,xml]----@@ -134,7 +158,8 @@ In default configurations this property uses the value of the `solr.max.booleanC=== The <solrcloud> Element-This element defines several parameters that relate so SolrCloud. This section is ignored unless theSolr instance is started with either `-DzkRun` or `-DzkHost`+This element defines several parameters that relate so SolrCloud.+This section is ignored unless theSolr instance is started with either `-DzkRun` or `-DzkHost``distribUpdateConnTimeout`::Used to set the underlying `connTimeout` for intra-cluster updates.@@ -166,13 +191,15 @@ When trying to elect a leader for a shard, this property sets the maximum time aTypically, the default value of `180000` (ms) is sufficient for conflicts to be resolved; you may need to increase this value if you have hundreds or thousands of small collections in SolrCloud.`zkClientTimeout`::-A timeout for connection to a ZooKeeper server. It is used with SolrCloud.+A timeout for connection to a ZooKeeper server.+It is used with SolrCloud.`zkHost`::In SolrCloud mode, the URL of the ZooKeeper host that Solr should use for cluster state information.`genericCoreNodeNames`::-If `TRUE`, node names are not based on the address of the node, but on a generic name that identifies the core. When a different machine takes over serving that core things will be much easier to understand.+If `TRUE`, node names are not based on the address of the node, but on a generic name that identifies the core.+When a different machine takes over serving that core things will be much easier to understand.`zkCredentialsProvider` & `zkACLProvider`::Optional parameters that can be specified if you are using <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>.@@ -184,7 +211,8 @@ If `TRUE`, the internal behavior of SolrCloud is changed to not use the Overseer=== The <logging> Element`class`::-The class to use for logging. The corresponding JAR file must be available to Solr, perhaps through a `<lib>` directive in `solrconfig.xml`.+The class to use for logging.+The corresponding JAR file must be available to Solr, perhaps through a `<lib>` directive in `solrconfig.xml`.`enabled`::true/false - whether to enable logging or not.@@ -195,7 +223,8 @@ true/false - whether to enable logging or not.The number of log events that are buffered.`threshold`::-The logging level above which your particular logging implementation will record. For example when using log4j one might specify DEBUG, WARN, INFO, etc.+The logging level above which your particular logging implementation will record.+For example when using log4j one might specify DEBUG, WARN, INFO, etc.=== The <shardHandlerFactory> Element@@ -206,37 +235,50 @@ Custom shard handlers can be defined in `solr.xml` if you wish to create a custo<shardHandlerFactory name="ShardHandlerFactory" class="qualified.class.name">-----Since this is a custom shard handler, sub-elements are specific to the implementation. The default and only shard handler provided by Solr is the `HttpShardHandlerFactory` in which case, the following sub-elements can be specified:+Since this is a custom shard handler, sub-elements are specific to the implementation.+The default and only shard handler provided by Solr is the `HttpShardHandlerFactory` in which case, the following sub-elements can be specified:`socketTimeout`::-The read timeout for intra-cluster query and administrative requests. The default is the same as the `distribUpdateSoTimeout` specified in the `<solrcloud>` section.+The read timeout for intra-cluster query and administrative requests.+The default is the same as the `distribUpdateSoTimeout` specified in the `<solrcloud>` section.`connTimeout`::-The connection timeout for intra-cluster query and administrative requests. Defaults to the `distribUpdateConnTimeout` specified in the `<solrcloud>` section.+The connection timeout for intra-cluster query and administrative requests.+Defaults to the `distribUpdateConnTimeout` specified in the `<solrcloud>` section.`urlScheme`::The URL scheme to be used in distributed search.`maxConnectionsPerHost`::-Maximum connections allowed per host. Defaults to `100000`.+Maximum connections allowed per host.+Defaults to `100000`.`corePoolSize`::-The initial core size of the threadpool servicing requests. Default is `0`.+The initial core size of the threadpool servicing requests.+Default is `0`.`maximumPoolSize`::-The maximum size of the threadpool servicing requests. Default is unlimited.+The maximum size of the threadpool servicing requests.+Default is unlimited.`maxThreadIdleTime`::-The amount of time in seconds that idle threads persist for in the queue, before being killed. Default is `5` seconds.+The amount of time in seconds that idle threads persist for in the queue, before being killed.+Default is `5` seconds.`sizeOfQueue`::-If the threadpool uses a backing queue, what is its maximum size to use direct handoff. Default is to use a SynchronousQueue.+If the threadpool uses a backing queue, what is its maximum size to use direct handoff.+Default is to use a SynchronousQueue.`fairnessPolicy`::-A boolean to configure if the threadpool favors fairness over throughput. Default is false to favor throughput.+A boolean to configure if the threadpool favors fairness over throughput.+Default is false to favor throughput.`replicaRouting`::-A NamedList specifying replica routing preference configuration. This may be used to select and configure replica routing preferences. `default=true` may be used to set the default base replica routing preference. Only positive default status assertions are respected; i.e., `default=false` has no effect. If no explicit default base replica routing preference is configured, the implicit default will be `random`.+A NamedList specifying replica routing preference configuration.+This may be used to select and configure replica routing preferences.+`default=true` may be used to set the default base replica routing preference.+Only positive default status assertions are respected; i.e., `default=false` has no effect.+If no explicit default base replica routing preference is configured, the implicit default will be `random`.----<shardHandlerFactory class="HttpShardHandlerFactory"><lst name="replicaRouting">@@ -248,19 +290,26 @@ A NamedList specifying replica routing preference configuration. This may be use</lst></shardHandlerFactory>-----Replica routing may also be specified (overriding defaults) per-request, via the `shards.preference` request parameter. If a request contains both `dividend` and `hash`, `dividend` takes priority for routing. For configuring `stable` routing, the `hash` parameter implicitly defaults to a hash of the String value of the main query parameter (i.e., `q`).+Replica routing may also be specified (overriding defaults) per-request, via the `shards.preference` request parameter.+If a request contains both `dividend` and `hash`, `dividend` takes priority for routing.+For configuring `stable` routing, the `hash` parameter implicitly defaults to a hash of the String value of the main query parameter (i.e., `q`).+-The `dividend` parameter must be configured explicitly; there is no implicit default. If only `dividend` routing is desired, `hash` may be explicitly set to the empty string, entirely disabling implicit hash-based routing.+The `dividend` parameter must be configured explicitly; there is no implicit default.+If only `dividend` routing is desired, `hash` may be explicitly set to the empty string, entirely disabling implicit hash-based routing.=== The <metrics> Element-The `<metrics>` element in `solr.xml` allows you to customize the metrics reported by Solr. You can define system properties that should not be returned, or define custom suppliers and reporters.+The `<metrics>` element in `solr.xml` allows you to customize the metrics reported by Solr.+You can define system properties that should not be returned, or define custom suppliers and reporters.-In a default `solr.xml` you will not see any `<metrics>` configuration. If you would like to customize the metrics for your installation, see the section <<metrics-reporting.adoc#metrics-configuration,Metrics Configuration>>.+In a default `solr.xml` you will not see any `<metrics>` configuration.+If you would like to customize the metrics for your installation, see the section <<metrics-reporting.adoc#metrics-configuration,Metrics Configuration>>.== Substituting JVM System Properties in solr.xml-Solr supports variable substitution of JVM system property values in `solr.xml`, which allows runtime specification of various configuration options. The syntax is `${propertyname[:option default value]}`. This allows defining a default that can be overridden when Solr is launched. If a default value is not specified, then the property must be specified at runtime or the `solr.xml` file will generate an error when parsed.+Solr supports variable substitution of JVM system property values in `solr.xml`, which allows runtime specification of various configuration options.+The syntax is `${propertyname[:option default value]}`. This allows defining a default that can be overridden when Solr is launched.+If a default value is not specified, then the property must be specified at runtime or the `solr.xml` file will generate an error when parsed.Any JVM system properties usually specified using the `-D` flag when starting the JVM, can be used as variables in the `solr.xml` file.diff --git a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adocindex 814d3518e67..c91f031208b 100644--- a/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc+++ b/solr/solr-ref-guide/src/configuring-solrconfig-xml.adoc@@ -1,16 +1,17 @@= Configuring solrconfig.xml-:page-children: datadir-and-directoryfactory-in-solrconfig, \- schema-factory-definition-in-solrconfig, \- indexconfig-in-solrconfig, \- requesthandlers-and-searchcomponents-in-solrconfig, \- initparams-in-solrconfig, \- updatehandlers-in-solrconfig, \- query-settings-in-solrconfig, \- requestdispatcher-in-solrconfig, \+:page-children: index-location-format, \+ index-segments-merging, \+ schema-factory, \+ commits-transaction-logs, \+ caches-warming, \+ requesthandlers-searchcomponents, \+ implicit-requesthandlers, \+ realtime-get, \+ initparams, \+ requestdispatcher, \update-request-processors, \- script-update-processor, \+ script-update-processor, \codec-factory-// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -42,165 +43,32 @@ In `solrconfig.xml`, you configure important features such as:* the Admin Web interface-* parameters related to replication and duplication (these parameters are covered in detail in <<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>)--The `solrconfig.xml` file is located in the `conf/` directory for each collection. Several well-commented example files can be found in the `server/solr/configsets/` directories demonstrating best practices for many different types of installations.--We've covered the options in the following sections:+* parameters related to replication and duplication when not running in SolrCloud mode-* <<datadir-and-directoryfactory-in-solrconfig.adoc#,DataDir and DirectoryFactory in SolrConfig>>-* <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>>-* <<indexconfig-in-solrconfig.adoc#,IndexConfig in SolrConfig>>-* <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,RequestHandlers and SearchComponents in SolrConfig>>-* <<initparams-in-solrconfig.adoc#,InitParams in SolrConfig>>-* <<updatehandlers-in-solrconfig.adoc#,UpdateHandlers in SolrConfig>>-* <<query-settings-in-solrconfig.adoc#,Query Settings in SolrConfig>>-* <<requestdispatcher-in-solrconfig.adoc#,RequestDispatcher in SolrConfig>>-* <<update-request-processors.adoc#,Update Request Processors>>-* <<codec-factory.adoc#,Codec Factory>>+The `solrconfig.xml` file is located in the `conf/` directory for each collection.+Several well-commented example files can be found in the `server/solr/configsets/` directories demonstrating best practices for many different types of installations.-Some SolrConfig aspects are covered in other sections.+Some `solrconfig.xml` aspects are documented in other sections.See <<libs.adoc#lib-directives-in-solrconfig,lib directives in SolrConfig>>, which can be used for both Plugins and Resources.-== Substituting Properties in Solr Config Files--Solr supports variable substitution of property values in configuration files, which allows runtime specification of various configuration options in `solrconfig.xml`. The syntax is `${propertyname[:option default value]`}. This allows defining a default that can be overridden when Solr is launched. If a default value is not specified, then the property _must_ be specified at runtime or the configuration file will generate an error when parsed.--There are multiple methods for specifying properties that can be used in configuration files. Of those below, strongly consider "config overlay" as the preferred approach, as it stays local to the configset and is easy to modify.--=== JVM System Properties--Any JVM System properties, usually specified using the `-D` flag when starting the JVM, can be used as variables in any XML configuration file in Solr.--For example, in the sample `solrconfig.xml` files, you will see this value which defines the locking type to use:--[source,xml]------<lockType>${solr.lock.type:native}</lockType>-------Which means the lock type defaults to "native" but when starting Solr, you could override this using a JVM system property by launching the Solr it with:--[source,bash]------bin/solr start -Dsolr.lock.type=none-------In general, any Java system property that you want to set can be passed through the `bin/solr` script using the standard `-Dproperty=value` syntax. Alternatively, you can add common system properties to the `SOLR_OPTS` environment variable defined in the Solr include file (`bin/solr.in.sh` or `bin/solr.in.cmd`). For more information about how the Solr include file works, refer to: <<taking-solr-to-production.adoc#,Taking Solr to Production>>.--=== Config API to Override solrconfig.xml--The <<config-api.adoc#,Config API>> allows you to use an API to modify Solr's configuration, specifically user defined properties. Changes made with this API are stored in a file named `configoverlay.json`. This file should only be edited with the API, but will look like this example:--[source,json]------{- "userProps":{"update.autoCreateFields":"false"},- "requestHandler":{"/myterms":{- "name":"/myterms",- "class":"solr.SearchHandler",- "defaults":{- "terms":true,- "distrib":false},- "components":["terms"]}}}-------For more details, see the section <<config-api.adoc#,Config API>>.--=== solrcore.properties--If the configuration directory for a Solr core contains a file named `solrcore.properties` that file can contain any arbitrary user-defined property names and values using the Java https://en.wikipedia.org/wiki/.properties[properties file format]. Those properties can then be used as variables in other configuration files for that Solr core.--For example, the following `solrcore.properties` file could be created in the `conf/` directory of a collection using one of the example configurations, to override the lockType used.--[source,properties]------#conf/solrcore.properties-solr.lock.type=none-------.Deprecation-[WARNING]-====-`solrcore.properties` won't work in SolrCloud mode (it is not read from ZooKeeper). This feature is likely to be removed in the future. Instead, use another mechanism like a config overlay.-====--[IMPORTANT]-====--The path and name of the `solrcore.properties` file can be overridden using the `properties` property in <<defining-core-properties.adoc#,`core.properties`>>.--====--=== User-Defined Properties in core.properties--Every Solr core has a `core.properties` file, automatically created when using the APIs. When you create a SolrCloud collection, you can pass through custom parameters by prefixing the parameter name with `_property.name_` as a parameter.--For example, to add a property named "my.custom.prop":--[.dynamic-tabs]----[example.tab-pane#v1customprop]-====-[.tab-label]*V1 API*--[source,bash]------http://localhost:8983/solr/admin/collections?action=CREATE&name=gettingstarted&numShards=1&property.my.custom.prop=edismax------====--[example.tab-pane#v2]-====-[.tab-label]*V2 API*--[source,bash]------curl -X POST -H 'Content-type: application/json' -d '{"create": {"name": "gettingstarted", "numShards": "1", "property.my.custom.prop": "edismax"}}' http://localhost:8983/api/collections------====-----This will create a `core.properties` file that has at least the following properties (others omitted for brevity):--[source,properties]------#core.properties-name=gettingstarted-my.custom.prop=edismax-------The `my.custom.prop` property can then be used as a variable, such as in `solrconfig.xml`:--[source,xml]------<requestHandler name="/select">- <lst name="defaults">- <str name="defType">${my.custom.prop}</str>- </lst>-</requestHandler>-------=== Implicit Core Properties--Several attributes of a Solr core are available as "implicit" properties that can be used in variable substitution, independent of where or how the underlying value is initialized.--For example, regardless of whether the name for a particular Solr core is explicitly configured in `core.properties` or inferred from the name of the instance directory, the implicit property `solr.core.name` is available for use as a variable in that core's configuration file:--[source,xml]------<requestHandler name="/select">- <lst name="defaults">- <str name="collection_name">${solr.core.name}</str>- </lst>-</requestHandler>-------All implicit properties use the `solr.core.` name prefix, and reflect the runtime value of the equivalent <<defining-core-properties.adoc#,`core.properties` property>>:--* `solr.core.name`-* `solr.core.config`-* `solr.core.schema`-* `solr.core.dataDir`-* `solr.core.transient`-* `solr.core.loadOnStartup`+****+// This tags the below list so it can be used in the parent page section list+// tag::solrconfig-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<index-location-format.adoc#,Index Location and Format>>: Where and how Solr's indexes are stored.+| <<index-segments-merging.adoc#,Index Segments and Merging>>: Lucene index writers, including segment management, merges, and locks.+| <<schema-factory.adoc#,Schema Factory>>: Schema file formats.+| <<commits-transaction-logs.adoc#,Commits and Transaction Logs>>: Update requests and commit settings.+| <<caches-warming.adoc#,Caches and Query Warming>>: Caches, query warming, and query listeners.+| <<requesthandlers-searchcomponents.adoc#,Request Handlers and Search Components>>: Request processors and handlers for search features.+| <<implicit-requesthandlers.adoc#,Implicit Request Handlers>>: Request end-points automatically provided by Solr.+| <<realtime-get.adoc#,RealTime Get>>: Get the latest version of a document without opening a searcher.+| <<initparams.adoc#,InitParams>>: Default parameters for request handlers.+| <<requestdispatcher.adoc#,RequestDispatcher>>: Advanced request parsing and HTTP cache headers.+| <<update-request-processors.adoc#,Update Request Processors>>: Plugins for update requests.+| <<script-update-processor.adoc#,Script Update Processor>>: Java scripting engines during document updates.+| <<codec-factory.adoc#,Codec Factory>>: Lucene codecs when writing data to disk.+|===+//end::solrconfig-sections[]+****diff --git a/solr/solr-ref-guide/src/content-streams.adoc b/solr/solr-ref-guide/src/content-streams.adocindex f3077878f18..9e87596efe4 100644--- a/solr/solr-ref-guide/src/content-streams.adoc+++ b/solr/solr-ref-guide/src/content-streams.adoc@@ -18,24 +18,29 @@Content streams are bulk data passed with a request to Solr.-When Solr RequestHandlers are accessed using path based URLs, the `SolrQueryRequest` object containing the parameters of the request may also contain a list of ContentStreams containing bulk data for the request. (The name SolrQueryRequest is a bit misleading: it is involved in all requests, regardless of whether it is a query request or an update request.)+When Solr RequestHandlers are accessed using path based URLs, the `SolrQueryRequest` object containing the parameters of the request may also contain a list of ContentStreams containing bulk data for the request.+(The name SolrQueryRequest is a bit misleading: it is involved in all requests, regardless of whether it is a query request or an update request.)== Content Stream SourcesCurrently request handlers can get content streams in a variety of ways:* For multipart file uploads, each file is passed as a stream.-* For POST requests where the content-type is not `application/x-www-form-urlencoded`, the raw POST body is passed as a stream. The full POST body is parsed as parameters and included in the Solr parameters.+* For POST requests where the content-type is not `application/x-www-form-urlencoded`, the raw POST body is passed as a stream.+The full POST body is parsed as parameters and included in the Solr parameters.* The contents of parameter `stream.body` is passed as a stream.* If remote streaming is enabled and URL content is called for during request handling, the contents of each `stream.url` and `stream.file` parameters are fetched and passed as a stream.-By default, curl sends a `contentType="application/x-www-form-urlencoded"` header. If you need to test a SolrContentHeader content stream, you will need to set the content type with curl's `-H` flag.+By default, curl sends a `contentType="application/x-www-form-urlencoded"` header.+If you need to test a SolrContentHeader content stream, you will need to set the content type with curl's `-H` flag.== Remote Streaming-Remote streaming lets you send the contents of a URL as a stream to a given Solr RequestHandler. You could use remote streaming to send a remote or local file to an update plugin.+Remote streaming lets you send the contents of a URL as a stream to a given Solr RequestHandler.+You could use remote streaming to send a remote or local file to an update plugin.-Remote streaming is disabled by default. Enabling it is not recommended in a production situation without additional security between you and untrusted remote clients.+Remote streaming is disabled by default.+Enabling it is not recommended in a production situation without additional security between you and untrusted remote clients.In `solrconfig.xml`, you can enable it by changing the following `enableRemoteStreaming` parameter to `true`:@@ -75,7 +80,8 @@ curl -X POST -H 'Content-type: application/json' -d '{"set-property": {"requestD[IMPORTANT]====-If `enableRemoteStreaming="true"` is used, be aware that this allows _anyone_ to send a request to any URL or local file. If the <<Debugging Requests,DumpRequestHandler>> is enabled, it will allow anyone to view any file on your system.+If `enableRemoteStreaming="true"` is used, be aware that this allows _anyone_ to send a request to any URL or local file.+If the <<Debugging Requests,DumpRequestHandler>> is enabled, it will allow anyone to view any file on your system.====The source of the data can be compressed using gzip, and Solr will generally detect this.@@ -84,4 +90,5 @@ Gzip doesn't apply to `stream.body`.== Debugging Requests-The implicit "dump" RequestHandler (see <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>>) simply outputs the contents of the Solr QueryRequest using the specified writer type `wt`. This is a useful tool to help understand what streams are available to the RequestHandlers.+The implicit "dump" RequestHandler (see <<implicit-requesthandlers.adoc#,Implicit Request Handlers>>) simply outputs the contents of the Solr QueryRequest using the specified writer type `wt`.+This is a useful tool to help understand what streams are available to the RequestHandlers.diff --git a/solr/solr-ref-guide/src/controlling-results.adoc b/solr/solr-ref-guide/src/controlling-results.adocnew file mode 100644index 00000000000..9e623265bf3--- /dev/null+++ b/solr/solr-ref-guide/src/controlling-results.adoc@@ -0,0 +1,58 @@+= Controlling Results+:page-children: faceting, \+ json-facet-api, \+ collapse-and-expand-results, \+ result-grouping, \+ result-clustering, \+ highlighting, \+ query-elevation-component, \+ document-transformers, \+ response-writers, \+ exporting-result-sets, \+ pagination-of-results+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Once users submit a query, Solr provides a number of options for how to present the results.++Features like facets, grouping, collapsing, and clustering provide ways to group similar results together.++Highlighting shows users their query terms in context with surrounding text, helping them decide if a document "matches" their query.++Solr offers several ways to get results, or control how "pages" of results are returned to your client.++****+// This tags the below list so it can be used in the parent page section list+// tag::results-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<faceting.adoc#,Faceting>>: Categorize search results based on indexed terms.+| <<json-facet-api.adoc#facet-analytics-module,JSON Facet API>>: JSON Facet API.+| <<collapse-and-expand-results.adoc#,Collapse and Expand Results>>: Collapse documents into groups and expand the results.+| <<result-grouping.adoc#,Result Grouping>>: Group results based on common field values.+| <<result-clustering.adoc#,Result Clustering>>: Group search results based on cluster analysis applied to text fields.+| <<highlighting.adoc#,Highlighting>>: Highlighting search terms in document snippets.+| <<query-elevation-component.adoc#,Query Elevation Component>>: Force documents to the top of the results for certain queries.+| <<document-transformers.adoc#,Document Transformers>>: Compute information and add to individual documents.+| <<response-writers.adoc#,Response Writers>>: Format options for search results.+| <<exporting-result-sets.adoc#,Exporting Result Sets>>: Export large result sets out of Solr.+| <<pagination-of-results.adoc#,Pagination of Results>>: Offering paginated results.+|+|===+// end::results-sections[]+****diff --git a/solr/solr-ref-guide/src/copying-fields.adoc b/solr/solr-ref-guide/src/copy-fields.adocsimilarity index 55%rename from solr/solr-ref-guide/src/copying-fields.adocrename to solr/solr-ref-guide/src/copy-fields.adocindex a1e388f1446..2404b702b8e 100644--- a/solr/solr-ref-guide/src/copying-fields.adoc+++ b/solr/solr-ref-guide/src/copy-fields.adoc@@ -1,4 +1,4 @@-= Copying Fields+= Copy Fields// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,8 @@// specific language governing permissions and limitations// under the License.-You might want to interpret some document fields in more than one way. Solr has a mechanism for making copies of fields so that you can apply several distinct field types to a single piece of incoming information.+You might want to interpret some document fields in more than one way.+Solr has a mechanism for making copies of fields so that you can apply several distinct field types to a single piece of incoming information.The name of the field you want to copy is the _source_, and the name of the copy is the _destination_. In `schema.xml`, it's very simple to make copies of fields:@@ -25,15 +26,22 @@ The name of the field you want to copy is the _source_, and the name of the copy<copyField source="cat" dest="text" maxChars="30000" />-----In this example, we want Solr to copy the `cat` field to a field named `text`. Fields are copied before <<understanding-analyzers-tokenizers-and-filters.adoc#,analysis>> is done, meaning you can have two fields with identical original content, but which use different analysis chains and are stored in the index differently.+In this example, we want Solr to copy the `cat` field to a field named `text`.+Fields are copied before <<document-analysis.adoc#,analysis>> is done, meaning you can have two fields with identical original content, but which use different analysis chains and are stored in the index differently.-In the example above, if the `text` destination field has data of its own in the input documents, the contents of the `cat` field will be added as additional values – just as if all of the values had originally been specified by the client. Remember to configure your fields as `multivalued="true"` if they will ultimately get multiple values (either from a multivalued source or from multiple `copyField` directives).+In the example above, if the `text` destination field has data of its own in the input documents, the contents of the `cat` field will be added as additional values – just as if all of the values had originally been specified by the client.+Remember to configure your fields as `multivalued="true"` if they will ultimately get multiple values (either from a multivalued source or from multiple `copyField` directives).-A common usage for this functionality is to create a single "search" field that will serve as the default query field when users or clients do not specify a field to query. For example, `title`, `author`, `keywords`, and `body` may all be fields that should be searched by default, with copy field rules for each field to copy to a `catchall` field (for example, it could be named anything). Later you can set a rule in `solrconfig.xml` to search the `catchall` field by default. One caveat to this is your index will grow when using copy fields. However, whether this becomes problematic for you and the final size will depend on the number of fields being copied, the number of destination fields being copied to, the analysis in use, and the available disk space.+A common usage for this functionality is to create a single "search" field that will serve as the default query field when users or clients do not specify a field to query.+For example, `title`, `author`, `keywords`, and `body` may all be fields that should be searched by default, with copy field rules for each field to copy to a `catchall` field (for example, it could be named anything). Later you can set a rule in `solrconfig.xml` to search the `catchall` field by default.+One caveat to this is your index will grow when using copy fields.+However, whether this becomes problematic for you and the final size will depend on the number of fields being copied, the number of destination fields being copied to, the analysis in use, and the available disk space.-The `maxChars` parameter, an `int` parameter, establishes an upper limit for the number of characters to be copied from the source value when constructing the value added to the destination field. This limit is useful for situations in which you want to copy some data from the source field, but also control the size of index files.+The `maxChars` parameter, an `int` parameter, establishes an upper limit for the number of characters to be copied from the source value when constructing the value added to the destination field.+This limit is useful for situations in which you want to copy some data from the source field, but also control the size of index files.-Both the source and the destination of `copyField` can contain either leading or trailing asterisks, which will match anything. For example, the following line will copy the contents of all incoming fields that match the wildcard pattern `*_t` to the text field.:+Both the source and the destination of `copyField` can contain either leading or trailing asterisks, which will match anything.+For example, the following line will copy the contents of all incoming fields that match the wildcard pattern `*_t` to the text field.:[source,xml]----@@ -42,10 +50,13 @@ Both the source and the destination of `copyField` can contain either leading or[IMPORTANT]====-The `copyField` command can use a wildcard (*) character in the `dest` parameter only if the `source` parameter contains one as well. `copyField` uses the matching glob from the source field for the `dest` field name into which the source content is copied.+The `copyField` command can use a wildcard (*) character in the `dest` parameter only if the `source` parameter contains one as well.+`copyField` uses the matching glob from the source field for the `dest` field name into which the source content is copied.====-Copying is done at the stream source level and no copy feeds into another copy. This means that copy fields cannot be chained i.e., _you cannot_ copy from `here` to `there` and then from `there` to `elsewhere`. However, the same source field can be copied to multiple destination fields:+Copying is done at the stream source level and no copy feeds into another copy.+This means that copy fields cannot be chained i.e., _you cannot_ copy from `here` to `there` and then from `there` to `elsewhere`.+However, the same source field can be copied to multiple destination fields:[source,xml]----diff --git a/solr/solr-ref-guide/src/core-discovery.adoc b/solr/solr-ref-guide/src/core-discovery.adocnew file mode 100644index 00000000000..ed2107fa35c--- /dev/null+++ b/solr/solr-ref-guide/src/core-discovery.adoc@@ -0,0 +1,131 @@+= Core Discovery+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Core discovery means that creating a core is as simple as a `core.properties` file located on disk.++== The core.properties File++In Solr, the term _core_ is used to refer to a single index and associated transaction log and configuration files (including the `solrconfig.xml` and schema files, among others).+Your Solr installation can have multiple cores if needed, which allows you to index data with different structures in the same server, and maintain control over how your data is presented to different audiences.+In SolrCloud mode you will be more familiar with the term _collection_.+Behind the scenes a collection consists of one or more cores.++Cores can be created using `bin/solr` script or as part of SolrCloud collection creation using the APIs.+Core-specific properties (such as the directories to use for the indexes or configuration files, the core name, and other options) are defined in a `core.properties` file.+Any `core.properties` file in any directory of your Solr installation (or in a directory under where `solr_home` is defined) will be found by Solr and the defined properties will be used for the core named in the file.++The `core.properties` file is a simple Java Properties file where each line is just a key=value pair, e.g., `name=core1`.+Notice that no quotes are required.++==+A minimal `core.properties` file looks like the example below.+However, it can also be empty, see information on placement of `core.properties` below.++[source,bash]+----+name=my_core_name+----++== Placement of core.properties++Solr cores are configured by placing a file named `core.properties` in a sub-directory under `solr.home`.+There are no a-priori limits to the depth of the tree, nor are there limits to the number of cores that can be defined.+Cores may be anywhere in the tree with the exception that cores may _not_ be defined under an existing core.+That is, the following is not allowed:++[source,text]+----+./cores/core1/core.properties+./cores/core1/coremore/core5/core.properties+----++In this example, the enumeration will stop at "core1".++The following is legal:++[source,text]+----+./cores/somecores/core1/core.properties+./cores/somecores/core2/core.properties+./cores/othercores/core3/core.properties+./cores/extracores/deepertree/core4/core.properties+----++It is possible to segment Solr into multiple cores, each with its own configuration and indices.+Cores may be dedicated to a single application or to very different ones, but all are administered through a common administration interface.+You can create new Solr cores on the fly, shutdown cores, even replace one running core with another, all without ever stopping or restarting Solr.++Your `core.properties` file can be empty if necessary.+Suppose `core.properties` is located in `./cores/core1` (relative to `solr_home`) but is empty.+In this case, the core name is assumed to be "core1".+The instanceDir will be the folder containing `core.properties` (i.e., `./cores/core1`).+The dataDir will be `../cores/core1/data`, etc.++[NOTE]+====+You can run Solr without configuring any cores.+====++== Defining core.properties Files++The minimal `core.properties` file is an empty file, in which case all of the properties are defaulted appropriately.++Java properties files allow the hash (`#`) or bang (`!`) characters to specify comment-to-end-of-line.++The following properties are available:++`name`:: The name of the SolrCore.+You'll use this name to reference the SolrCore when running commands with the `CoreAdminHandler`.++`config`:: The configuration file name for a given core.+The default is `solrconfig.xml`.++`schema`:: The schema file name for a given core.+The default is `schema.xml` but please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use.+See <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for more details.++`dataDir`:: The core's data directory (where indexes are stored) as either an absolute pathname, or a path relative to the value of `instanceDir`.+This is `data` by default.++`configSet`:: The name of a defined configset, if desired, to use to configure the core (see the section <<config-sets.adoc#,Configsets>> for more details).++`properties`:: The name of the properties file for this core.+The value can be an absolute pathname or a path relative to the value of `instanceDir`.++`transient`:: If `true`, the core can be unloaded if Solr reaches the `transientCacheSize`.+The default is `false`.+Cores are unloaded in order of least recently used first.+_Setting this to `true` is not recommended in SolrCloud mode._++`loadOnStartup`:: If `true`, the default, the core will loaded when Solr starts.+_Setting this to `false` is not recommended in SolrCloud mode._++`coreNodeName`:: Used only in SolrCloud, this is a unique identifier for the node hosting this replica.+By default a `coreNodeName` is generated automatically, but setting this attribute explicitly allows you to manually assign a new core to replace an existing replica.+For example, this can be useful when replacing a machine that has had a hardware failure by restoring from backups on a new machine with a new hostname or port.++`ulogDir`:: The absolute or relative directory for the update log for this core (SolrCloud only).++`shard`:: The shard to assign this core to (SolrCloud only).++`collection`:: The name of the collection this core is part of (SolrCloud only).++`roles`:: Future parameter for SolrCloud or a way for users to mark nodes for their own use.++Additional user-defined properties may be specified for use as variables.+For more information on how to define local properties, see the section <<property-substitution.adoc#,Property Substitution in `solrconfig.xml`>>.diff --git a/solr/solr-ref-guide/src/core-specific-tools.adoc b/solr/solr-ref-guide/src/core-specific-tools.adocdeleted file mode 100644index bd23a11d3be..00000000000--- a/solr/solr-ref-guide/src/core-specific-tools.adoc+++ /dev/null@@ -1,47 +0,0 @@-= Core-Specific Tools-:page-children: ping, plugins-stats-screen, replication-screen, segments-info-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Core-Specific tools are a group of UI screens that allow you to see core-level information.--In the left-hand navigation bar, you will see a pull-down menu titled "Core Selector". Clicking on the menu will show a list of Solr cores hosted on this Solr node, with a search box that can be used to find a specific core by name.--When you select a core from the pull-down, the main display of the page will show some basic metadata about the core, and a secondary menu will appear in the left nav with links to additional core specific administration screens.--.Core overview screen-image::images/core-specific-tools/core_dashboard.png[image,width=515,height=250]--The core-specific UI screens are listed below, with a link to the section of this guide to find out more:--// TODO: SOLR-10655 BEGIN: refactor this into a 'core-screens-list.include.adoc' file for reuse-* <<ping.adoc#,Ping>> - lets you ping a named core and determine whether the core is active.-* <<plugins-stats-screen.adoc#,Plugins/Stats>> - shows statistics for plugins and other installed components.-* <<replication-screen.adoc#,Replication>> - shows you the current replication status for the core, and lets you enable/disable replication.-* <<segments-info.adoc#,Segments Info>> - Provides a visualization of the underlying Lucene index segments.-// TODO: SOLR-10655 END--If you are running a single node instance of Solr, additional UI screens normally displayed on a per-collection bases will also be listed:--// TODO: SOLR-10655 BEGIN: refactor this into a 'collection-screens-list.include.adoc' file for reuse-* <<analysis-screen.adoc#,Analysis>> - lets you analyze the data found in specific fields.-* <<documents-screen.adoc#,Documents>> - provides a simple form allowing you to execute various Solr indexing commands directly from the browser.-* <<files-screen.adoc#,Files>> - shows the current core configuration files such as `solrconfig.xml`.-* <<query-screen.adoc#,Query>> - lets you submit a structured query about various elements of a core.-* <<stream-screen.adoc#,Stream>> - allows you to submit streaming expressions and see results and parsing explanations.-* <<schema-browser-screen.adoc#,Schema Browser>> - displays schema data in a browser window.-// TODO: SOLR-10655 ENDdiff --git a/solr/solr-ref-guide/src/coreadmin-api.adoc b/solr/solr-ref-guide/src/coreadmin-api.adocindex fe3ecef983c..5900191f4fb 100644--- a/solr/solr-ref-guide/src/coreadmin-api.adoc+++ b/solr/solr-ref-guide/src/coreadmin-api.adoc@@ -17,11 +17,13 @@// specific language governing permissions and limitations// under the License.-The Core Admin API is primarily used under the covers by the <<collections-api.adoc#,Collections API>> when running a <<solrcloud.adoc#,SolrCloud>> cluster.+The Core Admin API is primarily used under the covers by the <<collections-api.adoc#,Collections API>> when running a <<cluster-types.adoc#solrcloud-mode,SolrCloud>> cluster.-SolrCloud users should not typically use the CoreAdmin API directly, but the API may be useful for users of single-node or leader/follower Solr installations for core maintenance operations.+SolrCloud users should not typically use the CoreAdmin API directly, but the API may be useful for users of user-managed clusters or single-node installations for core maintenance operations.-The CoreAdmin API is implemented by the CoreAdminHandler, which is a special purpose <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,request handler>> that is used to manage Solr cores. Unlike other request handlers, the CoreAdminHandler is not attached to a single core. Instead, there is a single instance of the CoreAdminHandler in each Solr node that manages all the cores running in that node and is accessible at the `/solr/admin/cores` path.+The CoreAdmin API is implemented by the CoreAdminHandler, which is a special purpose <<requesthandlers-searchcomponents.adoc#,request handler>> that is used to manage Solr cores.+Unlike other request handlers, the CoreAdminHandler is not attached to a single core.+Instead, there is a single instance of the CoreAdminHandler in each Solr node that manages all the cores running in that node and is accessible at the `/solr/admin/cores` path.CoreAdmin actions can be executed by via HTTP requests that specify an `action` request parameter, with additional action specific arguments provided as additional parameters.@@ -86,14 +88,17 @@ curl -X GET http://localhost:8983/api/cores?indexInfo=falseThe name of a core, as listed in the "name" attribute of a `<core>` element in `solr.xml`. This parameter is required in v1, and part of the url in the v2 API.`indexInfo`::-If `false`, information about the index will not be returned with a core STATUS request. In Solr implementations with a large number of cores (i.e., more than hundreds), retrieving the index information for each core can take a lot of time and isn't always required. The default is `true`.+If `false`, information about the index will not be returned with a core STATUS request.+In Solr implementations with a large number of cores (i.e., more than hundreds), retrieving the index information for each core can take a lot of time and isn't always required.+The default is `true`.[[coreadmin-create]]== CREATEThe `CREATE` action creates a new core and registers it.-If a Solr core with the given name already exists, it will continue to handle requests while the new core is initializing. When the new core is ready, it will take new requests and the old core will be unloaded.+If a Solr core with the given name already exists, it will continue to handle requests while the new core is initializing.+When the new core is ready, it will take new requests and the old core will be unloaded.`admin/cores?action=CREATE&name=_core-name_&instanceDir=_path/to/dir_&config=solrconfig.xml&dataDir=data`[.dynamic-tabs]@@ -134,56 +139,76 @@ curl -X POST http://localhost:8983/api/cores -H 'Content-Type: application/json'====---Note that this command is the only one of the Core Admin API commands that *does not* support the `core` parameter. Instead, the `name` parameter is required, as shown below.+Note that this command is the only one of the Core Admin API commands that *does not* support the `core` parameter.+Instead, the `name` parameter is required, as shown below.-.CREATE must be able to find a configuration!-[WARNING]-====-Your CREATE call must be able to find a configuration, or it will not succeed.+Note that CREATE must be able to find a configuration or it will not succeed.-When you are running SolrCloud and create a new core for a collection, the configuration will be inherited from the collection. Each collection is linked to a configName, which is stored in ZooKeeper. This satisfies the configuration requirement. There is something to note, though: if you're running SolrCloud, you should *NOT* use the CoreAdmin API at all. Use the <<collections-api.adoc#,Collections API>>.+When you are running SolrCloud and create a new core for a collection, the configuration will be inherited from the collection.+Each collection is linked to a `configName`, which is stored in ZooKeeper.+This satisfies the configuration requirement.+That said, if you're running SolrCloud, you should *NOT* use the CoreAdmin API at all.+Instead, use the <<collections-api.adoc#,Collections API>>.-When you are not running SolrCloud, if you have <<config-sets.adoc#,Configsets>> defined, you can use the `configSet` parameter as documented below. If there are no configsets, then the `instanceDir` specified in the CREATE call must already exist, and it must contain a `conf` directory which in turn must contain `solrconfig.xml`, your schema (usually named either `managed-schema` or `schema.xml`), and any files referenced by those configs.+With a user-managed cluster, if you have <<config-sets.adoc#,Configsets>> defined, you can use the `configSet` parameter as documented below.+If there are no configsets, then the `instanceDir` specified in the CREATE call must already exist, and it must contain a `conf` directory which in turn must contain `solrconfig.xml`, your schema (usually named either `managed-schema` or `schema.xml`), and any files referenced by those configs.-The config and schema filenames can be specified with the `config` and `schema` parameters, but these are expert options. One thing you could do to avoid creating the `conf` directory is use `config` and `schema` parameters that point at absolute paths, but this can lead to confusing configurations unless you fully understand what you are doing.-====+The config and schema filenames can be specified with the `config` and `schema` parameters, but these are expert options.+One thing you could do to avoid creating the `conf` directory is use `config` and `schema` parameters that point at absolute paths, but this can lead to confusing configurations unless you fully understand what you are doing..CREATE and the `core.properties` file[IMPORTANT]====-The `core.properties` file is built as part of the CREATE command. If you create a `core.properties` file yourself in a core directory and then try to use CREATE to add that core to Solr, you will get an error telling you that another core is already defined there. The `core.properties` file must NOT exist before calling the CoreAdmin API with the CREATE command.+The `core.properties` file is built as part of the CREATE command.+If you create a `core.properties` file yourself in a core directory and then try to use CREATE to add that core to Solr, you will get an error telling you that another core is already defined there.+The `core.properties` file must NOT exist before calling the CoreAdmin API with the CREATE command.======= CREATE Core Parameters`name`::-The name of the new core. Same as `name` on the `<core>` element. This parameter is required.+The name of the new core.+Same as `name` on the `<core>` element.+This parameter is required.`instanceDir`::-The directory where files for this core should be stored. Same as `instanceDir` on the `<core>` element. The default is the value specified for the `name` parameter if not supplied. This directory must be inside `SOLR_HOME`, `SOLR_DATA_HOME` or one of the paths specified by system property `solr.allowPaths`.+The directory where files for this core should be stored.+Same as `instanceDir` on the `<core>` element.+The default is the value specified for the `name` parameter if not supplied.+This directory must be inside `SOLR_HOME`, `SOLR_DATA_HOME` or one of the paths specified by system property `solr.allowPaths`.`config`::Name of the config file (i.e., `solrconfig.xml`) relative to `instanceDir`.`schema`::-Name of the schema file to use for the core. Please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use. See <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for details.+Name of the schema file to use for the core.+Please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use.+See <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for details.`dataDir`::-Name of the data directory relative to `instanceDir`. If absolute value is used, it must be inside `SOLR_HOME`, `SOLR_DATA_HOME` or one of the paths specified by system property `solr.allowPaths`.+Name of the data directory relative to `instanceDir`.+If absolute value is used, it must be inside `SOLR_HOME`, `SOLR_DATA_HOME` or one of the paths specified by system property `solr.allowPaths`.`configSet`::-Name of the configset to use for this core. For more information, see the section <<config-sets.adoc#,Configsets>>.+Name of the configset to use for this core.+For more information, see the section <<config-sets.adoc#,Configsets>>.`collection`::-The name of the collection to which this core belongs. The default is the name of the core. `collection._param_=_value_` causes a property of `_param_=_value_` to be set if a new collection is being created. Use `collection.configName=_config-name_` to point to the configuration for a new collection.+The name of the collection to which this core belongs.+The default is the name of the core.+`collection._param_=_value_` causes a property of `_param_=_value_` to be set if a new collection is being created.+Use `collection.configName=_config-name_` to point to the configuration for a new collection.+-WARNING: While it's possible to create a core for a non-existent collection, this approach is not supported and not recommended. Always create a collection using the <<collections-api.adoc#,Collections API>> before creating a core directly for it.+WARNING: While it's possible to create a core for a non-existent collection, this approach is not supported and not recommended.+Always create a collection using the <<collections-api.adoc#,Collections API>> before creating a core directly for it.`shard`::-The shard id this core represents. Normally you want to be auto-assigned a shard id.+The shard ID this core represents.+This should only be required in special circumstances; normally you want to be auto-assigned a shard ID.`property._name_=_value_`::-Sets the core property _name_ to _value_. See the section on defining <<defining-core-properties.adoc#defining-core-properties-files,core.properties file contents>>.+Sets the core property _name_ to _value_.+See the section on defining <<core-discovery.adoc#defining-core-properties-files,core.properties file contents>>.`async`::Request ID to track this action which will be processed asynchronously.@@ -199,7 +224,9 @@ http://localhost:8983/solr/admin/cores?action=CREATE&name=my_core&collection=my_[[coreadmin-reload]]== RELOAD-The RELOAD action loads a new core from the configuration of an existing, registered Solr core. While the new core is initializing, the existing one will continue to handle requests. When the new Solr core is ready, it takes over and the old core is unloaded.+The RELOAD action loads a new core from the configuration of an existing, registered Solr core.+While the new core is initializing, the existing one will continue to handle requests.+When the new Solr core is ready, it takes over and the old core is unloaded.[.dynamic-tabs]--@@ -229,17 +256,20 @@ curl -X POST http://localhost:8983/api/cores/techproducts -H 'Content-Type: appl====---This is useful when you've made changes to a Solr core's configuration on disk, such as adding new field definitions. Calling the RELOAD action lets you apply the new configuration without having to restart Solr.+This is useful when you've made changes to a Solr core's configuration on disk, such as adding new field definitions.+Calling the RELOAD action lets you apply the new configuration without having to restart Solr.[IMPORTANT]====-RELOAD performs "live" reloads of SolrCore, reusing some existing objects. Some configuration options, such as the `dataDir` location and `IndexWriter`-related settings in `solrconfig.xml` can not be changed and made active with a simple RELOAD action.+RELOAD performs "live" reloads of SolrCore, reusing some existing objects.+Some configuration options, such as the `dataDir` location and `IndexWriter`-related settings in `solrconfig.xml` can not be changed and made active with a simple RELOAD action.======= RELOAD Core Parameters`core`::-The name of the core, as listed in the "name" attribute of a `<core>` element in `solr.xml`. This parameter is required in v1, and part of the url in the v2 API.+The name of the core, as listed in the "name" attribute of a `<core>` element in `solr.xml`.+This parameter is required in v1, and part of the url in the v2 API.[[coreadmin-rename]]== RENAME@@ -251,10 +281,13 @@ The `RENAME` action changes the name of a Solr core.=== RENAME Parameters`core`::-The name of the Solr core to be renamed. This parameter is required.+The name of the Solr core to be renamed.+This parameter is required.`other`::-The new name for the Solr core. If the persistent attribute of `<solr>` is `true`, the new name will be written to `solr.xml` as the `name` attribute of the `<core>` attribute. This parameter is required.+The new name for the Solr core.+If the persistent attribute of `<solr>` is `true`, the new name will be written to `solr.xml` as the `name` attribute of the `<core>` attribute.+This parameter is required.`async`::Request ID to track this action which will be processed asynchronously.@@ -263,22 +296,28 @@ Request ID to track this action which will be processed asynchronously.[[coreadmin-swap]]== SWAP-`SWAP` atomically swaps the names used to access two existing Solr cores. This can be used to swap new content into production. The prior core remains available and can be swapped back, if necessary. Each core will be known by the name of the other, after the swap.+`SWAP` atomically swaps the names used to access two existing Solr cores.+This can be used to swap new content into production.+The prior core remains available and can be swapped back, if necessary.+Each core will be known by the name of the other, after the swap.`admin/cores?action=SWAP&core=_core-name_&other=_other-core-name_`[IMPORTANT]====-Do not use `SWAP` with a SolrCloud node. It is not supported and can result in the core being unusable.+Do not use `SWAP` with a SolrCloud node.+It is not supported and can result in the core being unusable.======= SWAP Parameters`core`::-The name of one of the cores to be swapped. This parameter is required.+The name of one of the cores to be swapped.+This parameter is required.`other`::-The name of one of the cores to be swapped. This parameter is required.+The name of one of the cores to be swapped.+This parameter is required.`async`::Request ID to track this action which will be processed asynchronously.@@ -287,11 +326,14 @@ Request ID to track this action which will be processed asynchronously.[[coreadmin-unload]]== UNLOAD-The `UNLOAD` action removes a core from Solr. Active requests will continue to be processed, but no new requests will be sent to the named core. If a core is registered under more than one name, only the given name is removed.+The `UNLOAD` action removes a core from Solr.+Active requests will continue to be processed, but no new requests will be sent to the named core.+If a core is registered under more than one name, only the given name is removed.`admin/cores?action=UNLOAD&core=_core-name_`-The `UNLOAD` action requires a parameter (`core`) identifying the core to be removed. If the persistent attribute of `<solr>` is set to `true`, the `<core>` element with this `name` attribute will be removed from `solr.xml`.+The `UNLOAD` action requires a parameter (`core`) identifying the core to be removed.+If the persistent attribute of `<solr>` is set to `true`, the `<core>` element with this `name` attribute will be removed from `solr.xml`.[IMPORTANT]====@@ -301,16 +343,20 @@ Unloading all cores in a SolrCloud collection causes the removal of that collect=== UNLOAD Parameters`core`::-The name of a core to be removed. This parameter is required.+The name of a core to be removed.+This parameter is required.`deleteIndex`::-If `true`, will remove the index when unloading the core. The default is `false`.+If `true`, will remove the index when unloading the core.+The default is `false`.`deleteDataDir`::-If `true`, removes the `data` directory and all sub-directories. The default is `false`.+If `true`, removes the `data` directory and all sub-directories.+The default is `false`.`deleteInstanceDir`::-If `true`, removes everything related to the core, including the index directory, configuration files and other related files. The default is `false`.+If `true`, removes everything related to the core, including the index directory, configuration files and other related files.+The default is `false`.`async`::Request ID to track this action which will be processed asynchronously.@@ -318,24 +364,33 @@ Request ID to track this action which will be processed asynchronously.[[coreadmin-mergeindexes]]== MERGEINDEXES-The `MERGEINDEXES` action merges one or more indexes to another index. The indexes must have completed commits, and should be locked against writes until the merge is complete or the resulting merged index may become corrupted. The target core index must already exist and have a compatible schema with the one or more indexes that will be merged to it. Another commit on the target core should also be performed after the merge is complete.+The `MERGEINDEXES` action merges one or more indexes to another index.+The indexes must have completed commits, and should be locked against writes until the merge is complete or the resulting merged index may become corrupted.+The target core index must already exist and have a compatible schema with the one or more indexes that will be merged to it.+Another commit on the target core should also be performed after the merge is complete.`admin/cores?action=MERGEINDEXES&core=_new-core-name_&indexDir=_path/to/core1/data/index_&indexDir=_path/to/core2/data/index_`-In this example, we use the `indexDir` parameter to define the index locations of the source cores. The `core` parameter defines the target index. A benefit of this approach is that we can merge any Lucene-based index that may not be associated with a Solr core.+In this example, we use the `indexDir` parameter to define the index locations of the source cores.+The `core` parameter defines the target index.+A benefit of this approach is that we can merge any Lucene-based index that may not be associated with a Solr core.Alternatively, we can instead use a `srcCore` parameter, as in this example:`admin/cores?action=mergeindexes&core=_new-core-name_&srcCore=_core1-name_&srcCore=_core2-name_`-This approach allows us to define cores that may not have an index path that is on the same physical server as the target core. However, we can only use Solr cores as the source indexes. Another benefit of this approach is that we don't have as high a risk for corruption if writes occur in parallel with the source index.+This approach allows us to define cores that may not have an index path that is on the same physical server as the target core.+However, we can only use Solr cores as the source indexes.+Another benefit of this approach is that we don't have as high a risk for corruption if writes occur in parallel with the source index.-We can make this call run asynchronously by specifying the `async` parameter and passing a request-id. This id can then be used to check the status of the already submitted task using the REQUESTSTATUS API.+We can make this call run asynchronously by specifying the `async` parameter and passing a request ID.+This ID can then be used to check the status of the already submitted task using the REQUESTSTATUS API.=== MERGEINDEXES Parameters`core`::-The name of the target core/index. This parameter is required.+The name of the target core/index.+This parameter is required.`indexDir`::Multi-valued, directories that would be merged.@@ -350,26 +405,37 @@ Request ID to track this action which will be processed asynchronously.[[coreadmin-split]]== SPLIT-The `SPLIT` action splits an index into two or more indexes. The index being split can continue to handle requests. The split pieces can be placed into a specified directory on the server's filesystem or it can be merged into running Solr cores.+The `SPLIT` action splits an index into two or more indexes.+The index being split can continue to handle requests.+The split pieces can be placed into a specified directory on the server's filesystem or it can be merged into running Solr cores.The `SPLIT` action supports five parameters, which are described in the table below.=== SPLIT Parameters`core`::-The name of the core to be split. This parameter is required.+The name of the core to be split.+This parameter is required.`path`::-Multi-valued, the directory path in which a piece of the index will be written. Either this parameter or `targetCore` must be specified. If this is specified, the `targetCore` parameter may not be used.+Multi-valued, the directory path in which a piece of the index will be written.+Either this parameter or `targetCore` must be specified.+If this is specified, the `targetCore` parameter may not be used.`targetCore`::-Multi-valued, the target Solr core to which a piece of the index will be merged. Either this parameter or `path` must be specified. If this is specified, the `path` parameter may not be used.+Multi-valued, the target Solr core to which a piece of the index will be merged.+Either this parameter or `path` must be specified.+If this is specified, the `path` parameter may not be used.`ranges`::-A comma-separated list of hash ranges in hexadecimal format. If this parameter is used, `split.key` should not be. See the <<SPLIT Examples>> below for an example of how this parameter can be used.+A comma-separated list of hash ranges in hexadecimal format.+If this parameter is used, `split.key` should not be.+See the <<SPLIT Examples>> below for an example of how this parameter can be used.`split.key`::-The key to be used for splitting the index. If this parameter is used, `ranges` should not be. See the <<SPLIT Examples>> below for an example of how this parameter can be used.+The key to be used for splitting the index.+If this parameter is used, `ranges` should not be.+See the <<SPLIT Examples>> below for an example of how this parameter can be used.`async`::Request ID to track this action which will be processed asynchronously.@@ -397,18 +463,24 @@ The `core` index will be split into two pieces and written into the two director[source,bash]http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&targetCore=core1&split.key=A!-Here all documents having the same route key as the `split.key` i.e., 'A!' will be split from the `core` index and written to the `targetCore`.+Here all documents having the same route key as the `split.key` i.e., `A!` will be split from the `core` index and written to the `targetCore`.*Usage with ranges parameter*:[source,bash]http://localhost:8983/solr/admin/cores?action=SPLIT&core=core0&targetCore=core1&targetCore=core2&targetCore=core3&ranges=0-1f4,1f5-3e8,3e9-5dc-This example uses the `ranges` parameter with hash ranges 0-500, 501-1000 and 1001-1500 specified in hexadecimal. Here the index will be split into three pieces with each targetCore receiving documents matching the hash ranges specified i.e., core1 will get documents with hash range 0-500, core2 will receive documents with hash range 501-1000 and finally, core3 will receive documents with hash range 1001-1500. At least one hash range must be specified. Please note that using a single hash range equal to a route key's hash range is NOT equivalent to using the `split.key` parameter because multiple route keys can hash to the same range.+This example uses the `ranges` parameter with hash ranges 0-500, 501-1000 and 1001-1500 specified in hexadecimal.+Here the index will be split into three pieces with each targetCore receiving documents matching the hash ranges specified i.e., core1 will get documents with hash range 0-500, core2 will receive documents with hash range 501-1000 and finally, core3 will receive documents with hash range 1001-1500.+At least one hash range must be specified.+Please note that using a single hash range equal to a route key's hash range is NOT equivalent to using the `split.key` parameter because multiple route keys can hash to the same range.-The `targetCore` must already exist and must have a compatible schema with the `core` index. A commit is automatically called on the `core` index before it is split.+The `targetCore` must already exist and must have a compatible schema with the `core` index.+A commit is automatically called on the `core` index before it is split.-This command is used as part of the <<shard-management.adoc#splitshard,SPLITSHARD>> command but it can be used for non-cloud Solr cores as well. When used against a non-cloud core without `split.key` parameter, this action will split the source index and distribute its documents alternately so that each split piece contains an equal number of documents. If the `split.key` parameter is specified then only documents having the same route key will be split from the source index.+This command is used as part of SolrCloud's <<shard-management.adoc#splitshard,SPLITSHARD>> command but it can be used for cores in user-managed clusters as well.+When used against a core in a user-managed cluster without `split.key` parameter, this action will split the source index and distribute its documents alternately so that each split piece contains an equal number of documents.+If the `split.key` parameter is specified then only documents having the same route key will be split from the source index.[[coreadmin-requeststatus]]== REQUESTSTATUS@@ -422,7 +494,8 @@ Request the status of an already submitted asynchronous CoreAdmin API call.The REQUESTSTATUS command has only one parameter.`requestid`::-The user defined request-id for the asynchronous request. This parameter is required.+The user defined request-id for the asynchronous request.+This parameter is required.The call below will return the status of an already submitted asynchronous CoreAdmin call.@@ -432,14 +505,16 @@ http://localhost:8983/solr/admin/cores?action=REQUESTSTATUS&requestid=1[[coreadmin-requestrecovery]]== REQUESTRECOVERY-The `REQUESTRECOVERY` action manually asks a core to recover by synching with the leader. This should be considered an "expert" level command and should be used in situations where the node (SorlCloud replica) is unable to become active automatically.+The `REQUESTRECOVERY` action manually asks a core to recover by synching with the leader.+This should be considered an "expert" level command and should be used in situations where the node (SorlCloud replica) is unable to become active automatically.`admin/cores?action=REQUESTRECOVERY&core=_core-name_`=== REQUESTRECOVERY Parameters`core`::-The name of the core to re-sync. This parameter is required.+The name of the core to re-sync.+This parameter is required.=== REQUESTRECOVERY Examplesdiff --git a/solr/solr-ref-guide/src/css/decoration.css b/solr/solr-ref-guide/src/css/decoration.cssindex ff3a3e2f4cc..b9bc1947e2d 100644--- a/solr/solr-ref-guide/src/css/decoration.css+++ b/solr/solr-ref-guide/src/css/decoration.css@@ -21,11 +21,21 @@border-color: #CDCECF;}-/* used for Download Solr button on docs home page */-.btn-home {- color: #ffffff;- background-color: #F35B38;- border-color: #E6E7E8;+/* used for query parser labels in query-parameters.adoc */++.badge-lucene {+ background-color: #6F8061;+ color: white;+}++.badge-dismax {+ background-color: #305CB3;+ color: white;+}++.badge-edismax {+ background-color: #FF833D;+ color: white;}input@@ -240,15 +250,39 @@ hr {}/** === END HORIZONTAL RULES === **/-/** === ANCHORJS === **/-/* This is the permalink icon after section headings */--.anchorjs-link:hover {- color: #216f9b;-}--*:hover > .anchorjs-link {- transition: color .25s linear;- text-decoration: none;-}-/** === END ANCHORJS === **/+/** === SECTION ANCHOR LINKS === **/++h1 .anchor::before,+h2 .anchor::before,+h3 .anchor::before,+h4 .anchor::before,+h5 .anchor::before,+h6 .anchor::before {+ content: "\00a7";+}++h1 .anchor,+h2 .anchor,+h3 .anchor,+h4 .anchor,+h5 .anchor,+h6 .anchor {+ position: absolute;+ text-decoration: none;+ width: 1.75ex;+ margin-left: -1ex;+ visibility: hidden;+ font-size: .8em;+ font-weight: 400;+ padding-top: .15em;+}++h1:hover .anchor,+h2:hover .anchor,+h3:hover .anchor,+h4:hover .anchor,+h5:hover .anchor,+h6:hover .anchor {+ visibility: visible;+}+/** === SECTION ANCHOR LINKS === **/diff --git a/solr/solr-ref-guide/src/css/navs.css b/solr/solr-ref-guide/src/css/navs.cssindex 204d52a1a7d..13262451c55 100644--- a/solr/solr-ref-guide/src/css/navs.css+++ b/solr/solr-ref-guide/src/css/navs.css@@ -175,10 +175,10 @@ ul.nav li ul {}.nav li > a > span:after {- content: '\25be';+ content: '\002b';}.nav li.active > a > span:after {- content: '\25b4';+ content: '\2212';}diff --git a/solr/solr-ref-guide/src/css/ref-guide.css b/solr/solr-ref-guide/src/css/ref-guide.cssindex 9037823b4aa..7d7530c21db 100644--- a/solr/solr-ref-guide/src/css/ref-guide.css+++ b/solr/solr-ref-guide/src/css/ref-guide.css@@ -151,16 +151,6 @@ imargin: 50px 10px 20px 10px;}-/* Special rules for homepage (index.html) so rows are all the same height */-.row.match-my-cols {- display: table;-}--.row.match-my-cols [class*="col-"] {- float: none;- display: table-cell;- vertical-align: top;-}/* General text paragraphs */@@ -314,6 +304,14 @@ dt[id]{margin-top: -40px}+/* used on scaling-solr for discrete headings in the sidebar block */+h3[id].discrete,+h4[id].discrete {+ padding-top: 20px;+ padding-bottom: 10px;+ margin-top: -20px;+}+.sidebarblock > .title small,h1 small,h2 small,@@ -720,7 +718,11 @@ ol.lowergreekdl dd{- margin-left: 1.125em;+ margin-left: 20px;+}++dl dd dl {+ margin-left: 30px;}dl dd:last-child,@@ -976,16 +978,33 @@ table thead tr thtable tr td,table tr th{- padding: .5625em .625em;+ padding: 9px 10px;color: rgba(0,0,0,.8);font-size: inherit;}-table tr.alt,+/** Used on some tables to tighten up space between cells+ Add [.lowpadding] as a role to the table to use this **/+.tableblock.lowpadding table tr td,+.tableblock.lowpadding table tr th+{+ padding: 2px 3px;+ color: rgba(0,0,0,.8);+ font-size: inherit;+}++/** table tr.alt,table tr.even,table tr:nth-of-type(even){background: #f8f8f7;+} **/++table.stripes-all tr,+table.stripes-odd tr:nth-of-type(odd),+table.stripes-even tr:nth-of-type(even),+table.stripes-hover tr:hover {+ background:#f8f8f7}table tbody tr td,@@ -1050,7 +1069,7 @@ table.tableblock th > p:last-childmargin-bottom: 0;}-table.spread+.stretch table.tableblock{width: 100%;}@@ -1165,6 +1184,17 @@ table thead thfont-weight: bold;}+/* Special formatting for tables on index.adoc home page */+.home-table table th,+.home-table table th a {+ color: #D9411E;+ font-size: 1.125em;+}++.home-table table thead {+ background-color: #fff;+}+tbody tr th{display: table-cell;diff --git a/solr/solr-ref-guide/src/working-with-currencies-and-exchange-rates.adoc b/solr/solr-ref-guide/src/currencies-exchange-rates.adocsimilarity index 72%rename from solr/solr-ref-guide/src/working-with-currencies-and-exchange-rates.adocrename to solr/solr-ref-guide/src/currencies-exchange-rates.adocindex ba7c39a4cba..20eaeed67ee 100644--- a/solr/solr-ref-guide/src/working-with-currencies-and-exchange-rates.adoc+++ b/solr/solr-ref-guide/src/currencies-exchange-rates.adoc@@ -1,4 +1,4 @@-= Working with Currencies and Exchange Rates+= Currencies and Exchange Rates// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,8 @@// specific language governing permissions and limitations// under the License.-The `currency` FieldType provides support for monetary values to Solr/Lucene with query-time currency conversion and exchange rates. The following features are supported:+The `currency` FieldType provides support for monetary values to Solr/Lucene with query-time currency conversion and exchange rates.+The following features are supported:* Point queries* Range queries@@ -34,7 +35,8 @@ The `currency` FieldType provides support for monetary values to Solr/Lucene witCurrencyField has been deprecated in favor of CurrencyFieldType; all configuration examples below use CurrencyFieldType.====-The `currency` field type is defined in `schema.xml`. This is the default configuration of this type.+The `currency` field type is defined in `schema.xml`.+This is the default configuration of this type.[source,xml]----@@ -43,7 +45,10 @@ The `currency` field type is defined in `schema.xml`. This is the default configdefaultCurrency="USD" currencyConfig="currency.xml" />-----In this example, we have defined the name and class of the field type, and defined the `defaultCurrency` as "USD", for U.S. Dollars. We have also defined a `currencyConfig` to use a file called "currency.xml". This is a file of exchange rates between our default currency to other currencies. There is an alternate implementation that would allow regular downloading of currency data. See <<Exchange Rates>> below for more.+In this example, we have defined the name and class of the field type, and defined the `defaultCurrency` as "USD", for U.S. Dollars.+We have also defined a `currencyConfig` to use a file called "currency.xml". This is a file of exchange rates between our default currency to other currencies.+There is an alternate implementation that would allow regular downloading of currency data.+See <<Exchange Rates>> below for more.Many of the example schemas that ship with Solr include a <<dynamic-fields.adoc#,dynamic field>> that uses this type, such as this example:@@ -54,7 +59,9 @@ Many of the example schemas that ship with Solr include a <<dynamic-fields.adoc#This dynamic field would match any field that ends in `_c` and make it a currency typed field.-At indexing time, money fields can be indexed in a native currency. For example, if a product on an e-commerce site is listed in Euros, indexing the price field as "1000,EUR" will index it appropriately. The price should be separated from the currency by a comma, and the price must be encoded with a floating point value (a decimal point).+At indexing time, money fields can be indexed in a native currency.+For example, if a product on an e-commerce site is listed in Euros, indexing the price field as "1000,EUR" will index it appropriately.+The price should be separated from the currency by a comma, and the price must be encoded with a floating point value (a decimal point).During query processing, range and point queries are both supported.@@ -69,23 +76,28 @@ You must specify parameters `amountLongSuffix` and `codeStrSuffix`, correspondindefaultCurrency="USD" currencyConfig="currency.xml" />-----In the above example, the raw amount field will use the `"*_l_ns"` dynamic field, which must exist in the schema and use a long field type, i.e., one that extends `LongValueFieldType`. The currency code field will use the `"*_s_ns"` dynamic field, which must exist in the schema and use a string field type, i.e., one that is or extends `StrField`.+In the above example, the raw amount field will use the `"*_l_ns"` dynamic field, which must exist in the schema and use a long field type, i.e., one that extends `LongValueFieldType`.+The currency code field will use the `"*_s_ns"` dynamic field, which must exist in the schema and use a string field type, i.e., one that is or extends `StrField`..Atomic Updates won't work if dynamic sub-fields are stored[NOTE]====-As noted on <<updating-parts-of-documents.adoc#field-storage,Updating Parts of Documents>>, stored dynamic sub-fields will cause indexing to fail when you use Atomic Updates. To avoid this problem, specify `stored="false"` on those dynamic fields.+As noted in <<partial-document-updates.adoc#field-storage,Atomic Update Field Storage>>, stored dynamic sub-fields will cause indexing to fail when you use Atomic Updates.+To avoid this problem, specify `stored="false"` on those dynamic fields.====== Exchange Rates-You configure exchange rates by specifying a provider. Natively, two provider types are supported: `FileExchangeRateProvider` or `OpenExchangeRatesOrgProvider`.+You configure exchange rates by specifying a provider.+Natively, two provider types are supported: `FileExchangeRateProvider` or `OpenExchangeRatesOrgProvider`.=== FileExchangeRateProvider-This provider requires you to provide a file of exchange rates. It is the default, meaning that to use this provider you only need to specify the file path and name as a value for `currencyConfig` in the definition for this type.+This provider requires you to provide a file of exchange rates.+It is the default, meaning that to use this provider you only need to specify the file path and name as a value for `currencyConfig` in the definition for this type.-There is a sample `currency.xml` file included with Solr, found in the same directory as the `schema.xml` file. Here is a small snippet from this file:+There is a sample `currency.xml` file included with Solr, found in the same directory as the `schema.xml` file.+Here is a small snippet from this file:[source,xml]----@@ -110,9 +122,11 @@ There is a sample `currency.xml` file included with Solr, found in the same dire=== OpenExchangeRatesOrgProvider-You can configure Solr to download exchange rates from http://www.OpenExchangeRates.Org[OpenExchangeRates.Org], with updates rates between USD and 170 currencies hourly. These rates are symmetrical only.+You can configure Solr to download exchange rates from http://www.OpenExchangeRates.Org[OpenExchangeRates.Org], with updates rates between USD and 170 currencies hourly.+These rates are symmetrical only.-In this case, you need to specify the `providerClass` in the definitions for the field type and sign up for an API key. Here is an example:+In this case, you need to specify the `providerClass` in the definitions for the field type and sign up for an API key.+Here is an example:[source,xml]----@@ -123,4 +137,5 @@ In this case, you need to specify the `providerClass` in the definitions for theratesFileLocation="http://www.openexchangerates.org/api/latest.json?app_id=yourPersonalAppIdKey"/>-----The `refreshInterval` is minutes, so the above example will download the newest rates every 60 minutes. The refresh interval may be increased, but not decreased.+The `refreshInterval` is minutes, so the above example will download the newest rates every 60 minutes.+The refresh interval may be increased, but not decreased.diff --git a/solr/solr-ref-guide/src/working-with-dates.adoc b/solr/solr-ref-guide/src/date-formatting-math.adocsimilarity index 70%rename from solr/solr-ref-guide/src/working-with-dates.adocrename to solr/solr-ref-guide/src/date-formatting-math.adocindex 52d914b0df4..8bd66c25a3f 100644--- a/solr/solr-ref-guide/src/working-with-dates.adoc+++ b/solr/solr-ref-guide/src/date-formatting-math.adoc@@ -1,4 +1,4 @@-= Working with Dates+= Date Formatting and Date Math// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,7 +18,8 @@== Date Formatting-Solr's date fields (`DatePointField`, `DateRangeField` and the deprecated `TrieDateField`) represent "dates" as a point in time with millisecond precision. The format used is a restricted form of the canonical representation of dateTime in the http://www.w3.org/TR/xmlschema-2/#dateTime[XML Schema specification] – a restricted subset of https://en.wikipedia.org/wiki/ISO_8601[ISO-8601]. For those familiar with Java date handling, Solr uses {java-javadocs}java/time/format/DateTimeFormatter.html#ISO_INSTANT[DateTimeFormatter.ISO_INSTANT] for formatting, and parsing too with "leniency".+Solr's date fields (`DatePointField`, `DateRangeField` and the deprecated `TrieDateField`) represent "dates" as a point in time with millisecond precision.+The format used is a restricted form of the canonical representation of dateTime in the http://www.w3.org/TR/xmlschema-2/#dateTime[XML Schema specification] – a restricted subset of https://en.wikipedia.org/wiki/ISO_8601[ISO-8601]. For those familiar with Java date handling, Solr uses {java-javadocs}java/time/format/DateTimeFormatter.html#ISO_INSTANT[DateTimeFormatter.ISO_INSTANT] for formatting, and parsing too with "leniency".`YYYY-MM-DDThh:mm:ssZ`@@ -34,7 +35,8 @@ Note that no time zone can be specified; the String representations of dates is`1972-05-20T17:33:18Z`-You can optionally include fractional seconds if you wish, although any precision beyond milliseconds will be ignored. Here are example values with sub-seconds:+You can optionally include fractional seconds if you wish, although any precision beyond milliseconds will be ignored.+Here are example values with sub-seconds:* `1972-05-20T17:33:18.772Z`* `1972-05-20T17:33:18.77Z`@@ -45,7 +47,8 @@ There must be a leading `'-'` for dates prior to year 0000, and Solr will format.Query escaping may be required[WARNING]====-As you can see, the date format includes colon characters separating the hours, minutes, and seconds. Because the colon is a special character to Solr's most common query parsers, escaping is sometimes required, depending on exactly what you are trying to do.+As you can see, the date format includes colon characters separating the hours, minutes, and seconds.+Because the colon is a special character to Solr's most common query parsers, escaping is sometimes required, depending on exactly what you are trying to do.This is normally an invalid query: `datefield:1972-05-20T17:33:18.772Z`@@ -57,17 +60,22 @@ These are valid queries: +=== Date Range Formatting-Solr's `DateRangeField` supports the same point in time date syntax described above (with _date math_ described below) and more to express date ranges. One class of examples is truncated dates, which represent the entire date span to the precision indicated. The other class uses the range syntax (`[ TO ]`). Here are some examples:+Solr's `DateRangeField` supports the same point in time date syntax described above (with _date math_ described below) and more to express date ranges.+One class of examples is truncated dates, which represent the entire date span to the precision indicated.+The other class uses the range syntax (`[ TO ]`). Here are some examples:* `2000-11` – The entire month of November, 2000.* `1605-11-05` – The Fifth of November.* `2000-11-05T13` – Likewise but for an hour of the day (1300 to before 1400, i.e., 1pm to 2pm).-* `-0009` – The year 10 BC. A 0 in the year position is 0 AD, and is also considered 1 BC.+* `-0009` – The year 10 BC.+A 0 in the year position is 0 AD, and is also considered 1 BC.* `[2000-11-01 TO 2014-12-01]` – The specified date range at a day resolution.* `[2014 TO 2014-12-01]` – From the start of 2014 till the end of the first day of December.* `[* TO 2014-12-01]` – From the earliest representable time thru till the end of the day on 2014-12-01.-Limitations: The range syntax doesn't support embedded date math. If you specify a date instance supported by DatePointField with date math truncating it, like `NOW/DAY`, you still get the first millisecond of that day, not the entire day's range. Exclusive ranges (using `{` & `}`) work in _queries_ but not for _indexing_ ranges.+Limitations: The range syntax doesn't support embedded date math.+If you specify a date instance supported by DatePointField with date math truncating it, like `NOW/DAY`, you still get the first millisecond of that day, not the entire day's range.+Exclusive ranges (using `{` & `}`) work in _queries_ but not for _indexing_ ranges.== Date Math@@ -75,7 +83,8 @@ Solr's date field types also supports _date math_ expressions, which makes it ea=== Date Math Syntax-Date math expressions consist either adding some quantity of time in a specified unit, or rounding the current time by a specified unit. expressions can be chained and are evaluated left to right.+Date math expressions consist either adding some quantity of time in a specified unit, or rounding the current time by a specified unit.+Expressions can be chained and are evaluated left to right.For example: this represents a point in time two months from now:@@ -85,7 +94,8 @@ This is one day ago:`NOW-1DAY`-A slash is used to indicate rounding. This represents the beginning of the current hour:+A slash is used to indicate rounding.+This represents the beginning of the current hour:`NOW/HOUR`@@ -101,7 +111,8 @@ Note that while date math is most commonly used relative to `NOW` it can be appl==== NOW-The `NOW` parameter is used internally by Solr to ensure consistent date math expression parsing across multiple nodes in a distributed request. But it can be specified to instruct Solr to use an arbitrary moment in time (past or future) to override for all situations where the the special value of "```NOW```" would impact date math expressions.+The `NOW` parameter is used internally by Solr to ensure consistent date math expression parsing across multiple nodes in a distributed request.+But it can be specified to instruct Solr to use an arbitrary moment in time (past or future) to override for all situations where the the special value of "```NOW```" would impact date math expressions.It must be specified as a (long valued) milliseconds since epoch.@@ -153,9 +164,14 @@ http://localhost:8983/solr/my_collection/select?q=*:*&facet.range=my_date_field&== More DateRangeField Details-`DateRangeField` is almost a drop-in replacement for places where `DatePointField` is used. The only difference is that Solr's XML or SolrJ response formats will expose the stored data as a String instead of a Date. The underlying index data for this field will be a bit larger. Queries that align to units of time a second on up should be faster than TrieDateField, especially if it's in UTC.+`DateRangeField` is almost a drop-in replacement for places where `DatePointField` is used.+The only difference is that Solr's XML or SolrJ response formats will expose the stored data as a String instead of a Date.+The underlying index data for this field will be a bit larger.+Queries that align to units of time a second on up should be faster than TrieDateField, especially if it's in UTC.-The main point of `DateRangeField`, as its name suggests, is to allow indexing date ranges. To do that, simply supply strings in the format shown above. It also supports specifying 3 different relational predicates between the indexed data, and the query range:+The main point of `DateRangeField`, as its name suggests, is to allow indexing date ranges.+To do that, simply supply strings in the format shown above.+It also supports specifying 3 different relational predicates between the indexed data, and the query range:* `Intersects` (default)* `Contains`@@ -168,6 +184,7 @@ You can specify the predicate by querying using the `op` local-params parameterfq={!field f=dateRange op=Contains}[2013 TO 2018]-----Unlike most local parameters, `op` is actually _not_ defined by any query parser (`field`), it is defined by the field type, in this case `DateRangeField`. In the above example, it would find documents with indexed ranges that _contain_ (or equals) the range 2013 thru 2018. Multi-valued overlapping indexed ranges in a document are effectively coalesced.+Unlike most local params, `op` is actually _not_ defined by any query parser (`field`), it is defined by the field type, in this case `DateRangeField`.+In the above example, it would find documents with indexed ranges that _contain_ (or equals) the range 2013 thru 2018. Multi-valued overlapping indexed ranges in a document are effectively coalesced.For a DateRangeField example use-case, see https://cwiki.apache.org/confluence/display/solr/DateRangeField[see Solr's community wiki].diff --git a/solr/solr-ref-guide/src/de-duplication.adoc b/solr/solr-ref-guide/src/de-duplication.adocindex 133cb78a539..893c0c712a9 100644--- a/solr/solr-ref-guide/src/de-duplication.adoc+++ b/solr/solr-ref-guide/src/de-duplication.adoc@@ -18,11 +18,15 @@If duplicate, or near-duplicate documents are a concern in your index, de-duplication may be worth implementing.-Preventing duplicate or near duplicate documents from entering an index or tagging documents with a signature/fingerprint for duplicate field collapsing can be efficiently achieved with a low collision or fuzzy hash algorithm. Solr natively supports de-duplication techniques of this type via the `Signature` class and allows for the easy addition of new hash/signature implementations. A Signature can be implemented in a few ways:+Preventing duplicate or near duplicate documents from entering an index or tagging documents with a signature/fingerprint for duplicate field collapsing can be efficiently achieved with a low collision or fuzzy hash algorithm.+Solr natively supports de-duplication techniques of this type via the `Signature` class and allows for the easy addition of new hash/signature implementations.+A Signature can be implemented in a few ways:* MD5Signature: 128-bit hash used for exact duplicate detection.-* Lookup3Signature: 64-bit hash used for exact duplicate detection. This is much faster than MD5 and smaller to index.-* https://cwiki.apache.org/confluence/display/solr/TextProfileSignature[TextProfileSignature]: Fuzzy hashing implementation from Apache Nutch for near duplicate detection. It's tunable but works best on longer text.+* Lookup3Signature: 64-bit hash used for exact duplicate detection.+This is much faster than MD5 and smaller to index.+* https://cwiki.apache.org/confluence/display/solr/TextProfileSignature[TextProfileSignature]: Fuzzy hashing implementation from Apache Nutch for near duplicate detection.+It's tunable but works best on longer text.Other, more sophisticated algorithms for fuzzy/near hashing can be added later.@@ -30,7 +34,8 @@ Other, more sophisticated algorithms for fuzzy/near hashing can be added later.====Adding in the de-duplication process will change the `allowDups` setting so that it applies to an update term (with `signatureField` in this case) rather than the unique field Term.-Of course the `signatureField` could be the unique field, but generally you want the unique field to be unique. When a document is added, a signature will automatically be generated and attached to the document in the specified `signatureField`.+Of course the `signatureField` could be the unique field, but generally you want the unique field to be unique.+When a document is added, a signature will automatically be generated and attached to the document in the specified `signatureField`.====== Configuration Options@@ -57,25 +62,31 @@ The `SignatureUpdateProcessorFactory` has to be registered in `solrconfig.xml` aThe `SignatureUpdateProcessorFactory` takes several properties:`signatureClass`::-A Signature implementation for generating a signature hash. The default is `org.apache.solr.update.processor.Lookup3Signature`.+A Signature implementation for generating a signature hash.+The default is `org.apache.solr.update.processor.Lookup3Signature`.+-The full classpath of the implementation must be specified. The available options are described above, the associated classpaths to use are:+The full classpath of the implementation must be specified.+The available options are described above, the associated classpaths to use are:* `org.apache.solr.update.processor.Lookup3Signature`* `org.apache.solr.update.processor.MD5Signature`* `org.apache.solr.update.process.TextProfileSignature``fields`::-The fields to use to generate the signature hash in a comma separated list. By default, all fields on the document will be used.+The fields to use to generate the signature hash in a comma separated list.+By default, all fields on the document will be used.`signatureField`::-The name of the field used to hold the fingerprint/signature. The field should be defined in `schema.xml`. The default is `signatureField`.+The name of the field used to hold the fingerprint/signature.+The field should be defined in `schema.xml`. The default is `signatureField`.`enabled`::-Set to *false* to disable de-duplication processing. The default is *true*.+Set to *false* to disable de-duplication processing.+The default is *true*.`overwriteDupes`::-If *true*, the default, when a document exists that already matches this signature, it will be overwritten. If you are using `overwriteDupes=true` the `signatureField` must be `indexed="true"` in your Schema.+If *true*, the default, when a document exists that already matches this signature, it will be overwritten.+If you are using `overwriteDupes=true` the `signatureField` must be `indexed="true"` in your Schema..Using `SignatureUpdateProcessorFactory` in SolrCloud[WARNING]@@ -84,7 +95,8 @@ If *true*, the default, when a document exists that already matches this signatuThere are 2 important things to keep in mind when using `SignatureUpdateProcessorFactory` with SolrCloud:-. The `overwriteDupes=true` setting does not work _except_ in the special case of using the uniqueKey field as the `signatureField`. Attempting De-duplication on any other `signatureField` will not work correctly because of how updates are forwarded to replicas+. The `overwriteDupes=true` setting does not work _except_ in the special case of using the uniqueKey field as the `signatureField`.+Attempting De-duplication on any other `signatureField` will not work correctly because of how updates are forwarded to replicas. When using the uniqueKey field as the `signatureField`, `SignatureUpdateProcessorFactory` must be run prior to the `<<update-request-processors.adoc#update-processors-in-solrcloud,DistributedUpdateProcessor>>` to ensure that documents can be routed to the correct shard leader based on the (generated) uniqueKey field.(Using any other `signatureField` with `overwriteDupes=false` -- to generate a Signature for each document with out De-duplication -- has no limitations.)diff --git a/solr/solr-ref-guide/src/defining-core-properties.adoc b/solr/solr-ref-guide/src/defining-core-properties.adocdeleted file mode 100644index 9e9a611aff2..00000000000--- a/solr/solr-ref-guide/src/defining-core-properties.adoc+++ /dev/null@@ -1,95 +0,0 @@-= Defining core.properties-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Core discovery means that creating a core is as simple as a `core.properties` file located on disk.--The `core.properties` file is a simple Java Properties file where each line is just a key=value pair, e.g., `name=core1`. Notice that no quotes are required.--A minimal `core.properties` file looks like the example below. However, it can also be empty, see information on placement of `core.properties` below.--[source,bash]------name=my_core_name-------== Placement of core.properties--Solr cores are configured by placing a file named `core.properties` in a sub-directory under `solr.home`. There are no a-priori limits to the depth of the tree, nor are there limits to the number of cores that can be defined. Cores may be anywhere in the tree with the exception that cores may _not_ be defined under an existing core. That is, the following is not allowed:--[source,text]------./cores/core1/core.properties-./cores/core1/coremore/core5/core.properties-------In this example, the enumeration will stop at "core1".--The following is legal:--[source,text]------./cores/somecores/core1/core.properties-./cores/somecores/core2/core.properties-./cores/othercores/core3/core.properties-./cores/extracores/deepertree/core4/core.properties-------It is possible to segment Solr into multiple cores, each with its own configuration and indices. Cores may be dedicated to a single application or to very different ones, but all are administered through a common administration interface. You can create new Solr cores on the fly, shutdown cores, even replace one running core with another, all without ever stopping or restarting Solr.--Your `core.properties` file can be empty if necessary. Suppose `core.properties` is located in `./cores/core1` (relative to `solr_home`) but is empty. In this case, the core name is assumed to be "core1". The instanceDir will be the folder containing `core.properties` (i.e., `./cores/core1`). The dataDir will be `../cores/core1/data`, etc.--[NOTE]-====-You can run Solr without configuring any cores.-====--== Defining core.properties Files--The minimal `core.properties` file is an empty file, in which case all of the properties are defaulted appropriately.--Java properties files allow the hash (`#`) or bang (`!`) characters to specify comment-to-end-of-line.--The following properties are available:--`name`:: The name of the SolrCore. You'll use this name to reference the SolrCore when running commands with the CoreAdminHandler.--`config`:: The configuration file name for a given core. The default is `solrconfig.xml`.--`schema`:: The schema file name for a given core. The default is `schema.xml` but please note that if you are using a "managed schema" (the default behavior) then any value for this property which does not match the effective `managedSchemaResourceName` will be read once, backed up, and converted for managed schema use. See <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for more details.--`dataDir`:: The core's data directory (where indexes are stored) as either an absolute pathname, or a path relative to the value of `instanceDir`. This is `data` by default.--`configSet`:: The name of a defined configset, if desired, to use to configure the core (see the section <<config-sets.adoc#,Configsets>> for more details).--`properties`:: The name of the properties file for this core. The value can be an absolute pathname or a path relative to the value of `instanceDir`.--`transient`:: If *true*, the core can be unloaded if Solr reaches the `transientCacheSize`. The default if not specified is *false*. Cores are unloaded in order of least recently used first. _Setting this to *true* is not recommended in SolrCloud mode._--`loadOnStartup`:: If *true*, the default if it is not specified, the core will loaded when Solr starts. _Setting this to *false* is not recommended in SolrCloud mode._--`coreNodeName`:: Used only in SolrCloud, this is a unique identifier for the node hosting this replica. By default a `coreNodeName` is generated automatically, but setting this attribute explicitly allows you to manually assign a new core to replace an existing replica. For example, this can be useful when replacing a machine that has had a hardware failure by restoring from backups on a new machine with a new hostname or port.--`ulogDir`:: The absolute or relative directory for the update log for this core (SolrCloud).--`shard`:: The shard to assign this core to (SolrCloud).--`collection`:: The name of the collection this core is part of (SolrCloud).--`roles`:: Future parameter for SolrCloud or a way for users to mark nodes for their own use.--Additional user-defined properties may be specified for use as variables. For more information on how to define local properties, see the section <<configuring-solrconfig-xml.adoc#substituting-properties-in-solr-config-files,Substituting Properties in Solr Config Files>>.diff --git a/solr/solr-ref-guide/src/deployment-and-operations.adoc b/solr/solr-ref-guide/src/deployment-and-operations.adocdeleted file mode 100644index df96c0554e8..00000000000--- a/solr/solr-ref-guide/src/deployment-and-operations.adoc+++ /dev/null@@ -1,40 +0,0 @@-= Deployment and Operations-:page-children: solr-control-script-reference, solr-configuration-files, taking-solr-to-production, making-and-restoring-backups, running-solr-in-docker, running-solr-on-hdfs, aws-solrcloud-tutorial, upgrading-a-solr-cluster, solr-upgrade-notes-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--An important aspect of Solr is that all operations and deployment can be done online, with minimal or no impact to running applications. This includes minor upgrades and provisioning and removing nodes, backing up and restoring indexes and editing configurations--Common administrative tasks include:--<<solr-control-script-reference.adoc#,Solr Control Script Reference>>: This section provides information about all of the options available to the `bin/solr` / `bin\solr.cmd` scripts, which can start and stop Solr, configure authentication, and create or remove collections and cores.--<<solr-configuration-files.adoc#,Solr Configuration Files>>: Overview of the installation layout and major configuration files.--<<taking-solr-to-production.adoc#,Taking Solr to Production>>: Detailed steps to help you install Solr as a service and take your application to production.--<<making-and-restoring-backups.adoc#,Making and Restoring Backups>>: Describes backup strategies for your Solr indexes.--<<running-solr-in-docker.adoc#,Running Solr in Docker>>: How to use the official Solr Docker image.--<<running-solr-on-hdfs.adoc#,Running Solr on HDFS>>: How to use HDFS to store your Solr indexes and transaction logs.--<<aws-solrcloud-tutorial.adoc#,SolrCloud on AWS EC2>>: A tutorial on deploying Solr in Amazon Web Services (AWS) using EC2 instances.--<<upgrading-a-solr-cluster.adoc#,Upgrading a Solr Cluster>>: Information for upgrading a production SolrCloud cluster.--<<solr-upgrade-notes.adoc#,Solr Upgrade Notes>>: Information about changes made in Solr releases.diff --git a/solr/solr-ref-guide/src/deployment-guide.adoc b/solr/solr-ref-guide/src/deployment-guide.adocnew file mode 100644index 00000000000..3fcfcf8e6c6--- /dev/null+++ b/solr/solr-ref-guide/src/deployment-guide.adoc@@ -0,0 +1,100 @@+= Deployment Guide+:page-children: solr-control-script-reference, \+ installation-deployment, \+ scaling-solr, \+ monitoring-solr, \+ securing-solr, \+ client-apis+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+The Deployment Guide covers installation, upgrades, deployments, monitoring, and client integrations.++****+[discrete]+=== Solr CLI++<<solr-control-script-reference.adoc#,*Solr Control Script Reference*>>: The options available to the `bin/solr` or `bin\solr.cmd` scripts.+****++****+[discrete]+=== Installation & Deployment++<<installation-deployment.adoc#,*Installation and Deployment*>>++// This pulls the sub-section list from the child page to reduce errors+include::installation-deployment.adoc[tag=install-sections]+****++****+[discrete]+=== Scaling Solr++<<scaling-solr.adoc#,*Scaling Solr*>>++// This pulls the sub-section list from the child page to reduce errors+include::scaling-solr.adoc[tag=scaling-sections]+****++****+[discrete]+=== Monitoring Solr++<<monitoring-solr.adoc#,*Monitoring Solr*>>++// This pulls the sub-section list from the child page to reduce errors+include::monitoring-solr.adoc[tag=monitoring-sections]+****++****+[discrete]+=== Securing Solr++<<securing-solr.adoc#,*Securing Solr*>>++Authentication Plugins:++// This pulls the sub-section list from the child page to reduce errors+include::securing-solr.adoc[tag=list-of-authentication-plugins]++Authorization Plugins:++// This pulls the sub-section list from the child page to reduce errors+include::securing-solr.adoc[tag=list-of-authorization-plugins]++Audit Logging and SSL:++[width=100%,cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<audit-logging.adoc#,Audit Logging>>+| <<enabling-ssl.adoc#,Enabling SSL>>+| <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>+|+|===+****++****+[discrete]+=== Client APIs++<<client-apis.adoc#,*Client APIs*>>: Access Solr through various client APIs, including JavaScript, JSON, and Ruby.++// This pulls the sub-section list from the child page to reduce errors+include::client-apis.adoc[tag=client-sections]+****diff --git a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc b/solr/solr-ref-guide/src/dismax-query-parser.adocsimilarity index 93%rename from solr/solr-ref-guide/src/the-dismax-query-parser.adocrename to solr/solr-ref-guide/src/dismax-query-parser.adocindex ce20bfc367b..46c0c89e485 100644--- a/solr/solr-ref-guide/src/the-dismax-query-parser.adoc+++ b/solr/solr-ref-guide/src/dismax-query-parser.adoc@@ -1,4 +1,4 @@-= The DisMax Query Parser+= DisMax Query Parser// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -60,13 +60,14 @@ assigns `fieldOne` a boost of 2.3, leaves `fieldTwo` with the default boost (bec=== mm (Minimum Should Match) Parameter-When processing queries, Lucene/Solr recognizes three types of clauses: mandatory, prohibited, and "optional" (also known as "should" clauses). By default, all words or phrases specified in the `q` parameter are treated as "optional" clauses unless they are preceded by a "+" or a "-". When dealing with these "optional" clauses, the `mm` parameter makes it possible to say that a certain minimum number of those clauses must match. The DisMax query parser offers great flexibility in how the minimum number can be specified.+When processing queries, Lucene/Solr recognizes three types of clauses: mandatory, prohibited, and "optional" (also known as "should" clauses).+By default, all words or phrases specified in the `q` parameter are treated as "optional" clauses unless they are preceded by a "+" or a "-".+When dealing with these "optional" clauses, the `mm` parameter makes it possible to say that a certain minimum number of those clauses must match.+The DisMax query parser offers great flexibility in how the minimum number can be specified.The table below explains the various ways that mm values can be specified.-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="30,10,60",options="header"]+[%autowidth.stretch,options="header"]|===|Syntax |Example |Description|Positive integer |3 |Defines the minimum number of clauses that must match, regardless of how many clauses there are in total.@@ -150,13 +151,13 @@ Generally speaking, using `bq` (or `bf`, below) is considered a poor way to "boo"Multiplicative Boosting" is generally considered to be a more predictable method of influencing document score, because it acts as a "scaling factor" -- increasing (or decreasing) the scores of each document by a _relative_ amount.-The <<other-parsers.adoc#boost-query-parser,`{!boost}` QParser>> provides a convenient wrapper for implementing multiplicative boosting, and the <<the-extended-dismax-query-parser.adoc#extended-dismax-parameters,`{!edismax}` QParser>> offers a `boost` query parameter shortcut for using it.+The <<other-parsers.adoc#boost-query-parser,`{!boost}` QParser>> provides a convenient wrapper for implementing multiplicative boosting, and the <<edismax-query-parser.adoc#extended-dismax-parameters,`{!edismax}` QParser>> offers a `boost` query parameter shortcut for using it.======= bf (Boost Functions) Parameter-The `bf` parameter specifies functions (with optional <<the-standard-query-parser.adoc#boosting-a-term-with,query boost>>) that will be used to construct FunctionQueries which will be _added_ to the user's main query as optional clauses that will influence the score. Any <<function-queries.adoc#available-functions,function supported natively by Solr>> can be used, along with a boost value. For example:+The `bf` parameter specifies functions (with optional <<standard-query-parser.adoc#boosting-a-term-with,query boost>>) that will be used to construct FunctionQueries which will be _added_ to the user's main query as optional clauses that will influence the score. Any <<function-queries.adoc#available-functions,function supported natively by Solr>> can be used, along with a boost value. For example:[source,text]----diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adocdeleted file mode 100644index 894bf192cb9..00000000000--- a/solr/solr-ref-guide/src/distributed-requests.adoc+++ /dev/null@@ -1,216 +0,0 @@-= Distributed Requests-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--When a Solr node receives a search request, the request is routed behind the scenes to a replica of a shard that is part of the collection being searched.--The chosen replica acts as an aggregator: it creates internal requests to randomly chosen replicas of every shard in the collection, coordinates the responses, issues any subsequent internal requests as needed (for example, to refine facets values, or request additional stored fields), and constructs the final response for the client.--== Limiting Which Shards are Queried--While one of the advantages of using SolrCloud is the ability to query very large collections distributed across various shards, in some cases you may have configured Solr so you know <<shards-and-indexing-data-in-solrcloud.adoc#document-routing,you are only interested in results from a specific subset of shards>>. You have the option of searching over all of your data or just parts of it.--A query across all shards for a collection is simply a query that does not define a `shards` parameter:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*-------If you want to search just one shard, use the `shards` parameter to specify the shard by its logical ID, as in:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1-------If you want to search a group of shards, you can specify each shard separated by a comma in one request:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1,shard2-------In both of the above examples, while only the specific shards are queried, any random replica of the shard will get the request.--Alternatively, you can specify a list of replicas you wish to use in place of a shard IDs by separating the replica IDs with commas:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=localhost:7574/solr/gettingstarted,localhost:8983/solr/gettingstarted-------Or you can specify a list of replicas to choose from for a single shard (for load balancing purposes) by using the pipe symbol (|) between different replica IDs:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=localhost:7574/solr/gettingstarted|localhost:7500/solr/gettingstarted-------Finally, you can specify a list of shards (separated by commas) each defined by a list of replicas (seperated by pipes).--In the following example, 2 shards are queried, the first being a random replica from shard1, the second being a random replica from the explicit pipe delimited list:--[source,text]------http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1,localhost:7574/solr/gettingstarted|localhost:7500/solr/gettingstarted-------== Configuring the ShardHandlerFactory--For finer-grained control, you can directly configure and tune aspects of the concurrency and thread-pooling used within distributed search in Solr. The default configuration favors throughput over latency.--This is done by defining a `shardHandlerFactory` in the configuration for your search handler.--To add a `shardHandlerFactory` to the standard search handler, provide a configuration in `solrconfig.xml`, as in this example:--[source,xml]------<requestHandler name="/select" class="solr.SearchHandler">- <!-- other params go here -->- <shardHandlerFactory class="HttpShardHandlerFactory">- <int name="socketTimeout">1000</int>- <int name="connTimeout">5000</int>- </shardHandlerFactory>-</requestHandler>-------`HttpShardHandlerFactory` is the only `ShardHandlerFactory` implementation included out of the box with Solr, It accepts the following parameters:--`socketTimeout`::-The amount of time in ms that a socket is allowed to wait. The default is `0`, where the operating system's default will be used.--`connTimeout`::-The amount of time in ms that is accepted for binding / connecting a socket. The default is `0`, where the operating system's default will be used.--`maxConnectionsPerHost`::-The maximum number of concurrent connections that is made to each individual shard in a distributed search. The default is `100000`.--`corePoolSize`::-The retained lowest limit on the number of threads used in coordinating distributed search. The default is `0`.--`maximumPoolSize`::-The maximum number of threads used for coordinating distributed search. The default is `Integer.MAX_VALUE`.--`maxThreadIdleTime`::-The amount of time in seconds to wait for before threads are scaled back in response to a reduction in load. The default is `5`.--`sizeOfQueue`::-If specified, the thread pool will use a backing queue instead of a direct handoff buffer. High throughput systems will want to configure this to be a direct hand off (with `-1`). Systems that desire better latency will want to configure a reasonable size of queue to handle variations in requests. The default is `-1`.--`fairnessPolicy`::-Chooses the JVM specifics dealing with fair policy queuing, if enabled distributed searches will be handled in a First in First out fashion at a cost to throughput. If disabled throughput will be favored over latency. The default is `false`.--In addition, `HttpShardHandlerFactory` also depends on the following top-level property:--`allowUrls`::-See <<format-of-solr-xml.adoc#_allow_urls, Format of solr.allowUrls>>--[[distributedidf]]-== Configuring statsCache (Distributed IDF)--Document and term statistics are needed in order to calculate relevancy. Solr provides four implementations out of the box when it comes to document stats calculation:--* `LocalStatsCache`: This only uses local term and document statistics to compute relevance. In cases with uniform term distribution across shards, this works reasonably well. This option is the default if no `<statsCache>` is configured.-* `ExactStatsCache`: This implementation uses global values (across the collection) for document frequency.-* `ExactSharedStatsCache`: This is exactly like the exact stats cache in its functionality but the global stats are reused for subsequent requests with the same terms.-* `LRUStatsCache`: This implementation uses an LRU cache to hold global stats, which are shared between requests.--The implementation can be selected by setting `<statsCache>` in `solrconfig.xml`. For example, the following line makes Solr use the `ExactStatsCache` implementation:--[source,xml]------<statsCache class="org.apache.solr.search.stats.ExactStatsCache"/>-------== Avoiding Distributed Deadlock--Each shard serves top-level query requests and then makes sub-requests to all of the other shards. Care should be taken to ensure that the max number of threads serving HTTP requests is greater than the possible number of requests from both top-level clients and other shards. If this is not the case, the configuration may result in a distributed deadlock.--For example, a deadlock might occur in the case of two shards, each with just a single thread to service HTTP requests. Both threads could receive a top-level request concurrently, and make sub-requests to each other. Because there are no more remaining threads to service requests, the incoming requests will be blocked until the other pending requests are finished, but they will not finish since they are waiting for the sub-requests. By ensuring that Solr is configured to handle a sufficient number of threads, you can avoid deadlock situations like this.--== shards.preference Parameter--Solr allows you to pass an optional string parameter named `shards.preference` to indicate that a distributed query should sort the available replicas in the given order of precedence within each shard.--The syntax is: `shards.preference=_property_:__value__`. The order of the properties and the values are significant: the first one is the primary sort, the second is secondary, etc.--IMPORTANT: `shards.preference` is supported for single shard scenarios when using the SolrJ clients.--The properties that can be specified are as follows:--`replica.type`::-One or more replica types that are preferred. Any combination of `PULL`, `TLOG` and `NRT` is allowed.--`replica.location`::-One or more replica locations that are preferred.-+-A location starts with `http://hostname:port`. Matching is done for the given string as a prefix, so it's possible to e.g., leave out the port.-+-A special value `local` may be used to denote any local replica running on the same Solr instance as the one handling the query. This is useful when a query requests many fields or large fields to be returned per document because it avoids moving large amounts of data over the network when it is available locally. In addition, this feature can be useful for minimizing the impact of a problematic replica with degraded performance, as it reduces the likelihood that the degraded replica will be hit by other healthy replicas.-+-The value of `replica.location:local` diminishes as the number of shards (that have no locally-available replicas) in a collection increases because the query controller will have to direct the query to non-local replicas for most of the shards.-+-In other words, this feature is mostly useful for optimizing queries directed towards collections with a small number of shards and many replicas.-+-Also, this option should only be used if you are load balancing requests across all nodes that host replicas for the collection you are querying, as Solr's `CloudSolrClient` will do. If not load-balancing, this feature can introduce a hotspot in the cluster since queries won't be evenly distributed across the cluster.--`replica.base`::-Applied after sorting by inherent replica attributes, this property defines a fallback ordering among sets of preference-equivalent replicas; if specified, only one value may be specified for this property, and it must be specified last.-+-`random`, the default, randomly shuffles replicas for each request. This distributes requests evenly, but can result in sub-optimal cache usage for shards with replication factor > 1.-+-`stable:dividend:_paramName_` parses an integer from the value associated with the given parameter name; this integer is used as the dividend (mod equivalent replica count) to determine (via list rotation) order of preference among equivalent replicas.-+-`stable[:hash[:_paramName_]]` the string value associated with the given parameter name is hashed to a dividend that is used to determine replica preference order (analogous to the explicit `dividend` property above); `_paramName_` defaults to `q` if not specified, providing stable routing keyed to the string value of the "main query". Note that this may be inappropriate for some use cases (e.g., static main queries that leverage parameter substitution)--`node.sysprop`::-Query will be routed to nodes with same defined system properties as the current one. For example, if you start Solr nodes on different racks, you'll want to identify those nodes by a <<configuring-solrconfig-xml.adoc#jvm-system-properties,system property>> (e.g., `-Drack=rack1`). Then, queries can contain `shards.preference=node.sysprop:sysprop.rack`, to make sure you always hit shards with the same value of `rack`.--`replica.leader`::-Prefer replicas based on their leader status, set to either `true` or `false`. Consider a shard with two `TLOG` replicas and four `PULL` replicas (six replicas in total, one of which is the leader).-With `shards.preference=replica.leader:false`, 5 out of 6 replicas will be preferred. Contrast this with `shards.preference=replica.type:PULL` where only 4 of 6 replicas will be preferred.-Note that the non-leader `TLOG` replica behaves like a `PULL` replica from a search perspective; it pulls index updates from the leader just like a `PULL` replica and does not perform soft-commits.-The difference is that the non-leader `TLOG` replica also captures updates in its TLOG, so that it is a candidate to replace the current leader if it is lost.--Examples:--* Prefer stable routing (keyed to client "sessionId" parameter) among otherwise equivalent replicas:- `shards.preference=replica.base:stable:hash:sessionId&sessionId=abc123`--* Prefer PULL replicas:- `shards.preference=replica.type:PULL`--* Prefer PULL replicas, or TLOG replicas if PULL replicas not available:- `shards.preference=replica.type:PULL,replica.type:TLOG`--* Prefer any local replicas:- `shards.preference=replica.location:local`--* Prefer any replicas on a host called "server1" with "server2" as the secondary option:- `shards.preference=replica.location:http://server1,replica.location:http://server2`--* Prefer PULL replicas if available, otherwise TLOG replicas, and local ones among those:- `shards.preference=replica.type:PULL,replica.type:TLOG,replica.location:local`--* Prefer local replicas, and among them PULL replicas when available TLOG otherwise:- `shards.preference=replica.location:local,replica.type:PULL,replica.type:TLOG`--* Prefer any replica that is not a leader:- `shards.preference=replica.leader:false`--Note that if you provide the settings in a query string, they need to be properly URL-encoded.diff --git a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc b/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adocdeleted file mode 100644index 7e7ebe3af15..00000000000--- a/solr/solr-ref-guide/src/distributed-search-with-index-sharding.adoc+++ /dev/null@@ -1,178 +0,0 @@-= Distributed Search with Index Sharding-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--When using traditional index sharding, you will need to consider how to query your documents.--It is highly recommended that you use <<solrcloud.adoc#,SolrCloud>> when needing to scale up or scale out. The setup described below is legacy and was used prior to the existence of SolrCloud. SolrCloud provides for a truly distributed set of features with support for things like automatic routing, leader election, optimistic concurrency and other sanity checks that are expected out of a distributed system.--Everything on this page is specific to legacy setup of distributed search. Users trying out SolrCloud should not follow any of the steps or information below.--Update reorders (i.e., replica A may see update X then Y, and replica B may see update Y then X). *deleteByQuery* also handles reorders the same way, to ensure replicas are consistent. All replicas of a shard are consistent, even if the updates arrive in a different order on different replicas.--== Distributing Documents across Shards--When not using SolrCloud, it is up to you to get all your documents indexed on each shard of your server farm. Solr supports distributed indexing (routing) in its true form only in the SolrCloud mode.--In the legacy distributed mode, Solr does not calculate universal term/doc frequencies. For most large-scale implementations, it is not likely to matter that Solr calculates TF/IDF at the shard level. However, if your collection is heavily skewed in its distribution across servers, you may find misleading relevancy results in your searches. In general, it is probably best to randomly distribute documents to your shards.--== Executing Distributed Searches with the shards Parameter--If a query request includes the `shards` parameter, the Solr server distributes the request across all the shards listed as arguments to the parameter. The `shards` parameter uses this syntax:--`host:port/base_url,host:port/base_url*`--For example, the `shards` parameter below causes the search to be distributed across two Solr servers: *solr1* and **solr2**, both of which are running on port 8983:--`\http://localhost:8983/solr/core1/select?shards=solr1:8983/solr/core1,solr2:8983/solr/core1&indent=true&q=ipod+solr`--Rather than require users to include the shards parameter explicitly, it is usually preferred to configure this parameter as a default in the RequestHandler section of `solrconfig.xml`.--[IMPORTANT]-====-Do not add the `shards` parameter to the standard request handler; doing so may cause search queries may enter an infinite loop. Instead, define a new request handler that uses the `shards` parameter, and pass distributed search requests to that handler.-====--With Legacy mode, only query requests are distributed. This includes requests to the SearchHandler (or any handler extending from `org.apache.solr.handler.component.SearchHandler`) using standard components that support distributed search.--As in SolrCloud mode, when `shards.info=true`, distributed responses will include information about the shard (where each shard represents a logically different index or physical location)--The following components support distributed search:--* The *Query* component, which returns documents matching a query-* The *Facet* component, which processes facet.query and facet.field requests where facets are sorted by count (the default).-* The *Highlighting* component, which enables Solr to include "highlighted" matches in field values.-* The *Stats* component, which returns simple statistics for numeric fields within the DocSet.-* The *Debug* component, which helps with debugging.--=== URL Allow List--The nodes allowed in the `shards` parameter is configurable through the `allowUrls` property in `solr.xml`. This allow-list is automatically configured for SolrCloud but needs explicit configuration for leader/follower mode. Read more details in the section <<distributed-requests.adoc#configuring-the-shardhandlerfactory,Configuring the ShardHandlerFactory>>.--== Limitations to Distributed Search--Distributed searching in Solr has the following limitations:--* Each document indexed must have a unique key.-* If Solr discovers duplicate document IDs, Solr selects the first document and discards subsequent documents.-* The index for distributed searching may become momentarily out of sync if a commit happens between the first and second phase of the distributed search. This might cause a situation where a document that once matched a query and was subsequently changed may no longer match the query but will still be retrieved. This situation is expected to be quite rare, however, and is only possible for a single query request.-* The number of shards is limited by number of characters allowed for GET method's URI; most Web servers generally support at least 4000 characters, but many servers limit URI length to reduce their vulnerability to Denial of Service (DoS) attacks.-* Shard information can be returned with each document in a distributed search by including `fl=id, [shard]` in the search request. This returns the shard URL.-* In a distributed search, the data directory from the core descriptor overrides any data directory in `solrconfig.xml.`-* Update commands may be sent to any server with distributed indexing configured correctly. Document adds and deletes are forwarded to the appropriate server/shard based on a hash of the unique document id. *commit* commands and *deleteByQuery* commands are sent to every server in `shards`.--Formerly a limitation was that TF/IDF relevancy computations only used shard-local statistics. This is still the case by default. If your data isn't randomly distributed, or if you want more exact statistics, then remember to configure the ExactStatsCache.--== Avoiding Distributed Deadlock with Distributed Search--Like in SolrCloud mode, inter-shard requests could lead to a distributed deadlock. It can be avoided by following the instructions in the section <<distributed-requests.adoc#,Distributed Requests>>.--== Testing Index Sharding on Two Local Servers--For simple functional testing, it's easiest to just set up two local Solr servers on different ports. (In a production environment, of course, these servers would be deployed on separate machines.)--. Make two Solr home directories and copy `solr.xml` into the new directories:-+-[source,bash]------mkdir example/nodes-mkdir example/nodes/node1-# Copy solr.xml into this solr.home-cp server/solr/solr.xml example/nodes/node1/.-# Repeat the above steps for the second node-mkdir example/nodes/node2-cp server/solr/solr.xml example/nodes/node2/.------. Start the two Solr instances-+-[source,bash]------# Start first node on port 8983-bin/solr start -s example/nodes/node1 -p 8983--# Start second node on port 8984-bin/solr start -s example/nodes/node2 -p 8984------. Create a core on both the nodes with the sample_techproducts_configs.-+-[source,bash]------bin/solr create_core -c core1 -p 8983 -d sample_techproducts_configs-# Create a core on the Solr node running on port 8984-bin/solr create_core -c core1 -p 8984 -d sample_techproducts_configs------. In a third window, index an example document to each of the server:-+-[source,bash]------bin/post -c core1 example/exampledocs/monitor.xml -port 8983--bin/post -c core1 example/exampledocs/monitor2.xml -port 8984------. Search on the node on port 8983:-+-[source,bash]------curl http://localhost:8983/solr/core1/select?q=*:*&wt=xml&indent=true------+-This should bring back one document.-+-Search on the node on port 8984:-+-[source,bash]------curl http://localhost:8984/solr/core1/select?q=*:*&wt=xml&indent=true------+-This should also bring back a single document.-+-Now do a distributed search across both servers with your browser or `curl.` In the example below, an extra parameter 'fl' is passed to restrict the returned fields to id and name.-+-[source,bash]------curl http://localhost:8983/solr/core1/select?q=*:*&indent=true&shards=localhost:8983/solr/core1,localhost:8984/solr/core1&fl=id,name&wt=xml------+-This should contain both the documents as shown below:-+-[source,xml]------<response>- <lst name="responseHeader">- <int name="status">0</int>- <int name="QTime">8</int>- <lst name="params">- <str name="q">*:*</str>- <str name="shards">localhost:8983/solr/core1,localhost:8984/solr/core1</str>- <str name="indent">true</str>- <str name="fl">id,name</str>- <str name="wt">xml</str>- </lst>- </lst>- <result name="response" numFound="2" start="0" maxScore="1.0">- <doc>- <str name="id">3007WFP</str>- <str name="name">Dell Widescreen UltraSharp 3007WFP</str>- </doc>- <doc>- <str name="id">VA902B</str>- <str name="name">ViewSonic VA902B - flat panel display - TFT - 19"</str>- </doc>- </result>-</response>-----diff --git a/solr/solr-ref-guide/src/solr-tracing.adoc b/solr/solr-ref-guide/src/distributed-tracing.adocsimilarity index 92%rename from solr/solr-ref-guide/src/solr-tracing.adocrename to solr/solr-ref-guide/src/distributed-tracing.adocindex 6a87bcdd3d7..e6f19b3074d 100644--- a/solr/solr-ref-guide/src/solr-tracing.adoc+++ b/solr/solr-ref-guide/src/distributed-tracing.adoc@@ -1,4 +1,4 @@-= Distributed Solr Tracing+= Distributed Tracing// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -24,7 +24,7 @@ At the moment, only Jaeger is supported out of the box.A sampled distributed tracing query request on Jaeger looks like this:.Tracing of a Solr query-image::images/solr-tracing/query-request-tracing.png[]+image::images/distributed-tracing/query-request-tracing.png[]== Configuring Tracer@@ -61,7 +61,8 @@ Then Jaeger tracer is configured in `solr.xml` like this:<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />-----_There are no configuration elements in the XML_; instead, this 3rd party system is configured using System Properties or Environment Variables. The full list are listed at https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].+_There are no configuration elements in the XML_; instead, this 3rd party system is configured using System Properties or Environment Variables.+The full list are listed at https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].For example, to use the probabilistic sampler, you could set this environment variable:@@ -75,4 +76,4 @@ or System property:[source,bash]----bin/solr start -DJAEGER_SAMPLER_TYPE=probabilistic-----\ No newline at end of file+----diff --git a/solr/solr-ref-guide/src/docker-faq.adoc b/solr/solr-ref-guide/src/docker-faq.adocindex 5c4be6f8d9e..df90235a6fc 100644--- a/solr/solr-ref-guide/src/docker-faq.adoc+++ b/solr/solr-ref-guide/src/docker-faq.adoc@@ -225,7 +225,7 @@ Then go to `+http://localhost:8983/solr/#/~cloud+` (adjust the hostname for your== How can I run ZooKeeper and Solr with Docker Compose?-See the <<running-solr-in-docker.adoc#docker-compose,docker compose example>>.+See the <<solr-in-docker.adoc#docker-compose,docker compose example>>.== How can I get rid of "shared memory" warnings on Solr startup?diff --git a/solr/solr-ref-guide/src/document-analysis.adoc b/solr/solr-ref-guide/src/document-analysis.adocnew file mode 100644index 00000000000..2f60f12860d--- /dev/null+++ b/solr/solr-ref-guide/src/document-analysis.adoc@@ -0,0 +1,61 @@+= Document Analysis in Solr+:page-children: analyzers, \+ tokenizers, \+ filters, \+ charfilterfactories, \+ language-analysis, \+ phonetic-matching, \+ analysis-screen+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++The following sections describe how Solr breaks down and works with textual data.+There are three main concepts to understand: analyzers, tokenizers, and filters.++* <<analyzers.adoc#,Field analyzers>> are used both during ingestion, when a document is indexed, and at query time. An analyzer examines the text of fields and generates a token stream. Analyzers may be a single class or they may be composed of a series of tokenizer and filter classes.+* <<tokenizers.adoc#,Tokenizers>> break field data into lexical units, or _tokens_.+* <<filters.adoc#,Filters>> examine a stream of tokens and keep them, transform or discard them, or create new ones. Tokenizers and filters may be combined to form pipelines, or _chains_, where the output of one is input to the next. Such a sequence of tokenizers and filters is called an _analyzer_ and the resulting output of an analyzer is used to match query results or build indices.++== Using Analyzers, Tokenizers, and Filters++Although the analysis process is used for both indexing and querying, it is not required to use the same analysis process for both operations.+For indexing, you often want to simplify, or normalize, words.+For example, setting all letters to lowercase, eliminating punctuation and accents, mapping words to their stems, and so on.+Doing so can increase recall because, for example, "ram", "Ram" and "RAM" would all match a query for "ram".+To increase query-time precision, a filter could narrow the matches by, for example, ignoring all-cap acronyms if you're interested in male sheep, but not Random Access Memory.++The tokens output by the analysis process define the values, or _terms_, of that field and are used either to build an index of those terms when a new document is added, or to identify which documents contain the terms you are querying for.++=== For More Information++These sections will show you how to configure field analyzers and also serves as a reference for the details of configuring each of the available tokenizer and filter classes.+It also serves as a guide so that you can configure your own analysis classes if you have special needs that cannot be met with the included filters or tokenizers.++****+// This tags the below list so it can be used in the parent page section list+// tag::analysis-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<analyzers.adoc#,Analyzers>>: Overview of Solr analyzers.+| <<tokenizers.adoc#,Tokenizers>>: Tokenizers and tokenizer factory classes.+| <<filters.adoc#,Filters>>: Filters and filter factory classes.+| <<charfilterfactories.adoc#,CharFilterFactories>>: Filters for pre-processing input characters.+| <<language-analysis.adoc#,Language Analysis>>: Tokenizers and filters for character set conversion and specific languages.+| <<analysis-screen.adoc#,Analysis Screen>>: Admin UI for testing field analysis.+|===+// end::analysis-sections[]+****diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/document-transformers.adocsimilarity index 95%rename from solr/solr-ref-guide/src/transforming-result-documents.adocrename to solr/solr-ref-guide/src/document-transformers.adocindex 236864f2de1..689fdf74dc0 100644--- a/solr/solr-ref-guide/src/transforming-result-documents.adoc+++ b/solr/solr-ref-guide/src/document-transformers.adoc@@ -1,4 +1,4 @@-= Transforming Result Documents+= Document Transformers// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -27,7 +27,7 @@ When executing a request, a document transformer can be used by including it infl=id,name,score,[shard]-----Some transformers allow, or require, local parameters which can be specified as key value pairs inside the brackets:+Some transformers allow, or require, local params which can be specified as key value pairs inside the brackets:[source,plain]----@@ -86,7 +86,7 @@ In addition to using these request parameters, you can configure additional name</transformer>-----The `value` option forces an explicit value to always be used, while the `defaultValue` option provides a default that can still be overridden using the `v` and `t` local parameters.+The `value` option forces an explicit value to always be used, while the `defaultValue` option provides a default that can still be overridden using the `v` and `t` local params.=== [explain] - ExplainAugmenterFactory@@ -127,7 +127,7 @@ A default style can be configured by specifying an `args` parameter in your `solThis transformer returns all <<indexing-nested-documents.adoc#,descendant documents>> of each parent document matching your query. This is useful when you have indexed nested child documents and want to retrieve the child documents for the relevant parent documents for any type of search query.-Note that this transformer can be used even when the query used to match the result documents is not a <<other-parsers.adoc#block-join-query-parsers,Block Join query>>.+Note that this transformer can be used even when the query used to match the result documents is not a <<block-join-query-parser.adoc#,Block Join query>>.[source,plain]@@ -198,7 +198,7 @@ DocIdAugmenterFactory does not support any request parameters, or configuration=== [elevated] and [excluded]-These transformers are available only when using the <<the-query-elevation-component.adoc#,Query Elevation Component>>.+These transformers are available only when using the <<query-elevation-component.adoc#,Query Elevation Component>>.* `[elevated]` annotates each document to indicate if it was elevated or not.* `[excluded]` annotates each document to indicate if it would have been excluded - this is only supported if you also use the `markExcludes` parameter.@@ -318,7 +318,7 @@ q=*:*&fl=*,**foo**:[subquery]&**foo.**q=to be continued&**foo.**rows=10&**foo.**==== Document Field as an Input for Subquery Parameters-It's necessary to pass some document field values as a parameter for subquery. It's supported via an implicit *`row.__fieldname__`* parameter, and can be (but might not only) referred via local parameters syntax:+It's necessary to pass some document field values as a parameter for subquery. It's supported via an implicit *`row.__fieldname__`* parameter, and can be (but might not only) referred via local params syntax:[source,plain,subs="quotes"]q=name:john&fl=name,id,depts:[subquery]&depts.q={!terms f=id **v=$row.dept_id**}&depts.rows=10@@ -331,7 +331,9 @@ To log substituted subquery request parameters, add the corresponding parameter==== Cores and Collections in SolrCloud-Use `foo:[subquery fromIndex=departments]` to invoke subquery on another core on the same node. This is what `{!join}` does for non-SolrCloud mode. But with SolrCloud, just (and only) explicitly specify its native parameters like `collection, shards` for subquery, for example:+Use `foo:[subquery fromIndex=departments]` to invoke subquery on another core on the same node.+This is what `{!join}` does in a user-managed cluster.+With SolrCloud, only specify its native parameters like `collection, shards` for subquery, for example:[source,plain,subs="quotes"]q=\*:*&fl=\*,foo:[subquery]&foo.q=cloud&**foo.collection**=departmentsdiff --git a/solr/solr-ref-guide/src/documents-fields-and-schema-design.adoc b/solr/solr-ref-guide/src/documents-fields-and-schema-design.adocdeleted file mode 100644index 04c4ae599c0..00000000000--- a/solr/solr-ref-guide/src/documents-fields-and-schema-design.adoc+++ /dev/null@@ -1,44 +0,0 @@-= Documents, Fields, and Schema Design-:page-children: overview-of-documents-fields-and-schema-design, solr-field-types, defining-fields, copying-fields, dynamic-fields, other-schema-elements, schema-api, putting-the-pieces-together, docvalues, schemaless-mode, luke-request-handler-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section discusses how Solr organizes its data into documents and fields, as well as how to work with a schema in Solr.--This section includes the following topics:--<<overview-of-documents-fields-and-schema-design.adoc#,Overview of Documents, Fields, and Schema Design>>: An introduction to the concepts covered in this section.--<<solr-field-types.adoc#,Solr Field Types>>: Detailed information about field types in Solr, including the field types in the default Solr schema.--<<defining-fields.adoc#,Defining Fields>>: Describes how to define fields in Solr.--<<copying-fields.adoc#,Copying Fields>>: Describes how to populate fields with data copied from another field.--<<dynamic-fields.adoc#,Dynamic Fields>>: Information about using dynamic fields in order to catch and index fields that do not exactly conform to other field definitions in your schema.--<<schema-api.adoc#,Schema API>>: Use curl commands to read various parts of a schema or create new fields and copyField rules.--<<other-schema-elements.adoc#,Other Schema Elements>>: Describes other important elements in the Solr schema.--<<putting-the-pieces-together.adoc#,Putting the Pieces Together>>: A higher-level view of the Solr schema and how its elements work together.--<<docvalues.adoc#,DocValues>>: Describes how to create a docValues index for faster lookups.--<<schemaless-mode.adoc#,Schemaless Mode>>: Automatically add previously unknown schema fields using value-based field type guessing.--<<luke-request-handler.adoc#,Luke Request Handler>>: The request handler which provides access to information about fields in the index. This request handler powers the <<schema-browser-screen.adoc#,Schema Browser>> page of Solr's Admin UI.diff --git a/solr/solr-ref-guide/src/documents-fields-schema-design.adoc b/solr/solr-ref-guide/src/documents-fields-schema-design.adocnew file mode 100644index 00000000000..33f5ac56dde--- /dev/null+++ b/solr/solr-ref-guide/src/documents-fields-schema-design.adoc@@ -0,0 +1,99 @@+= Documents, Fields, and Schema Design+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+The fundamental premise of Solr is simple.+You give it a lot of information, then later you can ask it questions and find the piece of information you want.++The part where you feed in all the information is called _indexing_ or _updating_. When you ask a question, it's called a _query_.++One way to understand how Solr works is to think of a loose-leaf book of recipes.+Every time you add a recipe to the book, you update the index at the back.+You list each ingredient and the page number of the recipe you just added.+Suppose you add one hundred recipes.+Using the index, you can very quickly find all the recipes that use garbanzo beans, or artichokes, or coffee, as an ingredient.+Using the index is much faster than looking through each recipe one by one.+Imagine a book of one thousand recipes, or one million.++Solr allows you to build an index with many different fields, or types of entries.+The example above shows how to build an index with just one field, `ingredients`.+You could have other fields in the index for the recipe's cooking style, like `Asian`, `Cajun`, or `vegan`, and you could have an index field for preparation times.+Solr can answer questions like "What Cajun-style recipes that have blood oranges as an ingredient can be prepared in fewer than 30 minutes?"++The schema is the place where you tell Solr how it should build indexes from input documents.++== How Solr Sees the World++Solr's basic unit of information is a _document_, which is a set of data that describes something.+A recipe document would contain the ingredients, the instructions, the preparation time, the cooking time, the tools needed, and so on.+A document about a person, for example, might contain the person's name, biography, favorite color, and shoe size.+A document about a book could contain the title, author, year of publication, number of pages, and so on.++In the Solr universe, documents are composed of _fields_, which are more specific pieces of information. Shoe size could be a field.+First name and last name could be fields.++Fields can contain different kinds of data.+A name field, for example, is text (character data).+A shoe size field might be a floating point number so that it could contain values like 6 and 9.5.+Obviously, the definition of fields is flexible (you could define a shoe size field as a text field rather than a floating point number, for example), but if you define your fields correctly, Solr will be able to interpret them correctly and your users will get better results when they perform a query.++You can tell Solr about the kind of data a field contains by specifying its _field type_.+The field type tells Solr how to interpret the field and how it can be queried.++When you add a document, Solr takes the information in the document's fields and adds that information to an index.+When you perform a query, Solr can quickly consult the index and return the matching documents.++== Field Analysis++_Field analysis_ tells Solr what to do with incoming data when building an index.+A more accurate name for this process would be _processing_ or even _digestion_, but the official name is _analysis_.++Consider, for example, a biography field in a person document.+Every word of the biography must be indexed so that you can quickly find people whose lives have had anything to do with ketchup, or dragonflies, or cryptography.++However, a biography will likely contains lots of words you don't care about and don't want clogging up your index—words like "the", "a", "to", and so forth.+Furthermore, suppose the biography contains the word "Ketchup", capitalized at the beginning of a sentence.+If a user makes a query for "ketchup", you want Solr to tell you about the person even though the biography contains the capitalized word.++The solution to both these problems is field analysis.+For the biography field, you can tell Solr how to break apart the biography into words.+You can tell Solr that you want to make all the words lower case, and you can tell Solr to remove accents marks.++Field analysis is an important part of a field type.+<<document-analysis.adoc#,Document Analysis in Solr>> is a detailed description of field analysis.++== Solr's Schema File++Solr stores details about the field types and fields it is expected to understand in a schema file.+This file is named either `managed-schema` or `schema.xml`.+The difference is determined by how you plan to manage Solr's schema in your installation: either programmatically or by hand-editing.++An important fact about the schema is that it is a Solr concept.+The actual data in your index is stored in Lucene, and Lucene does not have the concept of a schema.+This means that changes to the schema file will not have any impact on data already stored in the index.+In fact, changing the schema without reindexing your data can cause serious problems with the index, to the point where the only solution is to reindex your data entirely.++It is good practice to try to think about the data you plan to index: what kind of data is it?+How do you want to be able to search it?+How will fields from documents be displayed to users?++If you aren't sure yet, plan on some test indexing runs to see how the data in your documents is indexed with default settings.+Build into your implementation plan some time for iteration and start small.+The more you're able to define your schema before indexing all of your documents, the higher your chances for a successful search application for your users.++More information about the schema is in the section <<schema-elements.adoc#,Schema Elements>>.diff --git a/solr/solr-ref-guide/src/documents-screen.adoc b/solr/solr-ref-guide/src/documents-screen.adocindex d6f5d409ef5..15fe607131d 100644--- a/solr/solr-ref-guide/src/documents-screen.adoc+++ b/solr/solr-ref-guide/src/documents-screen.adoc@@ -31,14 +31,14 @@ The screen allows you to:====There are other ways to load data, see also these sections:-* <<uploading-data-with-index-handlers.adoc#,Uploading Data with Index Handlers>>-* <<uploading-data-with-solr-cell-using-apache-tika.adoc#,Uploading Data with Solr Cell using Apache Tika>>+* <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>+* <<indexing-with-tika.adoc#,Indexing with Solr Cell and Apache Tika>>====== Common Fields* Request-Handler: The first step is to define the RequestHandler. By default `/update` will be defined. Change the request handler to `/update/extract` to use Solr Cell.* Document Type: Select the Document Type to define the format of document to load. The remaining parameters may change depending on the document type selected.-* Document(s): Enter a properly-formatted Solr document corresponding to the `Document Type` selected. XML and JSON documents must be formatted in a Solr-specific format, a small illustrative document will be shown. CSV files should have headers corresponding to fields defined in the schema. More details can be found at: <<uploading-data-with-index-handlers.adoc#,Uploading Data with Index Handlers>>.+* Document(s): Enter a properly-formatted Solr document corresponding to the `Document Type` selected. XML and JSON documents must be formatted in a Solr-specific format, a small illustrative document will be shown. CSV files should have headers corresponding to fields defined in the schema. More details can be found at: <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>.* Commit Within: Specify the number of milliseconds between the time the document is submitted and when it is available for searching.* Overwrite: If `true` the new document will replace an existing document with the same value in the `id` field. If `false` multiple documents with the same id can be added.@@ -62,7 +62,7 @@ The Document Builder provides a wizard-like interface to enter fields of a documThe File Upload option allows choosing a prepared file and uploading it. If using `/update` for the Request-Handler option, you will be limited to XML, CSV, and JSON.Other document types (e.g., Word, PDF, etc.) can be indexed using the ExtractingRequestHandler (aka, Solr Cell). You must modify the RequestHandler to `/update/extract`, which must be defined in your `solrconfig.xml` file with your desired defaults. You should also add `&literal.id` shown in the "Extracting Request Handler Params" field so the file chosen is given a unique id.-More information can be found at: <<uploading-data-with-solr-cell-using-apache-tika.adoc#,Uploading Data with Solr Cell using Apache Tika>>+More information can be found at: <<indexing-with-tika.adoc#,Indexing with Solr Cell and Apache Tika>>.== Solr Commanddiff --git a/solr/solr-ref-guide/src/docvalues.adoc b/solr/solr-ref-guide/src/docvalues.adocindex 2f05d08d0e9..ecf4a6fe9e6 100644--- a/solr/solr-ref-guide/src/docvalues.adoc+++ b/solr/solr-ref-guide/src/docvalues.adoc@@ -77,9 +77,16 @@ If `docValues="true"` for a field, then DocValues will automatically be used any=== Retrieving DocValues During Search-Field values retrieved during search queries are typically returned from stored values. However, non-stored docValues fields will be also returned along with other stored fields when all fields (or pattern matching globs) are specified to be returned (e.g., "`fl=*`") for search queries depending on the effective value of the `useDocValuesAsStored` parameter for each field. For schema versions >= 1.6, the implicit default is `useDocValuesAsStored="true"`. See <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>> & <<defining-fields.adoc#,Defining Fields>> for more details.+Field values retrieved during search queries are typically returned from stored values.+However, non-stored docValues fields will be also returned along with other stored fields when all fields (or pattern matching globs) are specified to be returned (e.g., "`fl=*`") for search queries depending on the effective value of the `useDocValuesAsStored` parameter for each field. For schema versions >= 1.6, the implicit default is `useDocValuesAsStored="true"`.+See <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>> & <<fields.adoc#,Fields>> for more details.-When `useDocValuesAsStored="false"`, non-stored DocValues fields can still be explicitly requested by name in the <<common-query-parameters.adoc#fl-field-list-parameter,`fl` parameter>>, but will not match glob patterns (`"*"`). Note that returning DocValues along with "regular" stored fields at query time has performance implications that stored fields may not because DocValues are column-oriented and may therefore incur additional cost to retrieve for each returned document. Also note that while returning non-stored fields from DocValues, the values of a multi-valued field are returned in sorted order rather than insertion order and may have duplicates removed, see above. If you require the multi-valued fields to be returned in the original insertion order, then make your multi-valued field as stored (such a change requires reindexing).+When `useDocValuesAsStored="false"`, non-stored DocValues fields can still be explicitly requested by name in the <<common-query-parameters.adoc#fl-field-list-parameter,`fl` parameter>>, but will not match glob patterns (`"*"`).++Returning DocValues along with "regular" stored fields at query time has performance implications that stored fields may not because DocValues are column-oriented and may therefore incur additional cost to retrieve for each returned document.++While returning non-stored fields from DocValues, the values of a multi-valued field are returned in sorted order rather than insertion order and may have duplicates removed, see above.+If you require the multi-valued fields to be returned in the original insertion order, then make your multi-valued field as stored (such a change requires reindexing).In cases where the query is returning _only_ docValues fields performance may improve since returning stored fields requires disk reads and decompression whereas returning docValues fields in the fl list only requires memory access.diff --git a/solr/solr-ref-guide/src/dynamic-fields.adoc b/solr/solr-ref-guide/src/dynamic-fields.adocindex 65a91ffa2de..873bd223053 100644--- a/solr/solr-ref-guide/src/dynamic-fields.adoc+++ b/solr/solr-ref-guide/src/dynamic-fields.adoc@@ -18,11 +18,14 @@_Dynamic fields_ allow Solr to index fields that you did not explicitly define in your schema.-This is useful if you discover you have forgotten to define one or more fields. Dynamic fields can make your application less brittle by providing some flexibility in the documents you can add to Solr.+This is useful if you discover you have forgotten to define one or more fields.+Dynamic fields can make your application less brittle by providing some flexibility in the documents you can add to Solr.-A dynamic field is just like a regular field except it has a name with a wildcard in it. When you are indexing documents, a field that does not match any explicitly defined fields can be matched with a dynamic field.+A dynamic field is just like a regular field except it has a name with a wildcard in it.+When you are indexing documents, a field that does not match any explicitly defined fields can be matched with a dynamic field.-For example, suppose your schema includes a dynamic field with a name of `*_i`. If you attempt to index a document with a `cost_i` field, but no explicit `cost_i` field is defined in the schema, then the `cost_i` field will have the field type and analysis defined for `*_i`.+For example, suppose your schema includes a dynamic field with a name of `*_i`.+If you attempt to index a document with a `cost_i` field, but no explicit `cost_i` field is defined in the schema, then the `cost_i` field will have the field type and analysis defined for `*_i`.Like regular fields, dynamic fields have a name, a field type, and options.@@ -31,4 +34,5 @@ Like regular fields, dynamic fields have a name, a field type, and options.<dynamicField name="*_i" type="int" indexed="true" stored="true"/>-----It is recommended that you include basic dynamic field mappings (like that shown above) in your `schema.xml`. The mappings can be very useful.+It is recommended that you include basic dynamic field mappings (like that shown above) in your `schema.xml`.+The mappings can be very useful.diff --git a/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc b/solr/solr-ref-guide/src/edismax-query-parser.adocsimilarity index 67%rename from solr/solr-ref-guide/src/the-extended-dismax-query-parser.adocrename to solr/solr-ref-guide/src/edismax-query-parser.adocindex 7f1c4d7074e..c015ac73206 100644--- a/solr/solr-ref-guide/src/the-extended-dismax-query-parser.adoc+++ b/solr/solr-ref-guide/src/edismax-query-parser.adoc@@ -1,4 +1,4 @@-= The Extended DisMax (eDismax) Query Parser+= Extended DisMax (eDisMax) Query Parser// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,30 +16,35 @@// specific language governing permissions and limitations// under the License.-The Extended DisMax (eDisMax) query parser is an improved version of the <<the-dismax-query-parser.adoc#,DisMax query parser>>.+The Extended DisMax (eDisMax) query parser is an improved version of the <<dismax-query-parser.adoc#,DisMax query parser>>.-In addition to supporting all the DisMax query parser parameters, Extended Dismax:+In addition to supporting all the DisMax query parser parameters, Extended DisMax:-* supports <<the-standard-query-parser.adoc#,Solr's standard query parser>> syntax such as (non-exhaustive list):+* supports <<standard-query-parser.adoc#,Solr's standard query parser>> syntax such as (non-exhaustive list):** boolean operators such as AND (+, &&), OR (||), NOT (-).** optionally treats lowercase "and" and "or" as "AND" and "OR" in Lucene syntax mode** optionally allows embedded queries using other query parsers or functions* includes improved smart partial escaping in the case of syntax errors; fielded queries, +/-, and phrase queries are still supported in this mode.* improves proximity boosting by using word shingles; you do not need the query to match all words in the document before proximity boosting is applied.-* includes advanced stopword handling: stopwords are not required in the mandatory part of the query but are still used in the proximity boosting part. If a query consists of all stopwords, such as "to be or not to be", then all words are required.-* includes improved boost function: in Extended DisMax, the `boost` function is a multiplier <<the-dismax-query-parser.adoc#bq-bf-shortcomings,rather than an addend>>, improving your boost results; the additive boost functions of DisMax (`bf` and `bq`) are also supported.+* includes advanced stopword handling: stopwords are not required in the mandatory part of the query but are still used in the proximity boosting part.+If a query consists of all stopwords, such as "to be or not to be", then all words are required.+* includes improved boost function: in Extended DisMax, the `boost` function is a multiplier <<dismax-query-parser.adoc#bq-bf-shortcomings,rather than an addend>>, improving your boost results; the additive boost functions of DisMax (`bf` and `bq`) are also supported.* supports pure negative nested queries: queries such as `+foo (-foo)` will match all documents.* lets you specify which fields the end user is allowed to query, and to disallow direct fielded searches.== Extended DisMax Parameters-In addition to all the <<the-dismax-query-parser.adoc#dismax-query-parser-parameters,DisMax parameters>>, Extended DisMax includes these query parameters:+In addition to all the <<dismax-query-parser.adoc#dismax-query-parser-parameters,DisMax parameters>>, Extended DisMax includes these query parameters:`sow`::-Split on whitespace. If set to `true`, text analysis is invoked separately for each individual whitespace-separated term. The default is `false`; whitespace-separated term sequences will be provided to text analysis in one shot, enabling proper function of analysis filters that operate over term sequences, e.g., multi-word synonyms and shingles.+Split on whitespace.+If set to `true`, text analysis is invoked separately for each individual whitespace-separated term.+The default is `false`; whitespace-separated term sequences will be provided to text analysis in one shot, enabling proper function of analysis filters that operate over term sequences, e.g., multi-word synonyms and shingles.`mm`::- Minimum should match. See the <<the-dismax-query-parser.adoc#mm-minimum-should-match-parameter,DisMax mm parameter>> for a description of `mm`. The default eDisMax `mm` value differs from that of DisMax:+Minimum should match.+See the <<dismax-query-parser.adoc#mm-minimum-should-match-parameter,DisMax mm parameter>> for a description of `mm`.+The default eDisMax `mm` value differs from that of DisMax:+* The default `mm` value is 0%:** if the query contains an explicit operator other than "AND" ("-", "+", "OR", "NOT"); or@@ -47,12 +52,14 @@ Split on whitespace. If set to `true`, text analysis is invoked separately for e* The default `mm` value is 100% if `q.op` is "AND" and the query does not contain any explicit operators other than "AND".`mm.autoRelax`::-If `true`, the number of clauses required (<<the-dismax-query-parser.adoc#mm-minimum-should-match-parameter,minimum should match>>) will automatically be relaxed if a clause is removed (by e.g., stopwords filter) from some but not all <<the-dismax-query-parser.adoc#qf-query-fields-parameter,`qf`>> fields. Use this parameter as a workaround if you experience that queries return zero hits due to uneven stopword removal between the `qf` fields.+If `true`, the number of clauses required (<<dismax-query-parser.adoc#mm-minimum-should-match-parameter,minimum should match>>) will automatically be relaxed if a clause is removed (by e.g., stopwords filter) from some but not all <<dismax-query-parser.adoc#qf-query-fields-parameter,`qf`>> fields.+Use this parameter as a workaround if you experience that queries return zero hits due to uneven stopword removal between the `qf` fields.+Note that relaxing `mm` may cause undesired side effects, such as hurting the precision of the search, depending on the nature of your index content.`boost`::-A multivalued list of strings parsed as <<function-queries.adoc#available-functions,functions>> whose results will be multiplied into the score from the main query for all matching documents. This parameter is shorthand for wrapping the query produced by eDisMax using the <<other-parsers.adoc#boost-query-parser,`BoostQParserPlugin`>>.+A multivalued list of strings parsed as <<function-queries.adoc#available-functions,functions>> whose results will be multiplied into the score from the main query for all matching documents.+This parameter is shorthand for wrapping the query produced by eDisMax using the <<other-parsers.adoc#boost-query-parser,`BoostQParserPlugin`>>.These two examples are equivalent:[source,text]@@ -71,27 +78,32 @@ A Boolean parameter indicating if lowercase "and" and "or" should be treated theDefaults to `false`.`ps`::-Phrase Slop. The default amount of slop - distance between terms - on phrase queries built with `pf`, `pf2` and/or `pf3` fields (affects boosting). See also the section <<Using 'Slop'>> below.+Phrase Slop.+The default amount of slop - distance between terms - on phrase queries built with `pf`, `pf2` and/or `pf3` fields (affects boosting). See also the section <<Using 'Slop'>> below.`pf2`::-A multivalued list of fields with optional weights. Similar to `pf`, but based on word _pair_ shingles.+A multivalued list of fields with optional weights.+Similar to `pf`, but based on word _pair_ shingles.`ps2`::This is similar to `ps` but overrides the slop factor used for `pf2`. If not specified, `ps` is used.`pf3`::-A multivalued list of fields with optional weights, based on triplets of word shingles. Similar to `pf`, except that instead of building a phrase per field out of all the words in the input, it builds a set of phrases for each field out of word _triplet_ shingles.+A multivalued list of fields with optional weights, based on triplets of word shingles.+Similar to `pf`, except that instead of building a phrase per field out of all the words in the input, it builds a set of phrases for each field out of word _triplet_ shingles.`ps3`::This is similar to `ps` but overrides the slop factor used for `pf3`. If not specified, `ps` is used.`stopwords`::-A Boolean parameter indicating if the `StopFilterFactory` configured in the query analyzer should be respected when parsing the query. If this is set to `false`, then the `StopFilterFactory` in the query analyzer is ignored.+A Boolean parameter indicating if the `StopFilterFactory` configured in the query analyzer should be respected when parsing the query.+If this is set to `false`, then the `StopFilterFactory` in the query analyzer is ignored.`uf`::Specifies which schema fields the end user is allowed to explicitly query and to toggle whether embedded Solr queries are supported.-This parameter supports wildcards. Multiple fields must be separated by a space.+This parameter supports wildcards.+Multiple fields must be separated by a space.+The default is to allow all fields and no embedded Solr queries, equivalent to `uf=* -\_query_`.@@ -104,9 +116,10 @@ The default is to allow all fields and no embedded Solr queries, equivalent to `=== Field Aliasing using Per-Field qf Overrides-Per-field overrides of the `qf` parameter may be specified to provide 1-to-many aliasing from field names specified in the query string, to field names used in the underlying query. By default, no aliasing is used and field names specified in the query string are treated as literal field names in the index.+Per-field overrides of the `qf` parameter may be specified to provide 1-to-many aliasing from field names specified in the query string, to field names used in the underlying query.+By default, no aliasing is used and field names specified in the query string are treated as literal field names in the index.-== Examples of eDismax Queries+== Examples of eDisMax QueriesAll of the sample URLs in this section assume you are running Solr's "```techproducts```" example:@@ -164,7 +177,9 @@ f.name.qf=last_name first_name== Using 'Slop'-`Dismax` and `Edismax` can run queries against all query fields, and also run a query in the form of a phrase against the phrase fields. (This will work only for boosting documents, not actually for matching.) However, that phrase query can have a 'slop,' which is the distance between the terms of the query while still considering it a phrase match. For example:+`Dismax` and `Edismax` can run queries against all query fields, and also run a query in the form of a phrase against the phrase fields (this will work only for boosting documents, not actually for matching).+However, that phrase query can have a 'slop,' which is the distance between the terms of the query while still considering it a phrase match.+For example:[source,text]----@@ -174,7 +189,7 @@ pf=field1^50 field2^20defType=dismax-----With these parameters, the Dismax Query Parser generates a query that looks something like this:+With these parameters, the DisMax Query Parser generates a query that looks something like this:[source,text]----@@ -197,7 +212,8 @@ If you add the parameter `ps` (phrase slop), the second query will instead be:ps=10 field1:"foo bar"~10^50 OR field2:"foo bar"~10^20-----This means that if the terms "foo" and "bar" appear in the document with less than 10 terms between each other, the phrase will match. For example the doc that says:+This means that if the terms "foo" and "bar" appear in the document with less than 10 terms between each other, the phrase will match.+For example the doc that says:[source,text]----@@ -208,20 +224,25 @@ will match the phrase query.How does one use phrase slop? Usually it is configured in the request handler (in `solrconfig`).-With query slop (`qs`) the concept is similar, but it applies to explicit phrase queries from the user. For example, if you want to search for a name, you could enter:+With query slop (`qs`) the concept is similar, but it applies to explicit phrase queries from the user.+For example, if you want to search for a name, you could enter:[source,text]----q="Hans Anderson"-----A document that contains "Hans Anderson" will match, but a document that contains the middle name "Christian" or where the name is written with the last name first ("Anderson, Hans") won't. For those cases one could configure the query field `qs`, so that even if the user searches for an explicit phrase query, a slop is applied.+A document that contains "Hans Anderson" will match, but a document that contains the middle name "Christian" or where the name is written with the last name first ("Anderson, Hans") won't.+For those cases one could configure the query field `qs`, so that even if the user searches for an explicit phrase query, a slop is applied.-Finally, in addition to the phrase fields (`pf`) parameter, `edismax` also supports the `pf2` and `pf3` parameters, for fields over which to create bigram and trigram phrase queries. The phrase slop for these parameters' queries can be specified using the `ps2` and `ps3` parameters, respectively. If you use `pf2`/`pf3` but not `ps2`/`ps3`, then the phrase slop for these parameters' queries will be taken from the `ps` parameter, if any.+Finally, in addition to the phrase fields (`pf`) parameter, `edismax` also supports the `pf2` and `pf3` parameters, for fields over which to create bigram and trigram phrase queries.+The phrase slop for these parameters' queries can be specified using the `ps2` and `ps3` parameters, respectively.+If you use `pf2`/`pf3` but not `ps2`/`ps3`, then the phrase slop for these parameters' queries will be taken from the `ps` parameter, if any.=== Synonyms Expansion in Phrase Queries with Slop-When a phrase query with slop (e.g., `pf` with `ps`) triggers synonym expansions, a separate clause will be generated for each combination of synonyms. For example, with configured synonyms `dog,canine` and `cat,feline`, the query `"dog chased cat"` will generate the following phrase query clauses:+When a phrase query with slop (e.g., `pf` with `ps`) triggers synonym expansions, a separate clause will be generated for each combination of synonyms.+For example, with configured synonyms `dog,canine` and `cat,feline`, the query `"dog chased cat"` will generate the following phrase query clauses:* `"dog chased cat"`* `"canine chased cat"`diff --git a/solr/solr-ref-guide/src/enabling-ssl.adoc b/solr/solr-ref-guide/src/enabling-ssl.adocindex 95e9cd82242..b2de1a2ef4a 100644--- a/solr/solr-ref-guide/src/enabling-ssl.adoc+++ b/solr/solr-ref-guide/src/enabling-ssl.adoc@@ -31,7 +31,8 @@ This keystore will also be used as a truststore below.It's possible to use the keystore that comes with the JDK for these purposes, and to use a separate truststore, but those options aren't covered here.Run the commands below in the `server/etc/` directory in the binary Solr distribution.-It's assumed that you have the JDK `keytool` utility on your `PATH`, and that `openssl` is also on your `PATH`. See https://www.openssl.org/related/binaries.html for OpenSSL binaries for Windows and Solaris.+It's assumed that you have the JDK `keytool` utility on your `PATH`, and that `openssl` is also on your `PATH`.+See https://www.openssl.org/related/binaries.html for OpenSSL binaries for Windows and Solaris.The `-ext SAN=...` `keytool` option allows you to specify all the DNS names and/or IP addresses that will be allowed during hostname verification if you choose to require it.@@ -120,11 +121,13 @@ set SOLR_SSL_CHECK_PEER_NAME=true--.Client Authentication Settings-WARNING: Enable either `SOLR_SSL_NEED_CLIENT_AUTH` or `SOLR_SSL_WANT_CLIENT_AUTH` but not both at the same time. They are mutually exclusive and Jetty will select one of them which may not be what you expect. `SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION` should be set to false if you want to disable hostname verification.+WARNING: Enable either `SOLR_SSL_NEED_CLIENT_AUTH` or `SOLR_SSL_WANT_CLIENT_AUTH` but not both at the same time.+They are mutually exclusive and Jetty will select one of them which may not be what you expect.+`SOLR_SSL_CLIENT_HOSTNAME_VERIFICATION` should be set to false if you want to disable hostname verification.When you start Solr, the `bin/solr` script includes these settings and will pass them as system properties to the JVM.-If you are running Solr in standalone mode, you can skip to <<Run Single Node Solr using SSL>>.+If you are running Solr in a user-managed cluster or single-node installation, you can skip to <<Start User-Managed Cluster or Single-Node Solr>>.If you are using SolrCloud, however, you need to <<Configure ZooKeeper>> before starting Solr.@@ -155,7 +158,8 @@ Note that if the `javax.net.ssl.\*` configurations are not set, they will fallbaSolr requires three parameters to be configured in order to use the credential store file for keystore passwords.`solr.ssl.credential.provider.chain`::-The credential provider chain. This should be set to `hadoop`.+The credential provider chain.+This should be set to `hadoop`.`SOLR_HADOOP_CREDENTIAL_PROVIDER_PATH`::The path to the credential store file.@@ -190,12 +194,13 @@ set HADOOP_CREDSTORE_PASSWORD="credStorePass123"=== Configure ZooKeeper-NOTE: ZooKeeper does not support encrypted communication with clients like Solr. There are several related JIRA tickets where SSL support is being planned/worked on: https://issues.apache.org/jira/browse/ZOOKEEPER-235[ZOOKEEPER-235]; https://issues.apache.org/jira/browse/ZOOKEEPER-236[ZOOKEEPER-236]; https://issues.apache.org/jira/browse/ZOOKEEPER-1000[ZOOKEEPER-1000]; and https://issues.apache.org/jira/browse/ZOOKEEPER-2120[ZOOKEEPER-2120].+NOTE: ZooKeeper does not support encrypted communication with clients like Solr.+There are several related JIRA tickets where SSL support is being planned/worked on: https://issues.apache.org/jira/browse/ZOOKEEPER-235[ZOOKEEPER-235]; https://issues.apache.org/jira/browse/ZOOKEEPER-236[ZOOKEEPER-236]; https://issues.apache.org/jira/browse/ZOOKEEPER-1000[ZOOKEEPER-1000]; and https://issues.apache.org/jira/browse/ZOOKEEPER-2120[ZOOKEEPER-2120].After creating the keystore described above and before you start any SolrCloud nodes, you must configure your Solr cluster properties in ZooKeeper so that Solr nodes know to communicate via SSL.This section assumes you have created and started an external ZooKeeper.-See <<setting-up-an-external-zookeeper-ensemble.adoc#,Setting Up an External ZooKeeper Ensemble>> for more information.+See <<zookeeper-ensemble.adoc#,ZooKeeper Ensemble>> for more information.The `urlScheme` cluster-wide property needs to be set to `https` before any Solr node starts up.The examples below use the `zkcli` tool that comes with Solr to do this.@@ -221,7 +226,8 @@ C:\> server\scripts\cloud-scripts\zkcli.bat -zkhost server1:2181,server2:2181,se=====---Be sure to use the correct `zkhost` value for your system. If you have set up your ZooKeeper ensemble to use a <<taking-solr-to-production.adoc#zookeeper-chroot,chroot for Solr>>, make sure to include it in the `zkhost` string, e.g., `-zkhost server1:2181,server2:2181,server3:2181/solr`.+Be sure to use the correct `zkhost` value for your system.+If you have set up your ZooKeeper ensemble to use a <<taking-solr-to-production.adoc#zookeeper-chroot,chroot for Solr>>, make sure to include it in the `zkhost` string, e.g., `-zkhost server1:2181,server2:2181,server3:2181/solr`.=== Update Cluster Properties for Existing Collections@@ -240,7 +246,7 @@ Once this and all other steps are complete, you can go ahead and start Solr.== Starting Solr After Enabling SSL-=== Run Single Node Solr using SSL+=== Start User-Managed Cluster or Single-Node SolrStart Solr using the Solr control script as shown in the examples below.Customize the values for the parameters shown as needed and add any used in your system.@@ -266,11 +272,12 @@ C:\> bin\solr.cmd -p 8984====---=== Run SolrCloud with SSL+=== Start SolrCloud-NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from all of the `bin/solr`/`bin\solr.cmd` commands below.+NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from all of the `bin/solr`/`bin\solr.cmd` commands below.-Start each Solr node with the Solr control script as shown in the examples below. Customize the values for the parameters shown as necessary and add any used in your system.+Start each Solr node with the Solr control script as shown in the examples below.+Customize the values for the parameters shown as necessary and add any used in your system.If you created the SSL key without all DNS names or IP addresses on which Solr nodes run, you can tell Solr to skip hostname verification for inter-node communications by setting the `-Dsolr.ssl.checkPeerName=false` system property.@@ -300,9 +307,12 @@ C:\> bin\solr.cmd -cloud -s cloud\node1 -z server1:2181,server2:2181,server3:218[IMPORTANT]====-curl on OS X Mavericks (10.9) has degraded SSL support. For more information and workarounds to allow one-way SSL, see https://curl.se/mail/archive-2013-10/0036.html. curl on OS X Yosemite (10.10) is improved - 2-way SSL is possible - see https://curl.se/mail/archive-2014-10/0053.html.+curl on OS X Mavericks (10.9) has degraded SSL support.+For more information and workarounds to allow one-way SSL, see https://curl.se/mail/archive-2013-10/0036.html.+curl on OS X Yosemite (10.10) is improved - 2-way SSL is possible - see https://curl.se/mail/archive-2014-10/0053.html.-The curl commands in the following sections will not work with the system `curl` on OS X Yosemite (10.10). Instead, the certificate supplied with the `-E` parameter must be in PKCS12 format, and the file supplied with the `--cacert` parameter must contain only the CA certificate, and no key (see <<Convert the Certificate and Key to PEM Format for Use with curl,above>> for instructions on creating this file):+The curl commands in the following sections will not work with the system `curl` on OS X Yosemite (10.10).+Instead, the certificate supplied with the `-E` parameter must be in PKCS12 format, and the file supplied with the `--cacert` parameter must contain only the CA certificate, and no key (see <<Convert the Certificate and Key to PEM Format for Use with curl,above>> for instructions on creating this file):[source,bash]$ curl -E solr-ssl.keystore.p12:secret --cacert solr-ssl.cacert.pem ...@@ -379,14 +389,15 @@ Use `post.jar` to index some example documents to the SolrCloud collection creat[source,bash]-----cd example/exampledocs+$ cd example/exampledocs-java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml+$ java -Djavax.net.ssl.keyStorePassword=secret -Djavax.net.ssl.keyStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStore=../../server/etc/solr-ssl.keystore.p12 -Djavax.net.ssl.trustStorePassword=secret -Durl=https://localhost:8984/solr/mycollection/update -jar post.jar *.xml----=== Query Using curl-Use curl to query the SolrCloud collection created above, from a directory containing the PEM formatted certificate and key created above (e.g., `example/etc/`) - if you have not enabled client authentication (system property `-Djetty.ssl.clientAuth=true)`, then you can remove the `-E solr-ssl.pem:secret` option:+Use curl to query the SolrCloud collection created above, from a directory containing the PEM formatted certificate and key created above (e.g., `example/etc/`).+If you have not enabled client authentication (system property `-Djetty.ssl.clientAuth=true)`, then you can remove the `-E solr-ssl.pem:secret` option:[source,bash]----@@ -395,7 +406,8 @@ curl -E solr-ssl.pem:secret --cacert solr-ssl.pem "https://localhost:8984/solr/m=== Index a Document using CloudSolrClient-From a java client using SolrJ, index a document. In the code below, the `javax.net.ssl.*` system properties are set programmatically, but you could instead specify them on the java command line, as in the `post.jar` example above:+From a java client using SolrJ, index a document.+In the code below, the `javax.net.ssl.*` system properties are set programmatically, but you could instead specify them on the java command line, as in the `post.jar` example above:[source,java]----diff --git a/solr/solr-ref-guide/src/enhancing-queries.adoc b/solr/solr-ref-guide/src/enhancing-queries.adocnew file mode 100644index 00000000000..7d34db18ec4--- /dev/null+++ b/solr/solr-ref-guide/src/enhancing-queries.adoc@@ -0,0 +1,58 @@+= Enhancing Queries+:page-children: spell-checking, \+ suggester, \+ morelikethis, \+ query-re-ranking, \+ learning-to-rank, \+ tagger-handler, \+ analytics, \+ terms-component, \+ term-vector-component, \+ stats-component+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Solr provides many options for assisting users with their queries.++These options allow you to show users alternate spellings for their search terms, or provide suggestions for terms while they type.++Re-ranking provides an ability to show documents in an order based on a query that may be more complex than the user's query.+This forms the basis of Solr's Learning to Rank functionality, which can re-rank documents based on a machine-learned model.++The Tagger request handler provides basic named entity recognition functionality.++Finally, if you want to understand the terms in your index or get statistics from terms in the index, those options are covered here also.++****+// This tags the below list so it can be used in the parent page section list+// tag::queries-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<spell-checking.adoc#,Spell Checking>>: Check user spelling of query terms.+| <<suggester.adoc#,Suggester>>: Suggest query terms while the user types.+| <<morelikethis.adoc#,MoreLikeThis>>: Get results similar to result documents.+| <<query-re-ranking.adoc#,Query Re-Ranking>>: Re-rank top documents.+| <<learning-to-rank.adoc#,Learning to Rank>>: Use machine learned ranking models.+| <<tagger-handler.adoc#,Tagger Handler>>: Basic named entity tagging in text.+| <<analytics.adoc#,Analytics Component>>: Compute complex analytics over a result set.+| <<terms-component.adoc#,Terms Component>>: Access indexed terms and the documents that include them.+| <<term-vector-component.adoc#,Term Vector Component>>: Term information about specific documents.+| <<stats-component.adoc#,Stats Component>>: Get information from numeric fields within a document set.+|===+// end::queries-sections[]+****diff --git a/solr/solr-ref-guide/src/working-with-enum-fields.adoc b/solr/solr-ref-guide/src/enum-fields.adocsimilarity index 85%rename from solr/solr-ref-guide/src/working-with-enum-fields.adocrename to solr/solr-ref-guide/src/enum-fields.adocindex 8df25524a47..9e914b311c3 100644--- a/solr/solr-ref-guide/src/working-with-enum-fields.adoc+++ b/solr/solr-ref-guide/src/enum-fields.adoc@@ -1,4 +1,4 @@-= Working with Enum Fields+= Enum Fields// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,8 @@// specific language governing permissions and limitations// under the License.-EnumFieldType allows defining a field whose values are a closed set, and the sort order is pre-determined but is not alphabetic nor numeric. Examples of this are severity lists, or risk definitions.+The EnumFieldType allows defining a field whose values are a closed set, and the sort order is pre-determined but is not alphabetic nor numeric.+Examples of this are severity lists, or risk definitions..EnumField has been Deprecated[WARNING]@@ -36,7 +37,8 @@ The EnumFieldType type definition is quite simple, as in this example defining fBesides the `name` and the `class`, which are common to all field types, this type also takes two additional parameters:-`enumsConfig`:: the name of a configuration file that contains the `<enum/>` list of field values and their order that you wish to use with this field type. If a path to the file is not defined specified, the file should be in the `conf` directory for the collection.+`enumsConfig`:: the name of a configuration file that contains the `<enum/>` list of field values and their order that you wish to use with this field type.+If a path to the file is not defined specified, the file should be in the `conf` directory for the collection.`enumName`:: the name of the specific enumeration in the `enumsConfig` file to use for this type.Note that `docValues="true"` must be specified either in the EnumFieldType fieldType or field specification.@@ -45,7 +47,8 @@ Note that `docValues="true"` must be specified either in the EnumFieldType fieldThe file named with the `enumsConfig` parameter can contain multiple enumeration value lists with different names if there are multiple uses for enumerations in your Solr schema.-In this example, there are two value lists defined. Each list is between `enum` opening and closing tags:+In this example, there are two value lists defined.+Each list is between `enum` opening and closing tags:[source,xml]----diff --git a/solr/solr-ref-guide/src/errata.adoc b/solr/solr-ref-guide/src/errata.adocdeleted file mode 100644index 3076af3543e..00000000000--- a/solr/solr-ref-guide/src/errata.adoc+++ /dev/null@@ -1,25 +0,0 @@-= Errata-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--== Errata For This Documentation--Any mistakes found in this documentation after its release will be listed below.--=== Errata For Past Versions of This Documentation--Any known mistakes in past releases of this documentation will be noted below.diff --git a/solr/solr-ref-guide/src/exporting-result-sets.adoc b/solr/solr-ref-guide/src/exporting-result-sets.adocindex a4da8234f01..3c5e3a78099 100644--- a/solr/solr-ref-guide/src/exporting-result-sets.adoc+++ b/solr/solr-ref-guide/src/exporting-result-sets.adoc@@ -16,20 +16,21 @@// specific language governing permissions and limitations// under the License.--It's possible to export fully sorted result sets using a special <<query-re-ranking.adoc#,rank query parser>> and <<response-writers.adoc#,response writer>> specifically designed to work together to handle scenarios that involve sorting and exporting millions of records.+The `/export` request handler allows a fully sorted result set to be streamed out of Solr using a special <<query-re-ranking.adoc#,rank query parser>> and <<response-writers.adoc#,response writer>>.+These have been specifically designed to work together to handle scenarios that involve sorting and exporting millions of records.This feature uses a stream sorting technique that begins to send records within milliseconds and continues to stream results until the entire result set has been sorted and exported.-The cases where this functionality may be useful include: session analysis, distributed merge joins, time series roll-ups, aggregations on high cardinality fields, fully distributed field collapsing, and sort based stats.+The cases where this functionality may be useful include: session analysis, distributed merge joins, time series roll-ups, aggregations on high cardinality fields, fully distributed field collapsing, and sort-based stats.== Field Requirements-All the fields being sorted and exported must have docValues set to true. For more information, see the section on <<docvalues.adoc#,DocValues>>.+All the fields being sorted and exported must have docValues set to `true`.+For more information, see the section on <<docvalues.adoc#,DocValues>>.== The /export RequestHandler-The `/export` request handler with the appropriate configuration is one of Solr's out-of-the-box request handlers - see <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>> for more information.+The `/export` request handler with the appropriate configuration is one of Solr's out-of-the-box request handlers - see <<implicit-requesthandlers.adoc#,Implicit Request Handlers>> for more information.Note that this request handler's properties are defined as "invariants", which means they cannot be overridden by other properties passed at another time (such as at query time).@@ -37,11 +38,15 @@ Note that this request handler's properties are defined as "invariants", which mYou can use `/export` to make requests to export the result set of a query.-All queries must include `sort` and `fl` parameters, or the query will return an error. Filter queries are also supported.+All queries must include `sort` and `fl` parameters, or the query will return an error.+Filter queries are also supported.-An optional parameter `batchSize` determines the size of the internal buffers for partial results. The default value is `30000` but users may want to specify smaller values to limit the memory use (at the cost of degraded performance) or higher values to improve export performance (the relationship is not linear and larger values don't bring proportionally larger performance increases).+An optional parameter `batchSize` determines the size of the internal buffers for partial results.+The default value is `30000` but users may want to specify smaller values to limit the memory use (at the cost of degraded performance) or higher values to improve export performance (the relationship is not linear and larger values don't bring proportionally larger performance increases).-The supported response writers are `json` and `javabin`. For backward compatibility reasons `wt=xsort` is also supported as input, but `wt=xsort` behaves same as `wt=json`. The default output format is `json`.+The supported response writers are `json` and `javabin`.+For backward compatibility reasons `wt=xsort` is also supported as input, but `wt=xsort` behaves same as `wt=json`.+The default output format is `json`.Here is an example of an export request of some indexed log data:@@ -52,20 +57,28 @@ http://localhost:8983/solr/core_name/export?q=my-query&sort=severity+desc,timest=== Specifying the Sort Criteria-The `sort` property defines how documents will be sorted in the exported result set. Results can be sorted by any field that has a field type of int,long, float, double, string. The sort fields must be single valued fields.+The `sort` property defines how documents will be sorted in the exported result set.+Results can be sorted by any field that has a field type of int,long, float, double, string.+The sort fields must be single valued fields.-The export performance will get slower as you add more sort fields. If there is enough physical memory available outside of the JVM to load up the sort fields then the performance will be linearly slower with addition of sort fields.+The export performance will get slower as you add more sort fields.+If there is enough physical memory available outside of the JVM to load up the sort fields then the performance will be linearly slower with addition of sort fields.It can get worse otherwise.=== Specifying the Field List-The `fl` property defines the fields that will be exported with the result set. Any of the field types that can be sorted (i.e., int, long, float, double, string, date, boolean) can be used in the field list. The fields can be single or multi-valued. However, returning scores and wildcards are not supported at this time.+The `fl` property defines the fields that will be exported with the result set.+Any of the field types that can be sorted (i.e., int, long, float, double, string, date, boolean) can be used in the field list.+The fields can be single or multi-valued.+However, returning scores and wildcards are not supported at this time.=== Specifying the Local Streaming ExpressionThe optional `expr` property defines a <<streaming-expressions.adoc#,stream expression>> that allows documents to be processed locally before they are exported in the result set.-Expressions have to use a special `input()` stream that represents original results from the `/export` handler. Output from the stream expression then becomes the output from the `/export` handler. The `&streamLocalOnly=true` flag is always set for this streaming expression.+Expressions have to use a special `input()` stream that represents original results from the `/export` handler.+Output from the stream expression then becomes the output from the `/export` handler.+The `&streamLocalOnly=true` flag is always set for this streaming expression.Only stream <<stream-decorator-reference.adoc#,decorators>> and <<stream-evaluator-reference.adoc#,evaluators>> are supported in these expressions - using any of the <<stream-source-reference.adoc#,source>> expressions except for the pre-defined `input()` will result in an error.diff --git a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc b/solr/solr-ref-guide/src/external-files-processes.adocsimilarity index 72%rename from solr/solr-ref-guide/src/working-with-external-files-and-processes.adocrename to solr/solr-ref-guide/src/external-files-processes.adocindex 49dacce226f..7f44ee4f4d3 100644--- a/solr/solr-ref-guide/src/working-with-external-files-and-processes.adoc+++ b/solr/solr-ref-guide/src/external-files-processes.adoc@@ -1,4 +1,4 @@-= Working with External Files and Processes+= External Files and Processes// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,16 +16,27 @@// specific language governing permissions and limitations// under the License.+Solr supports storing field values in an external file with a field type called the `ExternalFileFieldType`.+It can also consume a stream of tokens that have already undergone analysis with a field type called the `PreAnalyzedFieldType`.+== The ExternalFileField Type-The `ExternalFileField` type makes it possible to specify the values for a field in a file outside the Solr index. For such a field, the file contains mappings from a key field to the field value. Another way to think of this is that, instead of specifying the field in documents as they are indexed, Solr finds values for this field in the external file.+The `ExternalFileField` type makes it possible to specify the values for a field in a file outside the Solr index.+For such a field, the file contains mappings from a key field to the field value.+Another way to think of this is that, instead of specifying the field in documents as they are indexed, Solr finds values for this field in the external file.[IMPORTANT]====-External fields are not searchable. They can be used only for function queries or display. For more information on function queries, see the section on <<function-queries.adoc#,Function Queries>>.+External fields are not searchable.+They can be used only for function queries or display.+For more information on function queries, see the section on <<function-queries.adoc#,Function Queries>>.====-The `ExternalFileField` type is handy for cases where you want to update a particular field in many documents more often than you want to update the rest of the documents. For example, suppose you have implemented a document rank based on the number of views. You might want to update the rank of all the documents daily or hourly, while the rest of the contents of the documents might be updated much less frequently. Without `ExternalFileField`, you would need to update each document just to change the rank. Using `ExternalFileField` is much more efficient because all document values for a particular field are stored in an external file that can be updated as frequently as you wish.+The `ExternalFileField` type is handy for cases where you want to update a particular field in many documents more often than you want to update the rest of the documents.+For example, suppose you have implemented a document rank based on the number of views.+You might want to update the rank of all the documents daily or hourly, while the rest of the contents of the documents might be updated much less frequently.+Without `ExternalFileField`, you would need to update each document just to change the rank.+Using `ExternalFileField` is much more efficient because all document values for a particular field are stored in an external file that can be updated as frequently as you wish.In `schema.xml`, the definition of this field type might look like this:@@ -34,18 +45,23 @@ In `schema.xml`, the definition of this field type might look like this:<fieldType name="entryRankFile" keyField="pkId" defVal="0" stored="false" indexed="false" class="solr.ExternalFileField"/>-----The `keyField` attribute defines the key that will be defined in the external file. It is usually the unique key for the index, but it doesn't need to be as long as the `keyField` can be used to identify documents in the index. A `defVal` defines a default value that will be used if there is no entry in the external file for a particular document.+The `keyField` attribute defines the key that will be defined in the external file.+It is usually the unique key for the index, but it doesn't need to be as long as the `keyField` can be used to identify documents in the index.+A `defVal` defines a default value that will be used if there is no entry in the external file for a particular document.=== Format of the External File-The file itself is located in Solr's index directory, which by default is `$SOLR_HOME/data`. The name of the file should be `external_fieldname_` or `external_fieldname_.*`. For the example above, then, the file could be named `external_entryRankFile` or `external_entryRankFile.txt`.+The file itself is located in Solr's index directory, which by default is `$SOLR_HOME/data`.+The name of the file should be `external_fieldname_` or `external_fieldname_.*`. For the example above, then, the file could be named `external_entryRankFile` or `external_entryRankFile.txt`.[TIP]====-If any files using the name pattern `.*` (such as `.txt`) appear, the last (after being sorted by name) will be used and previous versions will be deleted. This behavior supports implementations on systems where one may not be able to overwrite a file (for example, on Windows, if the file is in use).+If any files using the name pattern `.*` (such as `.txt`) appear, the last (after being sorted by name) will be used and previous versions will be deleted.+This behavior supports implementations on systems where one may not be able to overwrite a file (for example, on Windows, if the file is in use).====-The file contains entries that map a key field, on the left of the equals sign, to a value, on the right. Here are a few example entries:+The file contains entries that map a key field, on the left of the equals sign, to a value, on the right.+Here are a few example entries:[source,text]----@@ -54,11 +70,13 @@ doc34=3.14159doc40=42-----The keys listed in this file do not need to be unique. The file does not need to be sorted, but Solr will be able to perform the lookup faster if it is.+The keys listed in this file do not need to be unique.+The file does not need to be sorted, but Solr will be able to perform the lookup faster if it is.=== Reloading an External File-It's possible to define an event listener to reload an external file when either a searcher is reloaded or when a new searcher is started. See the section <<query-settings-in-solrconfig.adoc#query-related-listeners,Query-Related Listeners>> for more information, but a sample definition in `solrconfig.xml` might look like this:+It's possible to define an event listener to reload an external file when either a searcher is reloaded or when a new searcher is started.+See the section <<caches-warming.adoc#query-related-listeners,Query-Related Listeners>> for more information, but a sample definition in `solrconfig.xml` might look like this:[source,xml]----@@ -68,16 +86,23 @@ It's possible to define an event listener to reload an external file when either== The PreAnalyzedField Type-The `PreAnalyzedField` type provides a way to send to Solr serialized token streams, optionally with independent stored values of a field, and have this information stored and indexed without any additional text processing applied in Solr. This is useful if user wants to submit field content that was already processed by some existing external text processing pipeline (e.g., it has been tokenized, annotated, stemmed, synonyms inserted, etc.), while using all the rich attributes that Lucene's TokenStream provides (per-token attributes).+The `PreAnalyzedField` type provides a way to send to Solr serialized token streams, optionally with independent stored values of a field, and have this information stored and indexed without any additional text processing applied in Solr.+This is useful if user wants to submit field content that was already processed by some existing external text processing pipeline (e.g., it has been tokenized, annotated, stemmed, synonyms inserted, etc.), while using all the rich attributes that Lucene's TokenStream provides (per-token attributes).-The serialization format is pluggable using implementations of PreAnalyzedParser interface. There are two out-of-the-box implementations:+The serialization format is pluggable using implementations of PreAnalyzedParser interface.+There are two out-of-the-box implementations:-* <<JsonPreAnalyzedParser>>: as the name suggests, it parses content that uses JSON to represent field's content. This is the default parser to use if the field type is not configured otherwise.+* <<JsonPreAnalyzedParser>>: as the name suggests, it parses content that uses JSON to represent field's content.+This is the default parser to use if the field type is not configured otherwise.* <<SimplePreAnalyzedParser>>: uses a simple strict plain text format, which in some situations may be easier to create than JSON.-There is only one configuration parameter, `parserImpl`. The value of this parameter should be a fully qualified class name of a class that implements PreAnalyzedParser interface. The default value of this parameter is `org.apache.solr.schema.JsonPreAnalyzedParser`.+There is only one configuration parameter, `parserImpl`.+The value of this parameter should be a fully qualified class name of a class that implements PreAnalyzedParser interface.+The default value of this parameter is `org.apache.solr.schema.JsonPreAnalyzedParser`.-By default, the query-time analyzer for fields of this type will be the same as the index-time analyzer, which expects serialized pre-analyzed text. You must add a query type analyzer to your fieldType in order to perform analysis on non-pre-analyzed queries. In the example below, the index-time analyzer expects the default JSON serialization format, and the query-time analyzer will employ StandardTokenizer/LowerCaseFilter:+By default, the query-time analyzer for fields of this type will be the same as the index-time analyzer, which expects serialized pre-analyzed text.+You must add a query type analyzer to your fieldType in order to perform analysis on non-pre-analyzed queries.+In the example below, the index-time analyzer expects the default JSON serialization format, and the query-time analyzer will employ StandardTokenizer/LowerCaseFilter:[source,xml]----@@ -91,11 +116,10 @@ By default, the query-time analyzer for fields of this type will be the same as=== JsonPreAnalyzedParser-This is the default serialization format used by PreAnalyzedField type. It uses a top-level JSON map with the following keys:--// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed+This is the default serialization format used by PreAnalyzedField type.+It uses a top-level JSON map with the following keys:-[cols="20,60,20",options="header"]+[%autowidth.stretch,options="header"]|===|Key |Description |Required|`v` |Version key. Currently the supported version is `1`. |required@@ -108,11 +132,10 @@ Any other top-level key is silently ignored.==== Token Stream Serialization-The token stream is expressed as a JSON list of JSON maps. The map for each token consists of the following keys and values:+The token stream is expressed as a JSON list of JSON maps.+The map for each token consists of the following keys and values:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="10,20,20,30,20",options="header"]+[%autowidth.stretch,options="header"]|===|Key |Description |Lucene Attribute |Value |Required?|`t` |token |{lucene-javadocs}/core/org/apache/lucene/analysis/tokenattributes/CharTermAttribute.html[CharTermAttribute] |UTF-8 string representing the current token |required@@ -183,9 +206,7 @@ Please note that Unicode sequences (e.g., `\u0001`) are not supported.The following token attributes are supported, and identified with short symbolic names:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="10,30,30,30",options="header"]+[%autowidth.stretch,options="header"]|===|Name |Description |Lucene attribute |Value format|`i` |position increment |{lucene-javadocs}/core/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.html[PositionIncrementAttribute] |integerdiff --git a/solr/solr-ref-guide/src/faceting.adoc b/solr/solr-ref-guide/src/faceting.adocindex 258b1002b5c..81c78942e73 100644--- a/solr/solr-ref-guide/src/faceting.adoc+++ b/solr/solr-ref-guide/src/faceting.adoc@@ -47,7 +47,7 @@ Several parameters can be used to trigger faceting based on the indexed terms inWhen using these parameters, it is important to remember that "term" is a very specific concept in Lucene: it relates to the literal field/value pairs that are indexed after any analysis occurs. For text fields that include stemming, lowercasing, or word splitting, the resulting terms may not be what you expect.If you want Solr to perform both analysis (for searching) and faceting on the full literal strings, use the `copyField` directive in your Schema to create two versions of the field: one Text and one String. The Text field should have `indexed="true" docValues=“false"` if used for searching but not faceting and the String field should have `indexed="false" docValues="true"` if used for faceting but not searching.-(For more information about the `copyField` directive, see <<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>>.)+(For more information about the `copyField` directive, see <<fields-and-schema-design.adoc#,Fields and Schema Design>>.)Unless otherwise specified, all of the parameters below can be specified on a per-field basis with the syntax of `f.<fieldname>.facet.<parameter>`@@ -235,9 +235,9 @@ The default value for this parameter is `filter`..Date Ranges & Time Zones[NOTE]====-Range faceting on date fields is a common situation where the <<working-with-dates.adoc#tz,`TZ`>> parameter can be useful to ensure that the "facet counts per day" or "facet counts per month" are based on a meaningful definition of when a given day/month "starts" relative to a particular TimeZone.+Range faceting on date fields is a common situation where the <<date-formatting-math.adoc#tz,`TZ`>> parameter can be useful to ensure that the "facet counts per day" or "facet counts per month" are based on a meaningful definition of when a given day/month "starts" relative to a particular TimeZone.-For more information, see the examples in the <<working-with-dates.adoc#,Working with Dates>> section.+For more information, see the examples in the <<date-formatting-math.adoc#,Working with Dates>> section.======= facet.mincount in Range Faceting@@ -295,7 +295,7 @@ http://localhost:8983/solr/techproducts/select?q=*:*&facet.pivot=cat,popularity,=== Combining Stats Component With Pivots-In addition to some of the <<Local Parameters for Faceting,general local parameters>> supported by other types of faceting, a `stats` local parameters can be used with `facet.pivot` to refer to <<the-stats-component.adoc#,`stats.field`>> instances (by tag) that you would like to have computed for each Pivot Constraint.+In addition to some of the <<Local Params for Faceting,general local params>> supported by other types of faceting, a `stats` local params can be used with `facet.pivot` to refer to <<stats-component.adoc#,`stats.field`>> instances (by tag) that you would like to have computed for each Pivot Constraint.In the example below, two different (overlapping) sets of statistics are computed for each of the facet.pivot result hierarchies:@@ -573,9 +573,9 @@ Interval faceting supports output key replacement described below. Output keys c&facet=true-----== Local Parameters for Faceting+== Local Params for Faceting-The <<local-parameters-in-queries.adoc#,LocalParams syntax>> allows overriding global settings. It can also provide a method of adding metadata to other parameter values, much like XML attributes.+The <<local-params.adoc#,LocalParams syntax>> allows overriding global settings. It can also provide a method of adding metadata to other parameter values, much like XML attributes.=== Tagging and Excluding Filters@@ -602,7 +602,7 @@ To return counts for doctype values that are currently not selected, tag filters`q=mainquery&fq=status:public&fq={!tag=dt}doctype:pdf&facet=true&facet.field={!ex=dt}doctype`-Filter exclusion is supported for all types of facets. Both the `tag` and `ex` local parameters may specify multiple values by separating them with commas.+Filter exclusion is supported for all types of facets. Both the `tag` and `ex` local params may specify multiple values by separating them with commas.=== Changing the Output Keydiff --git a/solr/solr-ref-guide/src/field-properties-by-use-case.adoc b/solr/solr-ref-guide/src/field-properties-by-use-case.adocindex 4796f613e25..8425b64de33 100644--- a/solr/solr-ref-guide/src/field-properties-by-use-case.adoc+++ b/solr/solr-ref-guide/src/field-properties-by-use-case.adoc@@ -42,8 +42,8 @@ Notes:2. [[fpbuc_2,2]] Will be used if present, but not necessary.3. [[fpbuc_3,3]] (if termVectors=true)4. [[fpbuc_4,4]] A tokenizer must be defined for the field, but it doesn't need to be indexed.-5. [[fpbuc_5,5]] Described in <<understanding-analyzers-tokenizers-and-filters.adoc#,Understanding Analyzers, Tokenizers, and Filters>>.+5. [[fpbuc_5,5]] Described in <<document-analysis.adoc#,Document Analysis in Solr>>.6. [[fpbuc_6,6]] Term vectors are not mandatory here. If not true, then a stored field is analyzed. So term vectors are recommended, but only required if `stored=false`.7. [[fpbuc_7,7]] For most field types, either `indexed` or `docValues` must be true, but both are not required. <<docvalues.adoc#,DocValues>> can be more efficient in many cases. For `[Int/Long/Float/Double/Date]PointFields`, `docValues=true` is required.8. [[fpbuc_8,8]] Stored content will be used by default, but docValues can alternatively be used. See <<docvalues.adoc#,DocValues>>.-9. [[fpbuc_9,9]] Multi-valued sorting may be performed on docValues-enabled fields using the two-argument `field()` function, e.g., `field(myfield,min)`; see the <<function-queries.adoc#field-function,field() function in Function Queries>>.+9. [[fpbuc_9,9]] Multi-valued sorting may be performed on docValues-enabled fields using the two-argument `field()` function, e.g., `field(myfield,min)`; see the <<function-queries.adoc#field-function,field() function in Function Queries>>.diff --git a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adocindex 041ab7dab37..92a85eb8838 100644--- a/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc+++ b/solr/solr-ref-guide/src/field-type-definitions-and-properties.adoc@@ -50,7 +50,7 @@ Field types are defined in `schema.xml`. Each field type is defined between `fie----<1> The first line in the example above contains the field type name, `text_general`, and the name of the implementing class, `solr.TextField`.-<2> The rest of the definition is about field analysis, described in <<understanding-analyzers-tokenizers-and-filters.adoc#,Understanding Analyzers, Tokenizers, and Filters>>.+<2> The rest of the definition is about field analysis, described in <<document-analysis.adoc#,Document Analysis in Solr>>.The implementing class is responsible for making sure the field is handled correctly. In the class names in `schema.xml`, the string `solr` is shorthand for `org.apache.solr.schema` or `org.apache.solr.analysis`. Therefore, `solr.TextField` is really `org.apache.solr.schema.TextField`.@@ -95,9 +95,9 @@ Use `as_same_term` (default) to blend terms, i.e., `SynonymQuery(tshirt,tee)` wh`as_same_term` is appropriate when terms are true synonyms (television, tv). Use `pick_best` or `as_distinct_terms` when synonyms are expanding to hyponyms `(q=jeans w/ jeans=>jeans,pants)` and you want exact to come before parent and sibling concepts. See this http://opensourceconnections.com/blog/2017/11/21/solr-synonyms-mea-culpa/[blog article].`enableGraphQueries`::-For text fields, applicable when querying with <<the-standard-query-parser.adoc#standard-query-parser-parameters,`sow=false`>> (which is the default for the `sow` parameter). Use `true`, the default, for field types with query analyzers including graph-aware filters, e.g., <<filter-descriptions.adoc#synonym-graph-filter,Synonym Graph Filter>> and <<filter-descriptions.adoc#word-delimiter-graph-filter,Word Delimiter Graph Filter>>.+For text fields, applicable when querying with <<standard-query-parser.adoc#standard-query-parser-parameters,`sow=false`>> (which is the default for the `sow` parameter). Use `true`, the default, for field types with query analyzers including graph-aware filters, e.g., <<filters.adoc#synonym-graph-filter,Synonym Graph Filter>> and <<filters.adoc#word-delimiter-graph-filter,Word Delimiter Graph Filter>>.+-Use `false` for field types with query analyzers including filters that can match docs when some tokens are missing, e.g., <<filter-descriptions.adoc#shingle-filter,Shingle Filter>>.+Use `false` for field types with query analyzers including filters that can match docs when some tokens are missing, e.g., <<filters.adoc#shingle-filter,Shingle Filter>>.[[docvaluesformat]]`docValuesFormat`::@@ -120,9 +120,7 @@ The default values for each property depend on the underlying `FieldType` class,// TODO: SOLR-10655 BEGIN: refactor this into a 'field-default-properties.include.adoc' file for reuse-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="20,40,20,20",options="header"]+[%autowidth.stretch,options="header"]|===|Property |Description |Values |Implicit Default|indexed |If true, the value of the field can be used in queries to retrieve matching documents. |true or false |true@@ -142,8 +140,39 @@ The default values for each property depend on the underlying `FieldType` class,// TODO: SOLR-10655 END+== Choosing Appropriate Numeric Types++For general numeric needs, consider using one of the `IntPointField`, `LongPointField`, `FloatPointField`, or `DoublePointField` classes, depending on the specific values you expect. These "Dimensional Point" based numeric classes use specially encoded data structures to support efficient range queries regardless of the size of the ranges used. Enable <<docvalues.adoc#,DocValues>> on these fields as needed for sorting and/or faceting.++Some Solr features may not yet work with "Dimensional Points", in which case you may want to consider the equivalent `TrieIntField`, `TrieLongField`, `TrieFloatField`, and `TrieDoubleField` classes. These field types are deprecated and are likely to be removed in a future major Solr release, but they can still be used if necessary. Configure a `precisionStep="0"` if you wish to minimize index size, but if you expect users to make frequent range queries on numeric types, use the default `precisionStep` (by not specifying it) or specify it as `precisionStep="8"` (which is the default). This offers faster speed for range queries at the expense of increasing index size.++== Working With Text++Handling text properly will make your users happy by providing them with the best possible results for text searches.++One technique is using a text field as a catch-all for keyword searching. Most users are not sophisticated about their searches and the most common search is likely to be a simple keyword search. You can use `copyField` to take a variety of fields and funnel them all into a single text field for keyword searches.++In the `schema.xml` file for the "```techproducts```" example included with Solr, `copyField` declarations are used to dump the contents of `cat`, `name`, `manu`, `features`, and `includes` into a single field, `text`. In addition, it could be a good idea to copy `ID` into `text` in case users wanted to search for a particular product by passing its product number to a keyword search.++Another technique is using `copyField` to use the same field in different ways. Suppose you have a field that is a list of authors, like this:++`Schildt, Herbert; Wolpert, Lewis; Davies, P.`++For searching by author, you could tokenize the field, convert to lower case, and strip out punctuation:++`schildt / herbert / wolpert / lewis / davies / p`++For sorting, just use an untokenized field, converted to lower case, with punctuation stripped:++`schildt herbert wolpert lewis davies p`++Finally, for faceting, use the primary author only via a `StrField`:++`Schildt, Herbert`+== Field Type SimilarityA field type may optionally specify a `<similarity/>` that will be used when scoring documents that refer to fields with this type, as long as the "global" similarity for the collection allows it.-By default, any field type which does not define a similarity, uses `BM25Similarity`. For more details, and examples of configuring both global & per-type Similarities, please see <<other-schema-elements.adoc#similarity,Other Schema Elements>>.+By default, any field type which does not define a similarity, uses `BM25Similarity`.+For more details, and examples of configuring both global & per-type Similarities, please see <<schema-elements.adoc#similarity,Schema Elements>>.diff --git a/solr/solr-ref-guide/src/field-types-included-with-solr.adoc b/solr/solr-ref-guide/src/field-types-included-with-solr.adocindex a2b8be9b833..20bbe1aca06 100644--- a/solr/solr-ref-guide/src/field-types-included-with-solr.adoc+++ b/solr/solr-ref-guide/src/field-types-included-with-solr.adoc@@ -16,13 +16,13 @@// specific language governing permissions and limitations// under the License.-The following table lists the field types that are available in Solr and are recommended. The page further down, lists all the deprecated types for those migrating from older version of Solr. The {solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache.solr.schema`] package includes all the classes listed in this table.--// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed+The following table lists the field types that are available in Solr and are recommended.+The page further down, lists all the deprecated types for those migrating from older version of Solr.+The {solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache.solr.schema`] package includes all the classes listed in this table.== Recommended Field Types-[cols="25,75",options="header"]+[%autowidth.stretch,options="header"]|===|Class |Description|BBoxField | Indexes a single rectangle (bounding box) per document field and supports searching via a bounding box. See the section <<spatial-search.adoc#,Spatial Search>> for more information.@@ -33,17 +33,17 @@ The following table lists the field types that are available in Solr and are rec|CollationField |Supports Unicode collation for sorting and range queries. The ICUCollationField is a better choice if you can use ICU4J. See the section <<language-analysis.adoc#unicode-collation,Unicode Collation>> for more information.-|CurrencyFieldType |Supports currencies and exchange rates. See the section <<working-with-currencies-and-exchange-rates.adoc#,Working with Currencies and Exchange Rates>> for more information.+|CurrencyFieldType |Supports currencies and exchange rates. See the section <<currencies-exchange-rates.adoc#,Currencies and Exchange Rates>> for more information.-|DateRangeField |Supports indexing date ranges, to include point in time date instances as well (single-millisecond durations). See the section <<working-with-dates.adoc#,Working with Dates>> for more detail on using this field type. Consider using this field type even if it's just for date instances, particularly when the queries typically fall on UTC year/month/day/hour, etc., boundaries.+|DateRangeField |Supports indexing date ranges, to include point in time date instances as well (single-millisecond durations). See the section <<date-formatting-math.adoc#,Date Formatting and Date Math>> for more detail on using this field type. Consider using this field type even if it's just for date instances, particularly when the queries typically fall on UTC year/month/day/hour, etc., boundaries.-|DatePointField |Date field. Represents a point in time with millisecond precision, encoded using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. See the section <<working-with-dates.adoc#,Working with Dates>> for more details on the supported syntax. For single valued fields, `docValues="true"` must be used to enable sorting.+|DatePointField |Date field. Represents a point in time with millisecond precision, encoded using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. See the section <<date-formatting-math.adoc#,Working with Dates>> for more details on the supported syntax. For single valued fields, `docValues="true"` must be used to enable sorting.|DoublePointField |Double field (64-bit IEEE floating point). This class encodes double values using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. For single valued fields, `docValues="true"` must be used to enable sorting.-|ExternalFileField |Pulls values from a file on disk. See the section <<working-with-external-files-and-processes.adoc#,Working with External Files and Processes>> for more information.+|ExternalFileField |Pulls values from a file on disk. See the section <<external-files-processes.adoc#,External Files and Processes>> for more information.-|EnumFieldType |Allows defining an enumerated set of values which may not be easily sorted by either alphabetic or numeric order (such as a list of severities, for example). This field type takes a configuration file, which lists the proper order of the field values. See the section <<working-with-enum-fields.adoc#,Working with Enum Fields>> for more information.+|EnumFieldType |Allows defining an enumerated set of values which may not be easily sorted by either alphabetic or numeric order (such as a list of severities, for example). This field type takes a configuration file, which lists the proper order of the field values. See the section <<enum-fields.adoc#,Enum Fields>> for more information.|FloatPointField |Floating point field (32-bit IEEE floating point). This class encodes float values using a "Dimensional Points" based data structure that allows for very efficient searches for specific values, or ranges of values. For single valued fields, `docValues="true"` must be used to enable sorting.@@ -61,7 +61,7 @@ The following table lists the field types that are available in Solr and are rec|PreAnalyzedField |Provides a way to send to Solr serialized token streams, optionally with independent stored values of a field, and have this information stored and indexed without any additional text processing.-Configuration and usage of PreAnalyzedField is documented in the section <<working-with-external-files-and-processes.adoc#the-preanalyzedfield-type,Working with External Files and Processes>>.+Configuration and usage of PreAnalyzedField is documented in the section <<external-files-processes.adoc#the-preanalyzedfield-type,The PreAnalyzedField Type>>.|RandomSortField |Does not contain a value. Queries that sort on this field type will return results in random order. Use a dynamic field to use this feature.@@ -85,9 +85,9 @@ Configuration and usage of PreAnalyzedField is documented in the section <<work== Deprecated Field TypesNOTE: All Trie* numeric and date field types have been deprecated in favor of *Point field types.- Point field types are better at range queries (speed, memory, disk), however simple field:value queries underperform- relative to Trie. Either accept this, or continue to use Trie fields.- This shortcoming may be addressed in a future release.+Point field types are better at range queries (speed, memory, disk), however simple field:value queries underperform relative to Trie.+Either accept this, or continue to use Trie fields.+This shortcoming may be addressed in a future release.[cols="25,75",options="header"]|===diff --git a/solr/solr-ref-guide/src/field-types.adoc b/solr/solr-ref-guide/src/field-types.adocnew file mode 100644index 00000000000..985ae9107f5--- /dev/null+++ b/solr/solr-ref-guide/src/field-types.adoc@@ -0,0 +1,44 @@+= Field Types+:page-children: field-type-definitions-and-properties, \+ field-types-included-with-solr, \+ currencies-exchange-rates, \+ date-formatting-math, \+ enum-fields, \+ external-files-processes, \+ field-properties-by-use-case+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+The field type defines how Solr should interpret data in a field and how the field can be queried.+There are many field types included with Solr by default, and they can also be defined locally.++****++[%autowidth.stretch,cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>>+| <<field-types-included-with-solr.adoc#,Field Types Included with Solr>>+| <<currencies-exchange-rates.adoc#,Currencies and Exchange Rates>>+| <<date-formatting-math.adoc#,Date Formatting and Date Math>>+| <<enum-fields.adoc#,Enum Fields>>+| <<external-files-processes.adoc#,External Files and Processes>>+| <<field-properties-by-use-case.adoc#,Field Properties by Use Case>>+|+|===++****diff --git a/solr/solr-ref-guide/src/fields-and-schema-design.adoc b/solr/solr-ref-guide/src/fields-and-schema-design.adocnew file mode 100644index 00000000000..0bbf641a8a0--- /dev/null+++ b/solr/solr-ref-guide/src/fields-and-schema-design.adoc@@ -0,0 +1,42 @@+= Fields and Schema Design+:page-children: fields, \+ field-types, \+ copy-fields, \+ dynamic-fields, \+ docvalues, \+ luke-request-handler+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++This section discusses how Solr organizes its data into documents and fields, as well as how to work with a schema in Solr.++This section includes the following topics:++****+// This tags the below list so it can be used in the parent page section list+// tag::fields-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<fields.adoc#,Fields>>: Fields in Solr.+| <<field-types.adoc#,Field Types>>: Field types in Solr, including the field types in the default Solr schema.+| <<copy-fields.adoc#,Copy Fields>>: Fields with data copied from another field.+| <<dynamic-fields.adoc#,Dynamic Fields>>: Fields that inherit field type properties at index time.+| <<docvalues.adoc#,DocValues>>: DocValues indexes for faster lookups.+| <<luke-request-handler.adoc#,Luke Request Handler>>: Provides access to information about fields in the index.+|===+// end::fields-sections[]+****diff --git a/solr/solr-ref-guide/src/defining-fields.adoc b/solr/solr-ref-guide/src/fields.adocsimilarity index 76%rename from solr/solr-ref-guide/src/defining-fields.adocrename to solr/solr-ref-guide/src/fields.adocindex 3a0c8adeac3..02582ce84f3 100644--- a/solr/solr-ref-guide/src/defining-fields.adoc+++ b/solr/solr-ref-guide/src/fields.adoc@@ -1,4 +1,4 @@-= Defining Fields+= Fields// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,8 @@// specific language governing permissions and limitations// under the License.-Fields are defined in the fields element of `schema.xml`. Once you have the field types set up, defining the fields themselves is simple.+Fields are defined in the fields element of `schema.xml`.+Once you have the field types set up, defining the fields themselves is simple.== Example Field Definition@@ -32,23 +33,30 @@ The following example defines a field named `price` with a type named `float` anField definitions can have the following properties:`name`::-The name of the field. Field names should consist of alphanumeric or underscore characters only and not start with a digit. This is not currently strictly enforced, but other field names will not have first class support from all components and back compatibility is not guaranteed. Names with both leading and trailing underscores (e.g., `\_version_`) are reserved. Every field must have a `name`.+The name of the field.+Field names should consist of alphanumeric or underscore characters only and not start with a digit.+This is not currently strictly enforced, but other field names will not have first class support from all components and back compatibility is not guaranteed.+Names with both leading and trailing underscores (e.g., `\_version_`) are reserved.+Every field must have a `name`.`type`::-The name of the `fieldType` for this field. This will be found in the `name` attribute on the `fieldType` definition. Every field must have a `type`.+The name of the `fieldType` for this field.+This will be found in the `name` attribute on the `fieldType` definition.+Every field must have a `type`.`default`::-A default value that will be added automatically to any document that does not have a value in this field when it is indexed. If this property is not specified, there is no default.+A default value that will be added automatically to any document that does not have a value in this field when it is indexed.+If this property is not specified, there is no default.== Optional Field Type Override Properties-Fields can have many of the same properties as field types. Properties from the table below which are specified on an individual field will override any explicit value for that property specified on the the `fieldType` of the field, or any implicit default property value provided by the underlying `fieldType` implementation. The table below is reproduced from <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>>, which has more details:+Fields can have many of the same properties as field types.+Properties from the table below which are specified on an individual field will override any explicit value for that property specified on the the `fieldType` of the field, or any implicit default property value provided by the underlying `fieldType` implementation.+The table below is reproduced from <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>>, which has more details:// TODO: SOLR-10655 BEGIN: refactor this into a 'field-default-properties.include.adoc' file for reuse-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="20,40,20,20",options="header"]+[%autowidth.stretch,options="header"]|===|Property |Description |Values |Implicit Default|indexed |If true, the value of the field can be used in queries to retrieve matching documents. |true or false |truediff --git a/solr/solr-ref-guide/src/files-screen.adoc b/solr/solr-ref-guide/src/files-screen.adocdeleted file mode 100644index 57be896f169..00000000000--- a/solr/solr-ref-guide/src/files-screen.adoc+++ /dev/null@@ -1,37 +0,0 @@-= Files Screen-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Files screen lets you browse & view the various configuration files (such `solrconfig.xml` and the schema file) for the collection you selected.--.The Files Screen-image::images/files-screen/files-screen.png[image,height=400]--If you are using <<solrcloud.adoc#,SolrCloud>>, the files displayed are the configuration files for this collection stored in ZooKeeper. In a standalone Solr installations, all files in the `conf` directory are displayed.--While `solrconfig.xml` defines the behavior of Solr as it indexes content and responds to queries, the Schema allows you to define the types of data in your content (field types), the fields your documents will be broken into, and any dynamic fields that should be generated based on patterns of field names in the incoming documents. Any other configuration files are used depending on how they are referenced in either `solrconfig.xml` or your schema.--Configuration files cannot be edited with this screen, so a text editor of some kind must be used.--This screen is related to the <<schema-browser-screen.adoc#,Schema Browser Screen>>, in that they both can display information from the schema, but the Schema Browser provides a way to drill into the analysis chain and displays linkages between field types, fields, and dynamic field rules.--Many of the options defined in these configuration files are described throughout the rest of this Guide. In particular, you will want to review these sections:--* <<indexing-and-basic-data-operations.adoc#,Indexing and Basic Data Operations>>-* <<searching.adoc#,Searching>>-* <<the-well-configured-solr-instance.adoc#,The Well-Configured Solr Instance>>-* <<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>>diff --git a/solr/solr-ref-guide/src/filter-descriptions.adoc b/solr/solr-ref-guide/src/filters.adocsimilarity index 96%rename from solr/solr-ref-guide/src/filter-descriptions.adocrename to solr/solr-ref-guide/src/filters.adocindex 7d811babad5..eb72764d2fc 100644--- a/solr/solr-ref-guide/src/filter-descriptions.adoc+++ b/solr/solr-ref-guide/src/filters.adoc@@ -1,4 +1,4 @@-= Filter Descriptions+= Filters// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,43 +16,86 @@// specific language governing permissions and limitations// under the License.-Filters examine a stream of tokens and keep them, transform them or discard them, depending on the filter type being used.+Filters examine a stream of tokens and keep them, transform them, or discard them depending on the filter type being used.-You configure each filter with a `<filter>` element in `schema.xml` as a child of `<analyzer>`, following the `<tokenizer>` element. Filter definitions should follow a tokenizer or another filter definition because they take a `TokenStream` as input. For example:+== About Filters++Like <<tokenizers.adoc#,tokenizers>>, filters consume input and produce a stream of tokens.+Filters also derive from `org.apache.lucene.analysis.TokenStream` but unlike tokenizers, a filter's input is another TokenStream.+The job of a filter is usually easier than that of a tokenizer since in most cases a filter looks at each token in the stream sequentially and decides whether to pass it along, replace it, or discard it.++A filter may also do more complex analysis by looking ahead to consider multiple tokens at once, although this is less common.+One hypothetical use for such a filter might be to normalize state names that would be tokenized as two words.+For example, the single token "california" would be replaced with "CA", while the token pair "rhode" followed by "island" would become the single token "RI".++Because filters consume one `TokenStream` and produce a new `TokenStream`, they can be chained one after another indefinitely.+Each filter in the chain in turn processes the tokens produced by its predecessor.+The order in which you specify the filters is therefore significant.+Typically, the most general filtering is done first, and later filtering stages are more specialized.++=== Filter Configuration+Filters are configured with a `<filter>` element in the schema file as a child of `<analyzer>`, following the `<tokenizer>` element.++For example:[.dynamic-tabs]---[example.tab-pane#byname-filter]+[example.tab-pane#byname-filterexample]====[.tab-label]*With name*[source,xml]----<fieldType name="text" class="solr.TextField">- <analyzer type="index">+ <analyzer><tokenizer name="standard"/><filter name="lowercase"/>+ <filter name="englishPorter"/></analyzer></fieldType>----====-[example.tab-pane#byclass-filter]+[example.tab-pane#byclass-filterexample]====[.tab-label]*With class name (legacy)*[source,xml]----<fieldType name="text" class="solr.TextField">- <analyzer type="index">+ <analyzer><tokenizer class="solr.StandardTokenizerFactory"/><filter class="solr.LowerCaseFilterFactory"/>+ <filter class="solr.EnglishPorterFilterFactory"/></analyzer></fieldType>----====---The name/class attribute names a factory class that will instantiate a filter object as needed. Filter factory classes must implement the `org.apache.lucene.analysis.TokenFilterFactory` interface. Like tokenizers, filters are also instances of TokenStream and thus are producers of tokens. Unlike tokenizers, filters also consume tokens from a TokenStream. This allows you to mix and match filters, in any order you prefer, downstream of a tokenizer.+This example starts with Solr's standard tokenizer, which breaks the field's text into tokens.+All the tokens are then set to lowercase, which will facilitate case-insensitive matching at query time.++The last filter in the above example is a stemmer filter that uses the Porter stemming algorithm.++=== Stemming+A stemmer is basically a set of mapping rules that maps the various forms of a word back to the base, or _stem_, word from which they derive.++For example, in English the words "hugs", "hugging" and "hugged" are all forms of the stem word "hug".+The stemmer will replace all of these terms with "hug", which is what will be indexed.+This means that a query for "hug" will match the term "hugged", but not "huge".++Conversely, applying a stemmer to your query terms will allow queries containing non-stemmed terms, like "hugging", to match documents with different variations of the same stem word, such as "hugged".+This works because both the indexer and the query will map to the same stem ("hug").++Word stemming is, obviously, very language specific.+Solr includes several language-specific stemmers created by the http://snowball.tartarus.org/[Snowball] generator that are based on the Porter stemming algorithm.+The generic <<Snowball Porter Stemmer Filter>> can be used to configure any of these language stemmers.+Solr also includes a convenience wrapper for the English Snowball stemmer.+There are also several purpose-built stemmers for non-English languages.+These stemmers are described in <<language-analysis.adoc#,Language Analysis>>.++=== Filters with Arguments-Arguments may be passed to tokenizer factories to modify their behavior by setting attributes on the `<filter>` element. For example:+Arguments may be passed to tokenizer factories to modify their behavior by setting attributes on the `<filter>` element.+For example:[.dynamic-tabs]--@@ -88,7 +131,8 @@ The following sections describe the filter factories that are included in this r== ASCII Folding Filter-This filter converts alphabetic, numeric, and symbolic Unicode characters which are not in the Basic Latin Unicode block (the first 127 ASCII characters) to their ASCII equivalents, if one exists. This filter converts characters from the following Unicode blocks:+This filter converts alphabetic, numeric, and symbolic Unicode characters which are not in the Basic Latin Unicode block (the first 127 ASCII characters) to their ASCII equivalents, if one exists.+This filter converts characters from the following Unicode blocks:* http://www.unicode.org/charts/PDF/U0080.pdf[C1 Controls and Latin-1 Supplement] (PDF)* http://www.unicode.org/charts/PDF/U0100.pdf[Latin Extended-A] (PDF)diff --git a/solr/solr-ref-guide/src/function-queries.adoc b/solr/solr-ref-guide/src/function-queries.adocindex 8dfb206c116..3a57378e0c9 100644--- a/solr/solr-ref-guide/src/function-queries.adoc+++ b/solr/solr-ref-guide/src/function-queries.adoc@@ -18,7 +18,7 @@Function queries enable you to generate a relevancy score using the actual value of one or more numeric fields.-Function queries are supported by the <<the-dismax-query-parser.adoc#,DisMax>>, <<the-extended-dismax-query-parser.adoc#,Extended DisMax>>, and <<the-standard-query-parser.adoc#,standard>> query parsers.+Function queries are supported by the <<dismax-query-parser.adoc#,DisMax>>, <<edismax-query-parser.adoc#,Extended DisMax>>, and <<standard-query-parser.adoc#,standard>> query parsers.Function queries use _functions_. The functions can be a constant (numeric or string literal), a field, another function or a parameter substitution argument. You can use these functions to modify the ranking of results for users. These could be used to change the ranking of results based on a user's location, or some other calculation.@@ -57,7 +57,7 @@ the output would be:<float name="score">0.343</float>...-----* Use in a parameter that is explicitly for specifying functions, such as the eDisMax query parser's <<the-extended-dismax-query-parser.adoc#extended-dismax-parameters,`boost` parameter>>, or the DisMax query parser's <<the-dismax-query-parser.adoc#bf-boost-functions-parameter,`bf` (boost function) parameter>>. (Note that the `bf` parameter actually takes a list of function queries separated by white space and each with an optional boost. Make sure you eliminate any internal white space in single function queries when using `bf`). For example:+* Use in a parameter that is explicitly for specifying functions, such as the eDisMax query parser's <<edismax-query-parser.adoc#extended-dismax-parameters,`boost` parameter>>, or the DisMax query parser's <<dismax-query-parser.adoc#bf-boost-functions-parameter,`bf` (boost function) parameter>>. (Note that the `bf` parameter actually takes a list of function queries separated by white space and each with an optional boost. Make sure you eliminate any internal white space in single function queries when using `bf`). For example:+[source,text]----@@ -85,7 +85,7 @@ Returns the absolute value of the specified value or function.* `abs(-5)`=== childfield(field) Function-Returns the value of the given field for one of the matched child docs when searching by <<other-parsers.adoc#block-join-parent-query-parser,{!parent}>>. It can be used only in `sort` parameter.+Returns the value of the given field for one of the matched child docs when searching by <<block-join-query-parser.adoc#block-join-parent-query-parser,{!parent}>>. It can be used only in `sort` parameter.*Syntax Examples*@@ -253,7 +253,7 @@ Use the `field(myfield,min)` <<field Function,syntax for selecting the minimum v=== ms FunctionReturns milliseconds of difference between its arguments. Dates are relative to the Unix or POSIX time epoch, midnight, January 1, 1970 UTC.-Arguments may be the name of a `DatePointField`, `TrieDateField`, or date math based on a <<working-with-dates.adoc#,constant date or `NOW`>>.+Arguments may be the name of a `DatePointField`, `TrieDateField`, or date math based on a <<date-formatting-math.adoc#,constant date or `NOW`>>.* `ms()`: Equivalent to `ms(NOW)`, number of milliseconds since the epoch.* `ms(a):` Returns the number of milliseconds since the epoch that the argument represents.@@ -332,7 +332,7 @@ Returns the product of multiple values or functions, which are specified in a co* `mul(x,y)`=== query Function-Returns the score for the given subquery, or the default value for documents not matching the query. Any type of subquery is supported through either parameter de-referencing `$otherparam` or direct specification of the query string in the <<local-parameters-in-queries.adoc#,Local Parameters>> through the `v` key.+Returns the score for the given subquery, or the default value for documents not matching the query. Any type of subquery is supported through either parameter de-referencing `$otherparam` or direct specification of the query string in the <<local-params.adoc#,local params>> through the `v` key.*Syntax Examples*diff --git a/solr/solr-ref-guide/src/further-assistance.adoc b/solr/solr-ref-guide/src/further-assistance.adocdeleted file mode 100644index 3aa86cf667c..00000000000--- a/solr/solr-ref-guide/src/further-assistance.adoc+++ /dev/null@@ -1,21 +0,0 @@-= Further Assistance-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--There is a very active user community around Solr and Lucene. The solr-user mailing list, and #solr IRC channel are both great resources for asking questions.--To view the mailing list archives, subscribe to the list, or join the IRC channel, please see https://solr.apache.org/community.html.diff --git a/solr/solr-ref-guide/src/getting-started.adoc b/solr/solr-ref-guide/src/getting-started.adocindex c3bf8b58c82..a88c76db3c1 100644--- a/solr/solr-ref-guide/src/getting-started.adoc+++ b/solr/solr-ref-guide/src/getting-started.adoc@@ -1,5 +1,9 @@= Getting Started-:page-children: solr-tutorial, a-quick-overview, solr-system-requirements, installing-solr+:page-children: introduction, \+ solr-concepts, \+ solr-tutorial, \+ solr-admin-ui, \+ about-this-guide// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -20,12 +24,28 @@[.lead]Solr makes it easy for programmers to develop sophisticated, high-performance search applications with advanced features.-This section introduces you to the basic Solr architecture and features to help you get up and running quickly. It covers the following topics:+This section introduces you to the basic Solr architecture and features to help you get up and running quickly.+It covers the following topics:-<<solr-tutorial.adoc#,Solr Tutorial>>: This tutorial covers getting Solr up and running+****+<<introduction.adoc#,*Introduction to Solr*>>: Short introduction to Solr's primary features.+****-<<a-quick-overview.adoc#,A Quick Overview>>: A high-level overview of how Solr works.+****+<<solr-concepts.adoc#,*Solr Concepts*>>: Beginner information on Solr and information retrieval concepts.-<<solr-system-requirements.adoc#,Solr System Requirements>>: Solr System Requirement+// This pulls the sub-section list from the child page to reduce errors+include::solr-concepts.adoc[tag=concept-sections]+****-<<installing-solr.adoc#,Installing Solr>>: A walkthrough of the Solr installation process.+****+<<solr-tutorial.adoc#,*Solr Tutorial*>>: A multi-exercise tutorial that covers the main concepts of indexing and querying Solr.+****++****+<<solr-admin-ui.adoc#,*Solr Admin UI*>>: Solr's Web-based interface for administering Solr.+****++****+<<about-this-guide.adoc#,*About this Guide*>>: How to use this Reference Guide.+****diff --git a/solr/solr-ref-guide/src/graph-traversal.adoc b/solr/solr-ref-guide/src/graph-traversal.adocindex c46eb353496..23b9737ced6 100644--- a/solr/solr-ref-guide/src/graph-traversal.adoc+++ b/solr/solr-ref-guide/src/graph-traversal.adoc@@ -220,7 +220,7 @@ nodes(emails,gather="to")-----In the example above only emails that match the filter query will be included in the traversal. Any Solr query can be included here. So you can do fun things like <<spatial-search.adoc#,geospatial queries>>, apply any of the available <<query-syntax-and-parsing.adoc#,query parsers>>, or even write custom query parsers to limit the traversal.+In the example above only emails that match the filter query will be included in the traversal. Any Solr query can be included here. So you can do fun things like <<spatial-search.adoc#,geospatial queries>>, apply any of the available <<query-syntax-and-parsers.adoc#,query parsers>>, or even write custom query parsers to limit the traversal.== Root Streamsdiff --git a/solr/solr-ref-guide/src/graph.adoc b/solr/solr-ref-guide/src/graph.adocindex ef690312276..8d7294f7912 100644--- a/solr/solr-ref-guide/src/graph.adoc+++ b/solr/solr-ref-guide/src/graph.adoc@@ -32,12 +32,14 @@ In the next few sections below we'll review the graph theory behind Solr's graph=== SubgraphsA subgraph is a smaller subset of the nodes and connections of the-larger graph. Graph expressions allow you to flexibly define and materialize a subgraph from the larger graph+larger graph.+Graph expressions allow you to flexibly define and materialize a subgraph from the larger graphstored in the distributed index.Subgraphs play two important roles:-* They provide a local context for link analysis. The design of the subgraph defines the meaning of the link analysis.+* They provide a local context for link analysis.+The design of the subgraph defines the meaning of the link analysis.* They provide a foreground graph that can be compared to the background index for anomaly detection purposes.@@ -45,8 +47,10 @@ Subgraphs play two important roles:Graph expressions can be used to materialize *bipartite subgraphs*.A bipartite graph is a graph where the nodes are split into two-distinct categories. The links between those two categories can then-be analyzed to study how they relate. Bipartite graphs are often discussed+distinct categories.+The links between those two categories can then+be analyzed to study how they relate.+Bipartite graphs are often discussedin the context of collaborative filter recommender systems.A bipartite graph between *shopping baskets* and *products* is a useful example.@@ -83,7 +87,8 @@ Each cell has a 1 or 0 signifying if the product is in the basket.Let's look at how Solr graph expressions materializes this bipartite subgraph:-The `nodes` function is used to materialize a subgraph from the larger graph. Below is an example nodes function which materializes the bipartite graph shown in the matrix above.+The `nodes` function is used to materialize a subgraph from the larger graph.+Below is an example nodes function which materializes the bipartite graph shown in the matrix above.[source,text]----@@ -103,10 +108,12 @@ random(baskets, q="product_s:butter", fl="basket_s", rows="3")----The `random` function is searching the baskets collection with the query `product_s:butter`, and-returning 3 random samples. Each sample contains the `basket_s` field which is the basket id.+returning 3 random samples.+Each sample contains the `basket_s` field which is the basket id.The three basket id's that are returned by the random sample are the *root nodes* of the graph query.-The `nodes` function is the graph query. The nodes function is operating over the three root nodes returned+The `nodes` function is the graph query.+The nodes function is operating over the three root nodes returnedby the random function.It "walks" the graph by searching the `basket_s` field of the root nodes against the `basket_s` field in the index.This finds all the product records for the root baskets.@@ -168,18 +175,21 @@ The output of the shopping basket graph expression is as follows:----The `ancestors` property in the result contains a unique, alphabetically sorted set of all the *inbound links*-to the node in the subgraph. In this case it shows the baskets that are linked to each product.+to the node in the subgraph.+In this case it shows the baskets that are linked to each product.The ancestor links will only be tracked when the trackTraversal flag is turned on in the nodes expression.=== Link Analysis and Degree CentralityLink analysis is often performed to determine *node centrality*. When analyzing for centrality thegoal is to assign a weight to each node based on how connected it is in the subgraph.-There are different types of node centrality. Graph expressions very efficiently calculates+There are different types of node centrality.+Graph expressions very efficiently calculates*inbound degree centrality* (in-degree).Inbound degree centrality is calculated by counting the number of inbound-links to each node. For simplicity this document will sometimes refer+links to each node.+For simplicity this document will sometimes referto inbound degree simply as degree.Back to the shopping basket example:@@ -261,7 +271,8 @@ The output of this graph expression is as follows:The `count(+++*+++)` aggregation counts the "gathered" nodes, in this case the values in the `product_s` field.Notice that the `count(+++*+++)` result is the same as the number of ancestors.This will always be the case because the nodes function first deduplicates the edges before-counting the gathered nodes. Because of this the `count(+++*+++)` aggregation always calculates the+counting the gathered nodes.+Because of this the `count(+++*+++)` aggregation always calculates theinbound degree centrality for the gathered nodes.=== Dot Product@@ -276,22 +287,28 @@ This tells us that a nearest neighbor search, using a maximum inner product simi=== Limiting Basket Out-Degree-The recommendation can be made stronger by limiting the *out-degree* of the baskets. The out-degree is the-number of outbound links of a node in a graph. In the shopping basket example the outbound links-from the baskets link to products. So limiting the out-degree will limit the size of the baskets.+The recommendation can be made stronger by limiting the *out-degree* of the baskets.+The out-degree is the+number of outbound links of a node in a graph.+In the shopping basket example the outbound links+from the baskets link to products.+So limiting the out-degree will limit the size of the baskets.Why does limiting the size of the shopping baskets make a stronger recommendation? To answer this question it helpsto think about each shopping basket as *voting* for products that go with *butter*. In an election with two candidatesif you were to vote for both candidates the votes would cancel each other out and have no effect.-But if you vote for only one candidate your vote will affect the outcome. The same principle holds true-for recommendations. As a basket votes for more products it dilutes the strength of its recommendation for any-one product. A basket with just butter and one other item more strongly recommends that item.--The `maxDocFreq` parameter can be used to limit the graph "walk" to only include baskets that appear in-the index a certain number of times. Since each occurrence of a basket ID in the index is a link to a product,-limiting the document frequency of the basket ID will limit the out-degree of the basket. The `maxDocFreq` parameter is-applied per shard. If there is a single shard or documents are co-located by basket ID then the `maxDocFreq` will-be an exact count. Otherwise, it will return baskets with a max size of numShards * maxDocFreq.+But if you vote for only one candidate your vote will affect the outcome.+The same principle holds true+for recommendations.+As a basket votes for more products it dilutes the strength of its recommendation for any+one product.+A basket with just butter and one other item more strongly recommends that item.++The `maxDocFreq` parameter can be used to limit the graph "walk" to only include baskets that appear in the index a certain number of times.+Since each occurrence of a basket ID in the index is a link to a product, limiting the document frequency of the basket ID will limit the out-degree of the basket.+The `maxDocFreq` parameter is applied per shard.+If there is a single shard or documents are co-located by basket ID then the `maxDocFreq` will be an exact count.+Otherwise, it will return baskets with a max size of numShards * maxDocFreq.The example below shows the `maxDocFreq` parameter applied to the `nodes` expression.@@ -311,14 +328,16 @@ nodes(baskets,The degree of the node describes how many nodes in the subgraph link to it.But this does not tell us if the node is particularly central to this subgraph or if it is just a-very frequent node in the entire graph. Nodes that appear frequently in the subgraph but+very frequent node in the entire graph.+Nodes that appear frequently in the subgraph butinfrequently in the entire graph can be considered more *relevant* to the subgraph.The search index contains information about how frequently each node appears in the entire index.Using a technique similar to *tf-idf* document scoring, graph expressions can combine thedegree of the node with its inverse document frequency in the index to determine a relevancy score.-The `scoreNodes` function scores the nodes. Below is an example of the scoreNodes function applied to+The `scoreNodes` function scores the nodes.+Below is an example of the scoreNodes function applied tothe shopping basket node set.[source,text]@@ -332,10 +351,13 @@ scoreNodes(nodes(baskets,count(*)))-----The output now includes a `nodeScore` property. In the output below notice how *eggs* has a higher+The output now includes a `nodeScore` property.+In the output below notice how *eggs* has a highernodeScore than *milk* even though they have the same `count(+++*+++)`. This is because milk appears more-frequently in the entire index than eggs does. The `docFreq` property added by the `nodeScore` function-shows the document frequency in the index. Because of the lower `docFreq` eggs is considered more relevant+frequently in the entire index than eggs does.+The `docFreq` property added by the `nodeScore` function+shows the document frequency in the index.+Because of the lower `docFreq` eggs is considered more relevantto this subgraph, and a better recommendation to be paired with butter.[source,json]@@ -397,13 +419,13 @@ to this subgraph, and a better recommendation to be paired with butter.The examples above lay the groundwork for temporal graph queries.Temporal graph queries allow the `nodes` function to walk the graph using *windows of time* to surface-*cross-correlations* within the data. The nodes function currently supports graph walks using *ten second increments*+*cross-correlations* within the data.+The nodes function currently supports graph walks using *ten second increments*which is useful for *event correlation* and *root cause analysis* in log analytics.-In order to support temporal graph queries a ten second truncated timestamp in *ISO 8601* format must-be added to the log records as a string field at indexing time. Here is a sample ten second-truncated timestamp: `2021-02-10T20:51:30Z`. This small data change enables some very important-use cases so it's well worth the effort.+In order to support temporal graph queries a ten second truncated timestamp in *ISO 8601* format must be added to the log records as a string field at indexing time.+Here is a sample ten second truncated timestamp: `2021-02-10T20:51:30Z`.+This small data change enables some very important use cases so it's well worth the effort.Solr's indexing tool for Solr logs, described <<logs.adoc#,here>>, already adds the ten second truncated timestamps.So those using Solr to analyze Solr logs get temporal graph expressions for free.@@ -414,7 +436,8 @@ Once the ten second windows have been indexed with the log records we can devisecreates a set of *root events*. We can demonstrate this with an example using Solr log records.In this example we'll perform a Streaming Expression `facet` aggregation that finds the top 10, ten second windows-with the highest average query time. These time windows can be used to represent *slow query events* in a temporal+with the highest average query time.+These time windows can be used to represent *slow query events* in a temporalgraph query.Here is the facet function:@@ -543,8 +566,10 @@ Below is the resulting node set:----In this result set the `node` field holds the type of log events that occurred within the-same ten second windows as the root events. Notice that the event types include:-query, admin, update and error. The `count(+++*+++)` shows the degree centrality of the different+same ten second windows as the root events.+Notice that the event types include:+query, admin, update and error.+The `count(+++*+++)` shows the degree centrality of the differentlog event types.Notice that there is only one *error* event within the same ten second windows of the slow query events.@@ -552,9 +577,11 @@ Notice that there is only one *error* event within the same ten second windows o=== Window ParameterFor event correlation and root cause analysis it's not enough to find events that occur-within the *same* ten second root event windows. What's needed is to find events that occur+within the *same* ten second root event windows.+What's needed is to find events that occurwithin a window of time *prior to each root event*. The `window` parameter allows you to-specify this prior window of time as part of the query. The window parameter is an integer+specify this prior window of time as part of the query.+The window parameter is an integerwhich specifies the number of ten second time windows, prior to each root event window,to include in the graph walk.@@ -642,10 +669,12 @@ The window parameter doesn't capture the delay as we only know that an eventoccurred somewhere within a prior window.The `lag` parameter can be used to start calculating the window parameter a-number of ten second windows in the past. For example we could walk the graph in 20 second+number of ten second windows in the past.+For example we could walk the graph in 20 secondwindows starting from 30 seconds prior to a set of root events.By adjusting the lag and re-running the query we can determine which lagged-window has the highest degree. From this we can determine the delay.+window has the highest degree.+From this we can determine the delay.=== Node Scoring and Temporal Anomaly Detection@@ -656,7 +685,8 @@ but it does not establish if the event is a very common occurrence inthe entire graph or specific to the subgraph.The `scoreNodes` functions can be applied to score the nodes based on the degree and the-commonality of the node's term in the index. This will establish whether the event is anomalous to+commonality of the node's term in the index.+This will establish whether the event is anomalous tothe root events.[source,text]diff --git a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adocindex 82cd87a30a4..532c99bcbf0 100644--- a/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/hadoop-authentication-plugin.adoc@@ -18,54 +18,121 @@The Hadoop authentication plugin enables Solr to use the https://hadoop.apache.org/docs/stable/hadoop-auth/index.html[Hadoop authentication library] for securing Solr nodes.-This authentication plugin is a thin wrapper that delegates all functionality to the Hadoop authentication library. All configuration parameters for the library are passed through the plugin.+This authentication plugin is a thin wrapper that delegates all functionality to the Hadoop authentication library.+All configuration parameters for the library are passed through the plugin.This plugin can be particularly useful in leveraging an extended set of features or newly available features in the Hadoop authentication library.-Please note that the version of Hadoop library used by Solr is upgraded periodically. While Solr will ensure the stability and backwards compatibility of the structure of the plugin configuration (viz., the parameter names of this plugin), the values of these parameters may change based on the version of Hadoop library. Please review the Hadoop documentation for the version used by your Solr installation for more details.+Please note that the version of Hadoop library used by Solr is upgraded periodically.+While Solr will ensure the stability and backwards compatibility of the structure of the plugin configuration (viz., the parameter names of this plugin), the values of these parameters may change based on the version of the Hadoop library.+Please review the Hadoop documentation for v{ivy-hadoop-version} used by this version of Solr.-For some of the authentication schemes (e.g., Kerberos), Solr provides a native implementation of authentication plugin. If you require a more stable setup, in terms of configuration, ability to perform rolling upgrades, backward compatibility, etc., you should consider using such plugin. Please review the section <<authentication-and-authorization-plugins.adoc#,Authentication and Authorization Plugins>> for an overview of authentication plugin options in Solr.+For some authentication schemes (e.g., Kerberos), Solr provides a native implementation of authentication plugin.+If you require a more stable setup, in terms of configuration, ability to perform rolling upgrades, backward compatibility, etc., you should consider using one of these plugins.+Please review the section <<authentication-and-authorization-plugins.adoc#,Authentication and Authorization Plugins>> for an overview of authentication plugin options in Solr.There are two plugin classes:-* `HadoopAuthPlugin`: This can be used with standalone Solr as well as SolrCloud with <<authentication-and-authorization-plugins.adoc#securing-inter-node-requests,PKI authentication>> for internode communication.-* `ConfigurableInternodeAuthHadoopPlugin`: This is an extension of HadoopAuthPlugin that allows you to configure the authentication scheme for internode communication.+* `HadoopAuthPlugin`: This can be used with SolrCloud, user-managed, and single-node installations as well as SolrCloud with <<authentication-and-authorization-plugins.adoc#securing-inter-node-requests,PKI authentication>> for internode communication.+* `ConfigurableInternodeAuthHadoopPlugin`: This is an extension of `HadoopAuthPlugin` that allows you to configure the authentication scheme for internode communication.[TIP]====-For most SolrCloud or standalone Solr setups, the `HadoopAuthPlugin` should suffice.+For most installations, the `HadoopAuthPlugin` should suffice.====== Plugin Configuration`class`::-Should be either `solr.HadoopAuthPlugin` or `solr.ConfigurableInternodeAuthHadoopPlugin`. This parameter is required.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The plugin classname.+It should be either `solr.HadoopAuthPlugin` or `solr.ConfigurableInternodeAuthHadoopPlugin`.`type`::-The type of authentication scheme to be configured. See https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[configuration] options. This parameter is required.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The type of authentication scheme to be configured.+See the Hadoop https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[configuration options].`sysPropPrefix`::-The prefix to be used to define the Java system property for configuring the authentication mechanism. This property is required.+-The name of the Java system property is defined by appending the configuration parameter name to this prefix value. For example, if the prefix is `solr` then the Java system property `solr.kerberos.principal` defines the value of configuration parameter `kerberos.principal`.+[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The prefix to use to define a Java system property for configuring the authentication mechanism.+++The name of the Java system property is defined by appending the configuration parameter name to this prefix value.+For example, if the prefix is `solr` then the Java system property `solr.kerberos.principal` defines the value of configuration parameter `kerberos.principal`.`authConfigs`::-Configuration parameters required by the authentication scheme defined by the type property. This property is required. For more details, see https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[Hadoop configuration] options.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++Configuration parameters required by the authentication scheme defined by the type property.+For more details, see the Hadoop https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[configuration options].`defaultConfigs`::-Default values for the configuration parameters specified by the `authConfigs` property. The default values are specified as a collection of key-value pairs (i.e., `"property-name": "default_value"`).+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Default values for the configuration parameters specified by the `authConfigs` property.+The default values are specified as a collection of key-value pairs (i.e., `"property-name": "default_value"`).`enableDelegationToken`::-If `true`, the delegation tokens functionality will be enabled.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++If `true`, delegation tokens functionality will be enabled.+See the section <<Simple Authentication with Delegation Tokens>> for an example.`initKerberosZk`::-For enabling initialization of kerberos before connecting to ZooKeeper (if applicable).+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++If `true`, Kerberos will be initialized before connecting to ZooKeeper (if applicable).`proxyUserConfigs`::-Configures proxy users for the underlying Hadoop authentication mechanism. This configuration is expressed as a collection of key-value pairs (i.e., `"property-name": "default_value"`).+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Configures proxy users for the underlying Hadoop authentication mechanism.+This configuration is expressed as a collection of key-value pairs (i.e., `"property-name": "default_value"`).-`clientBuilderFactory`:: No |-The `HttpClientBuilderFactory` implementation used for the Solr internal communication. Only applicable for `ConfigurableInternodeAuthHadoopPlugin`.+`clientBuilderFactory`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The `HttpClientBuilderFactory` implementation used for the Solr internal communication.+Only applicable for `ConfigurableInternodeAuthHadoopPlugin`.== Example Configurations@@ -73,13 +140,16 @@ The `HttpClientBuilderFactory` implementation used for the Solr internal communiThis example lets you configure Solr to use Kerberos Authentication, similar to how you would use the <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>>.-After consulting the Hadoop authentication library's documentation, you can supply per host configuration parameters using the `solr.*` prefix. As an example, the Hadoop authentication library expects a parameter `kerberos.principal`, which can be supplied as a system property named `solr.kerberos.principal` when starting a Solr node. Refer to the section <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> for other typical configuration parameters.+After consulting the Hadoop authentication library's documentation, you can supply per-host configuration parameters using the `solr.*` prefix.-Please note that this example uses `ConfigurableInternodeAuthHadoopPlugin`, and hence you must provide the `clientBuilderFactory` implementation. As a result, all internode communication will use the Kerberos mechanism, instead of PKI authentication.+As an example, the Hadoop authentication library expects a parameter `kerberos.principal`, which can be supplied as a system property named `solr.kerberos.principal` when starting a Solr node.+Refer to the section <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> for other typical configuration parameters.-This configuration assumes that your servers are using the `solr` principal, and will be allowed to impersonate any other user with requests coming from any other host. For additional security, consider setting the host list to match your cluster nodes. The Hadoop https://hadoop.apache.org/docs/r3.2.2/hadoop-project-dist/hadoop-common/Superusers.html[proxy users] documentation contains more detail about available configuration options.+The example below uses `ConfigurableInternodeAuthHadoopPlugin`, and hence you must provide the `clientBuilderFactory` implementation.+As a result, all internode communication will use the Kerberos mechanism, instead of PKI authentication.-To setup this plugin, use the following in your `security.json` file.+This configuration assumes that your servers are using the `solr` principal, and will be allowed to impersonate any other user with requests coming from any other host.+For additional security, consider setting the host list to match your cluster nodes. The Hadoop https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/Superusers.html[proxy users documentation] contains more detail about available configuration options.[source,json]----@@ -107,13 +177,16 @@ To setup this plugin, use the following in your `security.json` file.----[WARNING]-For the `ConfigurableInternodeAuthHadoopPlugin`, user credential proxying relies on delegation token support. Without it, forwarded requests will authenticate as Solr server credentials instead of real-user credentials, and likely allowing authenticated-but-unauthorized users to query and index documents into your collections.+For the `ConfigurableInternodeAuthHadoopPlugin`, user credential proxying relies on delegation token support.+Without it, forwarded requests will authenticate as Solr server credentials instead of real-user credentials, and likely allowing authenticated-but-unauthorized users to query and index documents into your collections.=== Simple Authentication with Delegation Tokens-Similar to the previous example, this is an example of setting up a Solr cluster that uses delegation tokens. Refer to the parameters in the Hadoop authentication library's https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[documentation] or refer to the section <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> for further details. Please note that this example does not use Kerberos and the requests made to Solr must contain valid delegation tokens.+Similar to the previous example, this is an example of setting up a Solr cluster that uses delegation tokens.++Refer to the parameters in the Hadoop https://hadoop.apache.org/docs/stable/hadoop-auth/Configuration.html[authentication library's documentation] or refer to the section <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> for further details.-To setup this plugin, use the following in your `security.json` file.+Please note that this example does not use Kerberos and the requests made to Solr must contain valid delegation tokens.[source,json]----diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adocindex edd02282cb4..32dc185f7f5 100644--- a/solr/solr-ref-guide/src/highlighting.adoc+++ b/solr/solr-ref-guide/src/highlighting.adoc@@ -42,7 +42,7 @@ When using `*`, consider adding `hl.requireFieldMatch=true`.+Note that the field(s) listed here ought to have compatible text-analysis (defined in the schema) with field(s) referenced in the query to be highlighted.It may be necessary to modify `hl.q` and `hl.qparser` and/or modify the text analysis.-The following example uses the <<local-parameters-in-queries.adoc#,local-params>> syntax and <<the-extended-dismax-query-parser.adoc#,the edismax parser>> to highlight fields in `hl.fl`:+The following example uses the <<local-params.adoc#,local params>> syntax and <<edismax-query-parser.adoc#,the eDisMax parser>> to highlight fields in `hl.fl`:`&hl.fl=field1 field2&hl.q={!edismax qf=$hl.fl v=$q}&hl.qparser=lucene&hl.requireFieldMatch=true` (along with other applicable parameters, of course).+The default is the value of the `df` parameter which in turn has no default.@@ -55,7 +55,7 @@ When setting this, you might also need to set `hl.qparser`.The default is the value of the `q` parameter (already parsed).`hl.qparser`::-The <<query-syntax-and-parsing.adoc#,query parser>> to use for the `hl.q` query. It only applies when `hl.q` is set.+The <<query-syntax-and-parsers.adoc#,query parser>> to use for the `hl.q` query. It only applies when `hl.q` is set.+The default is the value of the `defType` parameter which in turn defaults to `lucene`.@@ -141,7 +141,7 @@ The `highlighting` section includes the ID of each document, and the field that== Choosing a Highlighter-Solr provides a `HighlightComponent` (a <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#defining-search-components,`SearchComponent`>>) and it's in the default list of components for search handlers. It offers a somewhat unified API over multiple actual highlighting implementations (or simply "highlighters") that do the business of highlighting.+Solr provides a `HighlightComponent` (a <<requesthandlers-searchcomponents.adoc#defining-search-components,`SearchComponent`>>) and it's in the default list of components for search handlers. It offers a somewhat unified API over multiple actual highlighting implementations (or simply "highlighters") that do the business of highlighting.There are many parameters supported by more than one highlighter, and sometimes the implementation details and semantics will be a bit different, so don't expect identical results when switching highlighters. You should use the `hl.method` parameter to choose a highlighter but it's also possible to explicitly configure an implementation by class name in `solrconfig.xml`.diff --git a/solr/solr-ref-guide/src/how-solrcloud-works.adoc b/solr/solr-ref-guide/src/how-solrcloud-works.adocdeleted file mode 100644index 19359084cc4..00000000000--- a/solr/solr-ref-guide/src/how-solrcloud-works.adoc+++ /dev/null@@ -1,54 +0,0 @@-= How SolrCloud Works-:page-children: shards-and-indexing-data-in-solrcloud, distributed-requests, aliases-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The following sections cover provide general information about how various SolrCloud features work. To understand these features, it's important to first understand a few key concepts that relate to SolrCloud.--* <<shards-and-indexing-data-in-solrcloud.adoc#,Shards and Indexing Data in SolrCloud>>-* <<distributed-requests.adoc#,Distributed Requests>>-* <<aliases.adoc#,Standard and Routed Aliases>>--If you are already familiar with SolrCloud concepts and basic functionality, you can skip to the section covering <<solrcloud-configuration-and-parameters.adoc#,SolrCloud Configuration and Parameters>>.--== Key SolrCloud Concepts--A SolrCloud cluster consists of some "logical" concepts layered on top of some "physical" concepts.--=== Logical Concepts--* A Cluster can host multiple Collections of Solr Documents.-* A collection can be partitioned into multiple Shards, which contain a subset of the Documents in the Collection.-* The number of Shards that a Collection has determines:-** The theoretical limit to the number of Documents that Collection can reasonably contain.-** The amount of parallelization that is possible for an individual search request.--=== Physical Concepts--* A Cluster is made up of one or more Solr Nodes, which are running instances of the Solr server process.-* Each Node can host multiple Cores.-* Each Core in a Cluster is a physical Replica for a logical Shard.-* Every Replica uses the same configuration specified for the Collection that it is a part of.-* The number of Replicas that each Shard has determines:-** The level of redundancy built into the Collection and how fault tolerant the Cluster can be in the event that some Nodes become unavailable.-** The theoretical limit in the number concurrent search requests that can be processed under heavy load.--WARNING: Make sure the DNS resolution in your cluster is stable, i.e.,-for each live host belonging to a Cluster the host name always corresponds to the-same specific IP and physical node. For example, in clusters deployed on AWS this would-require setting `preserve_hostname: true` in `/etc/cloud/cloud.cfg`. Changing DNS resolution-of live nodes may lead to unexpected errors. See SOLR-13159 for more details.diff --git a/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.png b/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.pngdeleted file mode 100644index 7c181b39909..00000000000Binary files a/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.png and /dev/null differdiff --git a/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.svg b/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.svgdeleted file mode 100644index 90adca0f13a..00000000000--- a/solr/solr-ref-guide/src/images/a-quick-overview/sample-client-app-arch.svg+++ /dev/null@@ -1,488 +0,0 @@-<?xml version="1.0" encoding="UTF-8" standalone="no"?>-<!-- Created with Inkscape (http://www.inkscape.org/) -->--<svg- xmlns:dc="http://purl.org/dc/elements/1.1/"- xmlns:cc="http://creativecommons.org/ns#"- xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"- xmlns:svg="http://www.w3.org/2000/svg"- xmlns="http://www.w3.org/2000/svg"- xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"- xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"- width="951.83209"- height="722.24835"- id="svg2"- version="1.1"- inkscape:version="0.48.3.1 r9886"- sodipodi:docname="arch.svg"- inkscape:export-filename="/home/hossman/tmp/arch.png"- inkscape:export-xdpi="90"- inkscape:export-ydpi="90">- <defs- id="defs4">- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="TriangleInL"- style="overflow:visible">- <path- id="path4575"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)"- inkscape:connector-curvature="0" />- </marker>- <marker- inkscape:stockid="TriangleOutL"- orient="auto"- refY="0"- refX="0"- id="TriangleOutL"- style="overflow:visible">- <path- id="path4584"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(0.8,0.8)"- inkscape:connector-curvature="0" />- </marker>- <marker- inkscape:stockid="Arrow2Lend"- orient="auto"- refY="0"- refX="0"- id="Arrow2Lend"- style="overflow:visible">- <path- id="path4463"- style="fill-rule:evenodd;stroke-width:0.625;stroke-linejoin:round"- d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"- transform="matrix(-1.1,0,0,-1.1,-1.1,0)"- inkscape:connector-curvature="0" />- </marker>- <clipPath- id="clipPath3037"- clipPathUnits="userSpaceOnUse">- <path- id="path3039"- d="m 0,102.654 203.005,0 L 203.005,0 0,0 0,102.654 z"- inkscape:connector-curvature="0" />- </clipPath>- <clipPath- id="clipPath3151"- clipPathUnits="userSpaceOnUse">- <path- id="path3153"- d="m 0,102.654 203.005,0 L 203.005,0 0,0 0,102.654 z"- inkscape:connector-curvature="0" />- </clipPath>- <clipPath- id="clipPath3151-9"- clipPathUnits="userSpaceOnUse">- <path- inkscape:connector-curvature="0"- id="path3153-7"- d="m 0,102.654 203.005,0 L 203.005,0 0,0 0,102.654 z" />- </clipPath>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="TriangleInL-6"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path4575-5"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="marker5640"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path5642"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="TriangleInL-2"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path4575-9"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="marker5640-2"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path5642-6"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="TriangleInL-0"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path4575-1"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- <marker- inkscape:stockid="TriangleInL"- orient="auto"- refY="0"- refX="0"- id="marker5640-8"- style="overflow:visible">- <path- inkscape:connector-curvature="0"- id="path5642-2"- d="m 5.77,0 -8.65,5 0,-10 8.65,5 z"- style="fill-rule:evenodd;stroke:#000000;stroke-width:1pt"- transform="scale(-0.8,-0.8)" />- </marker>- </defs>- <sodipodi:namedview- id="base"- pagecolor="#ffffff"- bordercolor="#666666"- borderopacity="1.0"- inkscape:pageopacity="0.0"- inkscape:pageshadow="2"- inkscape:zoom="0.50362889"- inkscape:cx="500.2462"- inkscape:cy="305.15545"- inkscape:document-units="px"- inkscape:current-layer="layer1"- showgrid="false"- inkscape:object-nodes="true"- inkscape:window-width="1025"- inkscape:window-height="709"- inkscape:window-x="125"- inkscape:window-y="99"- inkscape:window-maximized="0"- fit-margin-top="50"- fit-margin-right="50"- fit-margin-bottom="50"- fit-margin-left="50" />- <metadata- id="metadata7">- <rdf:RDF>- <cc:Work- rdf:about="">- <dc:format>image/svg+xml</dc:format>- <dc:type- rdf:resource="http://purl.org/dc/dcmitype/StillImage" />- <dc:title></dc:title>- </cc:Work>- </rdf:RDF>- </metadata>- <g- inkscape:label="Layer 1"- inkscape:groupmode="layer"- id="layer1"- transform="translate(185.02727,-109.08811)">- <g- id="g3236"- transform="translate(-255.1482,174.73183)">- <g- id="g3145"- inkscape:label="Solr_Logo_on_white"- transform="matrix(1.25,0,0,-1.25,160.03949,571.47669)">- <g- id="g3147">- <g- id="g3149"- clip-path="url(#clipPath3151)">- <g- id="g3155"- transform="translate(40.7606,31.7631)">- <path- inkscape:connector-curvature="0"- d="m 0,0 c -2.345,1.248 -4.995,2.127 -7.875,2.613 -2.829,0.476 -5.707,0.717 -8.554,0.717 -2.327,0 -4.681,0.201 -6.998,0.597 -2.256,0.384 -4.299,1.092 -6.076,2.104 -1.729,0.984 -3.159,2.361 -4.252,4.095 -1.078,1.706 -1.624,3.968 -1.624,6.697 0.052,2.41 0.612,4.472 1.666,6.129 1.068,1.684 2.464,3.052 4.147,4.067 1.726,1.041 3.718,1.806 5.921,2.274 3.751,0.796 7.713,0.945 11.749,0.382 1.575,-0.222 3.133,-0.615 4.63,-1.168 1.47,-0.545 2.852,-1.292 4.106,-2.217 1.236,-0.915 2.328,-2.063 3.246,-3.411 L 0.6,22.124 4.388,23.718 3.584,24.853 c -1.031,1.456 -2.173,2.767 -3.395,3.899 -1.247,1.151 -2.712,2.13 -4.356,2.906 -1.623,0.765 -3.48,1.354 -5.522,1.751 -2.021,0.392 -4.377,0.59 -7.003,0.59 -2.519,0 -5.12,-0.288 -7.73,-0.858 -2.649,-0.58 -5.106,-1.538 -7.301,-2.849 -2.242,-1.34 -4.09,-3.125 -5.493,-5.306 -1.425,-2.219 -2.149,-4.957 -2.149,-8.139 0,-3.34 0.671,-6.183 1.993,-8.449 1.314,-2.252 3.076,-4.077 5.237,-5.423 2.116,-1.32 4.585,-2.269 7.338,-2.82 2.69,-0.538 5.505,-0.811 8.368,-0.811 2.276,0 4.686,-0.172 7.164,-0.515 2.411,-0.333 4.655,-1.005 6.669,-1.997 1.962,-0.969 3.597,-2.316 4.857,-4.005 1.219,-1.635 1.838,-3.843 1.838,-6.562 0,-2.442 -0.596,-4.524 -1.769,-6.193 -1.212,-1.724 -2.786,-3.147 -4.679,-4.229 -1.934,-1.105 -4.123,-1.919 -6.51,-2.418 -2.425,-0.508 -4.825,-0.765 -7.132,-0.765 -3.873,0 -7.7,0.74 -11.374,2.2 -3.658,1.451 -6.902,3.632 -9.64,6.482 l -0.735,0.765 -3.021,-2.618 0.848,-0.848 c 2.703,-2.705 6.06,-5.06 9.975,-7.002 3.963,-1.967 8.656,-2.964 13.947,-2.964 2.516,0 5.186,0.288 7.938,0.855 2.79,0.577 5.408,1.562 7.78,2.931 2.41,1.39 4.415,3.209 5.962,5.406 1.589,2.257 2.395,5.081 2.395,8.398 0,3.361 -0.755,6.207 -2.245,8.459 C 4.368,-3.056 2.404,-1.28 0,0"- style="fill:#241f2b;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3157" />- </g>- <g- id="g3159"- transform="translate(76.3898,3.9853)">- <path- inkscape:connector-curvature="0"- d="m 0,0 c -2.732,0 -5.23,0.564 -7.426,1.676 -2.225,1.125 -4.189,2.657 -5.839,4.556 -1.659,1.907 -2.977,4.137 -3.917,6.629 -0.947,2.503 -1.484,5.127 -1.593,7.754 0,2.429 0.45,4.911 1.336,7.378 0.885,2.459 2.174,4.702 3.833,6.666 1.654,1.957 3.667,3.578 5.984,4.82 2.278,1.22 4.842,1.839 7.622,1.839 2.614,0 5.083,-0.579 7.338,-1.719 2.286,-1.157 4.297,-2.707 5.978,-4.607 1.685,-1.908 3.031,-4.139 4.002,-6.632 0.967,-2.49 1.457,-5.096 1.457,-7.745 0,-2.426 -0.45,-4.909 -1.335,-7.377 C 16.553,10.774 15.264,8.532 13.606,6.57 11.955,4.617 9.945,3.012 7.631,1.8 5.352,0.605 2.785,0 0,0 m 16.577,37.377 c -1.982,2.316 -4.391,4.232 -7.162,5.694 -2.809,1.482 -5.977,2.232 -9.415,2.232 -3.132,0 -6.124,-0.671 -8.893,-1.993 -2.749,-1.314 -5.18,-3.126 -7.225,-5.384 -2.037,-2.249 -3.67,-4.898 -4.854,-7.873 -1.186,-2.979 -1.788,-6.155 -1.788,-9.438 0,-3.103 0.555,-6.14 1.65,-9.027 1.093,-2.885 2.649,-5.503 4.625,-7.784 1.985,-2.291 4.399,-4.164 7.175,-5.566 2.797,-1.414 5.923,-2.162 9.29,-2.223 l 0.021,0 0.021,0 c 3.186,0.061 6.203,0.778 8.967,2.128 2.744,1.34 5.172,3.166 7.217,5.424 2.037,2.25 3.656,4.87 4.81,7.787 1.157,2.924 1.744,6.04 1.744,9.261 0,2.917 -0.539,5.89 -1.6,8.835 -1.063,2.945 -2.605,5.612 -4.583,7.927"- style="fill:#241f2b;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3161" />- </g>- <g- id="g3163"- transform="translate(119.7832,5.0906)">- <path- inkscape:connector-curvature="0"- d="m 0,0 c -0.67,-0.167 -1.26,-0.294 -1.752,-0.376 -0.518,-0.086 -1.08,-0.173 -1.685,-0.259 -0.557,-0.079 -1.108,-0.12 -1.638,-0.12 -1.337,0 -2.321,0.443 -3.01,1.355 -0.761,1.004 -1.131,2.001 -1.131,3.049 l 0,58.337 -3.985,0 0,-58.337 c 0,-2.189 0.691,-4.126 2.055,-5.754 1.413,-1.691 3.426,-2.547 5.983,-2.547 0.802,0 1.598,0.047 2.367,0.139 0.742,0.089 1.409,0.178 2.006,0.267 0.61,0.092 1.316,0.23 2.1,0.41 L 2.661,-3.524 0.887,0.222 0,0 z"- style="fill:#241f2b;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3165" />- </g>- <g- id="g3167"- transform="translate(136.9779,45.2734)">- <path- inkscape:connector-curvature="0"- d="m 0,0 c -2.073,-1.457 -3.904,-3.307 -5.471,-5.522 l 0,8.836 -3.985,0 0,-47.711 3.985,0 0,29.704 c 0.552,1.946 1.312,3.771 2.261,5.426 0.957,1.667 2.148,3.135 3.538,4.362 1.39,1.226 3.008,2.221 4.811,2.958 1.798,0.736 3.827,1.166 6.031,1.277 l 1.06,0.053 0,3.931 -1.116,0 C 6.867,3.314 3.128,2.199 0,0"- style="fill:#241f2b;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3169" />- </g>- <g- id="g3171"- transform="translate(193.9115,92.0414)">- <path- inkscape:connector-curvature="0"- d="M 0,0 -33,-35.677 8.473,-16.543 C 7.246,-10.213 4.242,-4.521 0,0"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3173" />- </g>- <g- id="g3175"- transform="translate(169.4268,102.6536)">- <path- inkscape:connector-curvature="0"- d="M 0,0 C -4.572,0 -8.928,-0.917 -12.9,-2.572 L -17.328,-39.886 4.799,-0.347 C 3.231,-0.122 1.63,0 0,0"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3177" />- </g>- <g- id="g3179"- transform="translate(202.7133,73.4428)">- <path- inkscape:connector-curvature="0"- d="m 0,0 -39.298,-21.992 36.87,4.375 c 1.748,4.065 2.72,8.543 2.72,13.249 C 0.292,-2.886 0.186,-1.431 0,0"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3181" />- </g>- <g- id="g3183"- transform="translate(188.2247,41.2478)">- <path- inkscape:connector-curvature="0"- d="M 0,0 C 4.391,2.972 8.037,6.958 10.612,11.619 L -23.947,4.756 0,0 z"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3185" />- </g>- <g- id="g3187"- transform="translate(176.2492,101.9586)">- <path- inkscape:connector-curvature="0"- d="M 0,0 -19.237,-41.695 16.448,-8.689 C 11.973,-4.384 6.313,-1.303 0,0"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3189" />- </g>- <g- id="g3191"- transform="translate(172.2334,35.622)">- <path- inkscape:connector-curvature="0"- d="M 0,0 C 3.718,0.308 7.264,1.217 10.54,2.638 L -8.818,4.935 0,0 z"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3193" />- </g>- <g- id="g3195"- transform="translate(138.8186,82.8789)">- <path- inkscape:connector-curvature="0"- d="M 0,0 C -1.51,-3.343 -2.489,-6.974 -2.829,-10.792 L 2.386,-20.112 0,0 z"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3197" />- </g>- <g- id="g3199"- transform="translate(153.617,98.7011)">- <path- inkscape:connector-curvature="0"- d="M 0,0 C -4.738,-2.534 -8.794,-6.17 -11.832,-10.567 L -6.965,-35.071 0,0 z"- style="fill:#da3522;fill-opacity:1;fill-rule:nonzero;stroke:none"- id="path3201" />- </g>- </g>- </g>- </g>- <rect- y="409.03134"- x="121.12093"- height="196.57332"- width="331.59335"- id="rect3234"- style="opacity:0.79680008;fill:none;stroke:#000000;stroke-width:2;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />- </g>- <g- id="g4393"- transform="translate(-254.1554,71.23301)">- <rect- y="90.344292"- x="120.12814"- height="196.57332"- width="331.59335"- id="rect3234-0"- style="fill:#0008de;fill-opacity:0.25531915;stroke:#000000;stroke-width:2;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />- <text- sodipodi:linespacing="125%"- id="text3402"- y="149.31458"- x="284.4502"- style="font-size:40px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"- xml:space="preserve"><tspan- y="149.31458"- x="284.4502"- id="tspan3404"- sodipodi:role="line">Content</tspan><tspan- id="tspan3406"- y="199.31458"- x="284.4502"- sodipodi:role="line">Management</tspan><tspan- id="tspan3408"- y="249.31458"- x="284.4502"- sodipodi:role="line">System</tspan></text>- </g>- <g- id="g4400"- transform="translate(301.80953,-76.445144)">- <text- sodipodi:linespacing="125%"- id="text3435"- y="744.53015"- x="249.76111"- style="font-size:40px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"- xml:space="preserve"><tspan- y="744.53015"- x="249.76111"- id="tspan3437"- sodipodi:role="line">End User</tspan><tspan- id="tspan3439"- y="794.53015"- x="249.76111"- sodipodi:role="line">Application</tspan></text>- <rect- y="660.20831"- x="82.401939"- height="196.57332"- width="331.59335"- id="rect3234-0-6-4"- style="fill:#0008de;fill-opacity:0.25531915;stroke:#000000;stroke-width:2;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none" />- </g>- <g- id="g4425"- transform="translate(757.254,-37.477991)">- <path- inkscape:transform-center-y="194.58772"- inkscape:transform-center-x="45.668548"- d="m -101.26504,410.02414 c 0,10.4178 -47.56035,18.8631 -106.22901,18.8631 -58.66866,0 -106.22901,-8.4453 -106.22901,-18.8631 0,-10.4178 47.56035,-18.8631 106.22901,-18.8631 58.66866,0 106.22901,8.4453 106.22901,18.8631 z"- sodipodi:ry="18.863096"- sodipodi:rx="106.22901"- sodipodi:cy="410.02414"- sodipodi:cx="-207.49405"- id="path3502-5"- style="opacity:0.79680008;fill:none;stroke:#000000;stroke-width:2;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"- sodipodi:type="arc"- transform="translate(0.24819563,-193.59494)" />- <path- sodipodi:open="true"- sodipodi:end="3.1415927"- sodipodi:start="0"- inkscape:transform-center-y="194.58772"- inkscape:transform-center-x="45.668548"- d="m -101.26504,410.02414 c 0,10.4178 -47.56035,18.8631 -106.22901,18.8631 -58.66866,0 -106.22901,-8.4453 -106.22901,-18.8631 0,0 0,0 0,0"- sodipodi:ry="18.863096"- sodipodi:rx="106.22901"- sodipodi:cy="410.02414"- sodipodi:cx="-207.49405"- id="path3502-5-4"- style="opacity:0.79680008;fill:none;stroke:#000000;stroke-width:2;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none"- sodipodi:type="arc"- transform="translate(0.24819563,-31.769434)" />- <path- inkscape:connector-curvature="0"- id="path4371"- d="m -313.47486,216.4292 c 0,161.82551 0,161.82551 0,161.82551"- style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />- <path- inkscape:connector-curvature="0"- id="path4373"- d="m -101.01684,216.4292 c 0,161.82551 0,161.82551 0,161.82551"- style="fill:none;stroke:#000000;stroke-width:1px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1" />- <path- inkscape:connector-curvature="0"- id="path4377"- d="m -243.16769,394.36964 c -34.17364,-2.31715 -64.43561,-9.27295 -66.76621,-15.34641 -0.34116,-0.88903 -0.62029,-36.66066 -0.62029,-79.4925 l 0,-77.87606 2.23379,1.4223 c 6.48455,4.12882 24.60949,8.18861 49.93322,11.1845 21.31886,2.5221 84.81991,2.52073 106.18372,-0.002 24.57791,-2.9026 40.06909,-6.37707 48.98167,-10.98594 l 3.14004,-1.62378 0,77.82597 c 0,76.61463 -0.0313,77.86572 -2.00819,80.37898 -5.00987,6.36901 -31.35126,12.054 -67.48743,14.5651 -16.40633,1.14008 -56.44098,1.11294 -73.59032,-0.0498 l 0,-3e-5 z"- style="fill:#00da00;fill-opacity:0.25531915;stroke:none" />- <path- inkscape:connector-curvature="0"- id="path4379"- d="m -243.16769,232.54414 c -33.27902,-2.25649 -64.58959,-9.3163 -66.6709,-15.03275 -1.48692,-4.08393 12.30392,-9.90261 31.05383,-13.10232 37.23448,-6.35415 101.85345,-6.78769 140.48042,-0.94251 19.32495,2.92433 35.44716,8.07086 36.86603,11.76837 2.72049,7.08948 -26.13239,14.44003 -68.13906,17.35907 -16.40633,1.14008 -56.44098,1.11295 -73.59032,-0.0499 z"- style="fill:#00da00;fill-opacity:0.25531915;stroke:none" />- <text- sodipodi:linespacing="125%"- id="text4381"- y="286.60049"- x="-205.57187"- style="font-size:40px;font-style:normal;font-weight:normal;text-align:center;line-height:125%;letter-spacing:0px;word-spacing:0px;text-anchor:middle;fill:#000000;fill-opacity:1;stroke:none;font-family:Sans"- xml:space="preserve"><tspan- y="286.60049"- x="-205.57187"- id="tspan4383"- sodipodi:role="line">Data</tspan><tspan- id="tspan4385"- y="336.60049"- x="-205.57187"- sodipodi:role="line">Source</tspan></text>- </g>- <path- style="fill:none;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-start:url(#TriangleInL);marker-end:url(#TriangleInL)"- d="m 31.769415,398.4829 c 0,146.93359 0,144.948 0,144.948"- id="path4436"- inkscape:connector-curvature="0" />- <path- style="fill:none;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-start:none;marker-end:url(#TriangleInL)"- d="m 550.00815,398.48291 c 0,146.93359 0,144.948 0,144.948"- id="path4436-9"- inkscape:connector-curvature="0" />- <path- style="fill:none;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-start:url(#TriangleInL);marker-end:url(#TriangleInL)"- d="m 396.12501,259.86396 c -146.93359,0 -144.948,0 -144.948,0"- id="path4436-0"- inkscape:connector-curvature="0" />- <path- style="fill:none;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:none;marker-start:none;marker-end:url(#TriangleInL)"- d="m 208.19,682.04983 c 146.93359,0 144.948,0 144.948,0"- id="path4436-6"- inkscape:connector-curvature="0" />- </g>-</svg>diff --git a/solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_0.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_0.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_0.pngrename to solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_0.pngdiff --git a/solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_1.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_1.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_1.pngrename to solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_1.pngdiff --git a/solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_2.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_2.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_2.pngrename to solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_2.pngdiff --git a/solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_3.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_3.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_3.pngrename to solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_3.pngdiff --git a/solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_4.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_4.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-your-analyzer/analysis_compare_4.pngrename to solr/solr-ref-guide/src/images/analysis-screen/analysis_compare_4.pngdiff --git a/solr/solr-ref-guide/src/images/analysis-screen/analysis_normal.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_normal.pngindex f180ca988dc..a3d2097be76 100644Binary files a/solr/solr-ref-guide/src/images/analysis-screen/analysis_normal.png and b/solr/solr-ref-guide/src/images/analysis-screen/analysis_normal.png differdiff --git a/solr/solr-ref-guide/src/images/analysis-screen/analysis_verbose.png b/solr/solr-ref-guide/src/images/analysis-screen/analysis_verbose.pngdeleted file mode 100644index 13f8fc744d4..00000000000Binary files a/solr/solr-ref-guide/src/images/analysis-screen/analysis_verbose.png and /dev/null differdiff --git a/solr/solr-ref-guide/src/images/combining-distribution-and-replication/distributed-replication.png b/solr/solr-ref-guide/src/images/combining-distribution-and-replication/distributed-replication.pngdeleted file mode 100644index 49af2a67ae4..00000000000Binary files a/solr/solr-ref-guide/src/images/combining-distribution-and-replication/distributed-replication.png and /dev/null differdiff --git a/solr/solr-ref-guide/src/images/files-screen/files-screen.png b/solr/solr-ref-guide/src/images/configuration-files/files-screen.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/files-screen/files-screen.pngrename to solr/solr-ref-guide/src/images/configuration-files/files-screen.pngdiff --git a/solr/solr-ref-guide/src/images/logging/level_menu.png b/solr/solr-ref-guide/src/images/configuring-logging/level_menu.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/logging/level_menu.pngrename to solr/solr-ref-guide/src/images/configuring-logging/level_menu.pngdiff --git a/solr/solr-ref-guide/src/images/logging/logging.png b/solr/solr-ref-guide/src/images/configuring-logging/logging.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/logging/logging.pngrename to solr/solr-ref-guide/src/images/configuring-logging/logging.pngdiff --git a/solr/solr-ref-guide/src/images/solr-tracing/query-request-tracing.png b/solr/solr-ref-guide/src/images/distributed-tracing/query-request-tracing.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-tracing/query-request-tracing.pngrename to solr/solr-ref-guide/src/images/distributed-tracing/query-request-tracing.pngdiff --git a/solr/solr-ref-guide/src/images/index-replication/leader-follower-replication.png b/solr/solr-ref-guide/src/images/index-replication/leader-follower-replication.pngdeleted file mode 100644index f38ada53275..00000000000Binary files a/solr/solr-ref-guide/src/images/index-replication/leader-follower-replication.png and /dev/null differdiff --git a/solr/solr-ref-guide/src/images/segments-info/segments_info.png b/solr/solr-ref-guide/src/images/index-segments-merging/segments_info.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/segments-info/segments_info.pngrename to solr/solr-ref-guide/src/images/index-segments-merging/segments_info.pngdiff --git a/solr/solr-ref-guide/src/images/solr-cell/sample-pdf-query.png b/solr/solr-ref-guide/src/images/indexing-with-tika/sample-pdf-query.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-cell/sample-pdf-query.pngrename to solr/solr-ref-guide/src/images/indexing-with-tika/sample-pdf-query.pngdiff --git a/solr/solr-ref-guide/src/images/running-solr/SolrAdminDashboard.png b/solr/solr-ref-guide/src/images/installing-solr/SolrAdminDashboard.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/running-solr/SolrAdminDashboard.pngrename to solr/solr-ref-guide/src/images/installing-solr/SolrAdminDashboard.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.png b/solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.pngrename to solr/solr-ref-guide/src/images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_1.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_1.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_1.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_1.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_10.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_10.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_10.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_10.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_11.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_11.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_11.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_11.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_12.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_12.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_12.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_12.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_13.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_13.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_13.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_13.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_14.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_14.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_14.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_14.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_15.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_15.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_15.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_15.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_2.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_2.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_2.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_2.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_3.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_3.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_3.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_3.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_4.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_4.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_4.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_4.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_5.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_5.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_5.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_5.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_7.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_7.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_7.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_7.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_9.png b/solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_9.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_9.pngrename to solr/solr-ref-guide/src/images/jdbc-squirrel/squirrelsql_solrjdbc_9.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_1.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_1.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_1.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_1.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_2.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_2.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_2.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_2.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_3.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_3.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_3.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_3.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_4.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_4.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_4.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_4.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_5.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_5.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_5.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_5.pngdiff --git a/solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_6.png b/solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_6.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_6.pngrename to solr/solr-ref-guide/src/images/jdbc-zeppelin/zeppelin_solrjdbc_6.pngdiff --git a/solr/solr-ref-guide/src/images/java-properties/javaproperties.png b/solr/solr-ref-guide/src/images/jvm-settings/javaproperties.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/java-properties/javaproperties.pngrename to solr/solr-ref-guide/src/images/jvm-settings/javaproperties.pngdiff --git a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png b/solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/grafana-solr-dashboard.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.pngrename to solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/grafana-solr-dashboard.pngdiff --git a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/prometheus-solr-ping.png b/solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/prometheus-solr-ping.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/prometheus-solr-ping.pngrename to solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/prometheus-solr-ping.pngdiff --git a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png b/solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/solr-exporter-diagram.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.pngrename to solr/solr-ref-guide/src/images/monitoring-with-prometheus-and-grafana/solr-exporter-diagram.pngdiff --git a/solr/solr-ref-guide/src/images/running-solr/solr34_responseHeader.png b/solr/solr-ref-guide/src/images/running-solr/solr34_responseHeader.pngdeleted file mode 100644index 27d5d6be7a7..00000000000Binary files a/solr/solr-ref-guide/src/images/running-solr/solr34_responseHeader.png and /dev/null differdiff --git a/solr/solr-ref-guide/src/images/overview-of-searching-in-solr/worddav88969a784fb8a63d8c46e9c043f5f953.png b/solr/solr-ref-guide/src/images/searching-in-solr/cnet-faceting.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/overview-of-searching-in-solr/worddav88969a784fb8a63d8c46e9c043f5f953.pngrename to solr/solr-ref-guide/src/images/searching-in-solr/cnet-faceting.pngdiff --git a/solr/solr-ref-guide/src/images/overview-of-searching-in-solr/worddav16392965e726e04513a21641fabad474.png b/solr/solr-ref-guide/src/images/searching-in-solr/search-process.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/overview-of-searching-in-solr/worddav16392965e726e04513a21641fabad474.pngrename to solr/solr-ref-guide/src/images/searching-in-solr/search-process.pngdiff --git a/solr/solr-ref-guide/src/images/getting-assistance/Assistance.png b/solr/solr-ref-guide/src/images/solr-admin-ui/Assistance.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/getting-assistance/Assistance.pngrename to solr/solr-ref-guide/src/images/solr-admin-ui/Assistance.pngdiff --git a/solr/solr-ref-guide/src/images/collection-specific-tools/collection_dashboard.png b/solr/solr-ref-guide/src/images/solr-admin-ui/collection_dashboard.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/collection-specific-tools/collection_dashboard.pngrename to solr/solr-ref-guide/src/images/solr-admin-ui/collection_dashboard.pngdiff --git a/solr/solr-ref-guide/src/images/core-specific-tools/core_dashboard.png b/solr/solr-ref-guide/src/images/solr-admin-ui/core_dashboard.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/core-specific-tools/core_dashboard.pngrename to solr/solr-ref-guide/src/images/solr-admin-ui/core_dashboard.pngdiff --git a/solr/solr-ref-guide/src/images/overview-of-the-solr-admin-ui/dashboard.png b/solr/solr-ref-guide/src/images/solr-admin-ui/dashboard.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/overview-of-the-solr-admin-ui/dashboard.pngrename to solr/solr-ref-guide/src/images/solr-admin-ui/dashboard.pngdiff --git a/solr/solr-ref-guide/src/images/overview-of-the-solr-admin-ui/login.png b/solr/solr-ref-guide/src/images/solr-admin-ui/login.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/overview-of-the-solr-admin-ui/login.pngrename to solr/solr-ref-guide/src/images/solr-admin-ui/login.pngdiff --git a/solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-key.png b/solr/solr-ref-guide/src/images/tutorial-aws/aws-key.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-key.pngrename to solr/solr-ref-guide/src/images/tutorial-aws/aws-key.pngdiff --git a/solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-security-create.png b/solr/solr-ref-guide/src/images/tutorial-aws/aws-security-create.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-security-create.pngrename to solr/solr-ref-guide/src/images/tutorial-aws/aws-security-create.pngdiff --git a/solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-security-edit.png b/solr/solr-ref-guide/src/images/tutorial-aws/aws-security-edit.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/aws-solrcloud-tutorial/aws-security-edit.pngrename to solr/solr-ref-guide/src/images/tutorial-aws/aws-security-edit.pngdiff --git a/solr/solr-ref-guide/src/images/replication-screen/replication.png b/solr/solr-ref-guide/src/images/user-managed-index-replication/replication.pngsimilarity index 100%rename from solr/solr-ref-guide/src/images/replication-screen/replication.pngrename to solr/solr-ref-guide/src/images/user-managed-index-replication/replication.pngdiff --git a/solr/solr-ref-guide/src/images/user-managed-index-replication/user-managed-replication.png b/solr/solr-ref-guide/src/images/user-managed-index-replication/user-managed-replication.pngnew file mode 100644index 00000000000..bd30a052e40Binary files /dev/null and b/solr/solr-ref-guide/src/images/user-managed-index-replication/user-managed-replication.png differdiff --git a/solr/solr-ref-guide/src/implicit-requesthandlers.adoc b/solr/solr-ref-guide/src/implicit-requesthandlers.adocindex 3370cbcefe4..0c5c9d099d0 100644--- a/solr/solr-ref-guide/src/implicit-requesthandlers.adoc+++ b/solr/solr-ref-guide/src/implicit-requesthandlers.adoc@@ -1,4 +1,4 @@-= Implicit RequestHandlers+= Implicit Request Handlers// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,7 @@// specific language governing permissions and limitations// under the License.-Solr ships with many out-of-the-box RequestHandlers, which are called implicit because they do not need to be configured in `solrconfig.xml` before you are able to use them.+Solr ships with many out-of-the-box request handlers, which are called implicit because they do not need to be configured in `solrconfig.xml` before you are able to use them.These handlers have pre-defined default parameters, known as _paramsets_, which can be modified if necessary.@@ -29,7 +29,7 @@ NOTE: All endpoint paths listed below should be placed after Solr's host and porMany of these handlers are used throughout the Admin UI to show information about Solr.[horizontal]-File:: Returns content of files in `${solr.home}/conf/`. This handler must have a collection name in the path to the endpoint.+File:: Return content of files in `${solr.home}/conf/`. This handler must have a collection name in the path to the endpoint.+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -37,7 +37,7 @@ File:: Returns content of files in `${solr.home}/conf/`. This handler must have|`solr/<collection>/admin/file` |{solr-javadocs}/core/org/apache/solr/handler/admin/ShowFileRequestHandler.html[ShowFileRequestHandler] |`_ADMIN_FILE`|===-Health:: Reporting the health of the node (_available only in SolrCloud mode_)+Health:: Report the health of the node (_available only in SolrCloud mode_)+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -69,7 +69,6 @@ Luke:: Expose the internal Lucene index. This handler must have a collection nam|`solr/<collection>/admin/luke` |{solr-javadocs}/core/org/apache/solr/handler/admin/LukeRequestHandler.html[LukeRequestHandler] |`_ADMIN_LUKE`|===-MBeans:: Provide info about all registered {solr-javadocs}/core/org/apache/solr/core/SolrInfoBean.html[SolrInfoMBeans]. This handler must have a collection name in the path to the endpoint.+*Documentation*: <<mbean-request-handler.adoc#,MBean Request Handler>>@@ -182,7 +181,8 @@ Dump:: Echo the request contents back to the client.|`solr/debug/dump` |{solr-javadocs}/core/org/apache/solr/handler/DumpRequestHandler.html[DumpRequestHandler] |`_DEBUG_DUMP`|===-Replication:: Replicate indexes for SolrCloud recovery and Leader/Follower index distribution. This handler must have a core name in the path to the endpoint.+Replication:: Replicate indexes for SolrCloud recovery and user-managed cluster index distribution.+This handler must have a core name in the path to the endpoint.+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -257,7 +257,7 @@ Streaming Expressions:: Distributed stream processing.Terms:: Return a field's indexed terms and the number of documents containing each term.+-*Documentation*: <<the-terms-component.adoc#using-the-terms-component-in-a-request-handler,Using the Terms Component in a Request Handler>>+*Documentation*: <<terms-component.adoc#using-the-terms-component-in-a-request-handler,Using the Terms Component in a Request Handler>>+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -270,7 +270,7 @@ Terms:: Return a field's indexed terms and the number of documents containing ea[horizontal]Update:: Add, delete and update indexed documents formatted as SolrXML, CSV, SolrJSON or javabin.+-*Documentation*: <<uploading-data-with-index-handlers.adoc#,Uploading Data with Index Handlers>>+*Documentation*: <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -280,7 +280,7 @@ Update:: Add, delete and update indexed documents formatted as SolrXML, CSV, SolCSV Updates:: Add and update CSV-formatted documents.+-*Documentation*: <<uploading-data-with-index-handlers.adoc#csv-update-convenience-paths,CSV Update Convenience Paths>>+*Documentation*: <<indexing-with-update-handlers.adoc#csv-update-convenience-paths,CSV Update Convenience Paths>>+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -290,7 +290,7 @@ CSV Updates:: Add and update CSV-formatted documents.JSON Updates:: Add, delete and update SolrJSON-formatted documents.+-*Documentation*: <<uploading-data-with-index-handlers.adoc#json-update-convenience-paths,JSON Update Convenience Paths>>+*Documentation*: <<indexing-with-update-handlers.adoc#json-update-convenience-paths,JSON Update Convenience Paths>>+[cols="3*.",frame=none,grid=cols,options="header"]|===@@ -312,8 +312,9 @@ Custom JSON Updates:: Add and update custom JSON-formatted documents.You can see configuration for all request handlers, including the implicit request handlers, via the <<config-api.adoc#,Config API>>.-To include the expanded paramset in the response, as well as the effective parameters from merging the paramset parameters with the built-in parameters, use the `expandParams` request parameter. For the `/export` request handler, you can make a request like this:+To include the expanded paramset in the response, as well as the effective parameters from merging the paramset parameters with the built-in parameters, use the `expandParams` request parameter.+For example, for the `/export` request handler, you can make a request like this:[.dynamic-tabs]--@@ -374,4 +375,6 @@ The response will look similar to:== How to Edit Implicit Handler Paramsets-Because implicit request handlers are not present in `solrconfig.xml`, configuration of their associated `default`, `invariant` and `appends` parameters may be edited via the <<request-parameters-api.adoc#, Request Parameters API>> using the paramset listed in the above table. However, other parameters, including SearchHandler components, may not be modified. The invariants and appends specified in the implicit configuration cannot be overridden.+Because implicit request handlers are not present in `solrconfig.xml`, configuration of their associated `default`, `invariant` and `appends` parameters may be edited via the <<request-parameters-api.adoc#, Request Parameters API>> using the paramset listed in the above table.+However, other parameters, including SearchHandler components, may not be modified.+The invariants and appends specified in the implicit configuration cannot be overridden.diff --git a/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc b/solr/solr-ref-guide/src/index-location-format.adocsimilarity index 89%rename from solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adocrename to solr/solr-ref-guide/src/index-location-format.adocindex 98cdb968fb7..4370356ccd1 100644--- a/solr/solr-ref-guide/src/datadir-and-directoryfactory-in-solrconfig.adoc+++ b/solr/solr-ref-guide/src/index-location-format.adoc@@ -1,4 +1,4 @@-= DataDir and DirectoryFactory in SolrConfig+= Index Location and Format// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -29,7 +29,7 @@ By default, Solr stores its index data in a directory called `/data` under the cThe `${solr.core.name}` substitution will cause the name of the current core to be substituted, which results in each core's data being kept in a separate subdirectory.-If you are using replication to replicate the Solr index (as described in <<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>), then the `<dataDir>` directory should correspond to the index directory used in the replication configuration.+If you are using <<user-managed-index-replication#,user-managed replication>> to replicate the Solr index, then the `<dataDir>` directory should correspond to the index directory used in the replication configuration.NOTE: If the environment variable `SOLR_DATA_HOME` is defined, or if `solr.data.home` is configured for your DirectoryFactory, or if `solr.xml` contains anelement `<solrDataHome>` then the location of data directory will be `<SOLR_DATA_HOME>/<instance_name>/data`.@@ -55,5 +55,5 @@ The {solr-javadocs}/core/org/apache/solr/core/RAMDirectoryFactory.html[`solr.RAM[NOTE]====-If you are using Hadoop and would like to store your indexes in HDFS, you should use the {solr-javadocs}/core/org/apache/solr/core/HdfsDirectoryFactory.html[`solr.HdfsDirectoryFactory`] instead of either of the above implementations. For more details, see the section <<running-solr-on-hdfs.adoc#,Running Solr on HDFS>>.+If you are using Hadoop and would like to store your indexes in HDFS, you should use the {solr-javadocs}/core/org/apache/solr/core/HdfsDirectoryFactory.html[`solr.HdfsDirectoryFactory`] instead of either of the above implementations. For more details, see the section <<solr-on-hdfs.adoc#,Solr on HDFS>>.====diff --git a/solr/solr-ref-guide/src/index-replication.adoc b/solr/solr-ref-guide/src/index-replication.adocdeleted file mode 100644index 738767dac08..00000000000--- a/solr/solr-ref-guide/src/index-replication.adoc+++ /dev/null@@ -1,384 +0,0 @@-= Index Replication-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Index Replication distributes complete copies of a leader index to one or more follower servers. The leader server continues to manage updates to the index. All querying is handled by the followers. This division of labor enables Solr to scale to provide adequate responsiveness to queries against large search volumes.--The figure below shows a Solr configuration using index replication. The leader server's index is replicated on the followers.--.A Solr index can be replicated across multiple follower servers, which then process requests.-image::images/index-replication/leader-follower-replication.png[image,width=203,height=235]---== Index Replication in Solr--Solr includes a Java implementation of index replication that works over HTTP:--* The configuration affecting replication is controlled by a single file, `solrconfig.xml`-* Supports the replication of configuration files as well as index files-* Works across platforms with same configuration-* No reliance on OS-dependent file system features (e.g., hard links)-* Tightly integrated with Solr; an admin page offers fine-grained control of each aspect of replication-* The Java-based replication feature is implemented as a request handler. Configuring replication is therefore similar to any normal request handler.--.Replication In SolrCloud-[NOTE]-====-Although there is no explicit concept of "leader/follower" nodes in a <<solrcloud.adoc#,SolrCloud>> cluster, the `ReplicationHandler` discussed on this page is still used by SolrCloud as needed to support "shard recovery" – but this is done in a peer to peer manner.--When using SolrCloud, the `ReplicationHandler` must be available via the `/replication` path. Solr does this implicitly unless overridden explicitly in your `solrconfig.xml`, but if you wish to override the default behavior, make certain that you do not explicitly set any of the "leader" or "follower" configuration options mentioned below, or they will interfere with normal SolrCloud operation.-====--== Replication Terminology--The table below defines the key terms associated with Solr replication.--Index::-A Lucene index is a directory of files. These files make up the searchable and returnable data of a Solr Core.--Distribution::-The copying of an index from the leader server to all followers. The distribution process takes advantage of Lucene's index file structure.--Inserts and Deletes::-As inserts and deletes occur in the index, the directory remains unchanged. Documents are always inserted into newly created segment files. Documents that are deleted are not removed from the segment files. They are flagged in the file, deletable, and are not removed from the segments until the segment is merged as part of normal index updates.--Leader and Follower::-A Solr replication leader is a single node which receives all updates initially and keeps everything organized. Solr replication follower nodes receive no updates directly, instead all changes (such as inserts, updates, deletes, etc.) are made against the single leader node. Changes made on the leader are distributed to all the follower nodes which service all query requests from the clients.--Update::-An update is a single change request against a single Solr instance. It may be a request to delete a document, add a new document, change a document, delete all documents matching a query, etc. Updates are handled synchronously within an individual Solr instance.--Optimization::-A process that compacts the index and merges segments in order to improve query performance. Optimization should only be run on the leader nodes. An optimized index may give query performance gains compared to an index that has become fragmented over a period of time with many updates. Distributing an optimized index requires a much longer time than the distribution of new segments to an un-optimized index.--WARNING: optimizing is not recommended unless it can be performed regularly as it may lead to a significantly larger portion of the index consisting of deleted documents than would normally be the case.--Segments::-A self contained subset of an index consisting of some documents and data structures related to the inverted index of terms in those documents.--mergeFactor::-A parameter that controls the number of segments in an index. For example, when mergeFactor is set to 3, Solr will fill one segment with documents until the limit maxBufferedDocs is met, then it will start a new segment. When the number of segments specified by mergeFactor is reached (in this example, 3) then Solr will merge all the segments into a single index file, then begin writing new documents to a new segment.--Snapshot::-A directory containing hard links to the data files of an index. Snapshots are distributed from the leader nodes when the followers pull them, "smart copying" any segments the follower node does not have in snapshot directory that contains the hard links to the most recent index data files.---== Configuring the ReplicationHandler--In addition to `ReplicationHandler` configuration options specific to the leader/follower roles, there are a few special configuration options that are generally supported (even when using SolrCloud).--* `maxNumberOfBackups` an integer value dictating the maximum number of backups this node will keep on disk as it receives `backup` commands.-* Similar to most other request handlers in Solr you may configure a set of <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#search-handlers,defaults, invariants, and/or appends>> parameters corresponding with any request parameters supported by the `ReplicationHandler` when <<HTTP API Commands for the ReplicationHandler,processing commands>>.--=== Configuring the Replication RequestHandler on a Leader Server--Before running a replication, you should set the following parameters on initialization of the handler:--`replicateAfter`::-String specifying action after which replication should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. If you use "startup", you need to have a "commit" and/or "optimize" entry also if you want to trigger replication on future commits or optimizes.--`backupAfter`::-String specifying action after which a backup should occur. Valid values are commit, optimize, or startup. There can be multiple values for this parameter. It is not required for replication, it just makes a backup.--`maxNumberOfBackups`::-Integer specifying how many backups to keep. This can be used to delete all but the most recent N backups.--`confFiles`::-The configuration files to replicate, separated by a comma.--`commitReserveDuration`::-If your commits are very frequent and your network is slow, you can tweak this parameter to increase the amount of time expected to be required to transfer data. The default is `00:00:10` i.e., 10 seconds.--The example below shows a possible 'leader' configuration for the `ReplicationHandler`, including a fixed number of backups and an invariant setting for the `maxWriteMBPerSec` request parameter to prevent followers from saturating its network interface--[source,xml]------<requestHandler name="/replication" class="solr.ReplicationHandler">- <lst name="leader">- <str name="replicateAfter">optimize</str>- <str name="backupAfter">optimize</str>- <str name="confFiles">schema.xml,stopwords.txt,elevate.xml</str>- </lst>- <int name="maxNumberOfBackups">2</int>- <str name="commitReserveDuration">00:00:10</str>- <lst name="invariants">- <str name="maxWriteMBPerSec">16</str>- </lst>-</requestHandler>-------==== Replicating solrconfig.xml--In the configuration file on the leader server, include a line like the following:--[source,xml]------<str name="confFiles">solrconfig_follower.xml:solrconfig.xml,x.xml,y.xml</str>-------This ensures that the local configuration `solrconfig_follower.xml` will be saved as `solrconfig.xml` on the follower. All other files will be saved with their original names.--On the leader server, the file name of the follower configuration file can be anything, as long as the name is correctly identified in the `confFiles` string; then it will be saved as whatever file name appears after the colon ':'.--=== Configuring the Replication RequestHandler on a Follower Server--The code below shows how to configure a ReplicationHandler on a follower.--[source,xml]------<requestHandler name="/replication" class="solr.ReplicationHandler">- <lst name="follower">-- <!-- fully qualified url for the replication handler of leader. It is- possible to pass on this as a request param for the fetchindex command -->- <str name="leaderUrl">http://remote_host:port/solr/core_name/replication</str>-- <!-- Interval in which the follower should poll leader. Format is HH:mm:ss .- If this is absent follower does not poll automatically.-- But a fetchindex can be triggered from the admin or the http API -->-- <str name="pollInterval">00:00:20</str>-- <!-- THE FOLLOWING PARAMETERS ARE USUALLY NOT REQUIRED-->-- <!-- To use compression while transferring the index files. The possible- values are internal|external. If the value is 'external' make sure- that your leader Solr has the settings to honor the accept-encoding header.- If it is 'internal' everything will be taken care of automatically.- USE THIS ONLY IF YOUR BANDWIDTH IS LOW.- THIS CAN ACTUALLY SLOW DOWN REPLICATION IN A LAN -->- <str name="compression">internal</str>-- <!-- The following values are used when the follower connects to the leader to- download the index files. Default values implicitly set as 5000ms and- 10000ms respectively. The user DOES NOT need to specify these unless the- bandwidth is extremely low or if there is an extremely high latency -->-- <str name="httpConnTimeout">5000</str>- <str name="httpReadTimeout">10000</str>-- <!-- If HTTP Basic authentication is enabled on the leader, then the follower- can be configured with the following -->-- <str name="httpBasicAuthUser">username</str>- <str name="httpBasicAuthPassword">password</str>- </lst>-</requestHandler>-------== Setting Up a Repeater with the ReplicationHandler--A leader may be able to serve only so many followers without affecting performance. Some organizations have deployed follower servers across multiple data centers. If each follower downloads the index from a remote data center, the resulting download may consume too much network bandwidth. To avoid performance degradation in cases like this, you can configure one or more followers as repeaters. A repeater is simply a node that acts as both a leader and a follower.--* To configure a server as a repeater, the definition of the Replication `requestHandler` in the `solrconfig.xml` file must include file lists of use for both leaders and followers.-* Be sure to set the `replicateAfter` parameter to commit, even if `replicateAfter` is set to optimize on the main leader. This is because on a repeater (or any follower), a commit is called only after the index is downloaded. The optimize command is never called on followers.-* Optionally, one can configure the repeater to fetch compressed files from the leader through the compression parameter to reduce the index download time.--Here is an example of a ReplicationHandler configuration for a repeater:--[source,xml]------<requestHandler name="/replication" class="solr.ReplicationHandler">- <lst name="leader">- <str name="replicateAfter">commit</str>- <str name="confFiles">schema.xml,stopwords.txt,synonyms.txt</str>- </lst>- <lst name="follower">- <str name="leaderUrl">http://leader.solr.company.com:8983/solr/core_name/replication</str>- <str name="pollInterval">00:00:60</str>- </lst>-</requestHandler>-------== Commit and Optimize Operations--When a commit or optimize operation is performed on the leader, the RequestHandler reads the list of file names which are associated with each commit point. This relies on the `replicateAfter` parameter in the configuration to decide which types of events should trigger replication.--These operations are supported:--* `commit`: Triggers replication whenever a commit is performed on the leader index.-* `optimize`: Triggers replication whenever the leader index is optimized.-* `startup`: Triggers replication whenever the leader index starts up.--The `replicateAfter` parameter can accept multiple arguments. For example:--[source,xml]------<str name="replicateAfter">startup</str>-<str name="replicateAfter">commit</str>-<str name="replicateAfter">optimize</str>-------== Follower Replication--The leader is totally unaware of the followers.--The follower continuously keeps polling the leader (depending on the `pollInterval` parameter) to check the current index version of the leader. If the follower finds out that the leader has a newer version of the index it initiates a replication process. The steps are as follows:--* The follower issues a `filelist` command to get the list of the files. This command returns the names of the files as well as some metadata (for example, size, a lastmodified timestamp, an alias if any).-* The follower checks with its own index if it has any of those files in the local index. It then runs the filecontent command to download the missing files. This uses a custom format (akin to the HTTP chunked encoding) to download the full content or a part of each file. If the connection breaks in between, the download resumes from the point it failed. At any point, the follower tries 5 times before giving up a replication altogether.-* The files are downloaded into a temp directory, so that if either the follower or the leader crashes during the download process, no files will be corrupted. Instead, the current replication will simply abort.-* After the download completes, all the new files are moved to the live index directory and the file's timestamp is same as its counterpart on the leader.-* A commit command is issued on the follower by the Follower's ReplicationHandler and the new index is loaded.--=== Replicating Configuration Files--To replicate configuration files, list them using using the `confFiles` parameter. Only files found in the `conf` directory of the leader's Solr instance will be replicated.--Solr replicates configuration files only when the index itself is replicated. That means even if a configuration file is changed on the leader, that file will be replicated only after there is a new commit/optimize on leader's index.--Unlike the index files, where the timestamp is good enough to figure out if they are identical, configuration files are compared against their checksum. The `schema.xml` files (on leader and follower) are judged to be identical if their checksums are identical.--As a precaution when replicating configuration files, Solr copies configuration files to a temporary directory before moving them into their ultimate location in the conf directory. The old configuration files are then renamed and kept in the same `conf/` directory. The ReplicationHandler does not automatically clean up these old files.--If a replication involved downloading of at least one configuration file, the ReplicationHandler issues a core-reload command instead of a commit command.--=== Resolving Corruption Issues on Follower Servers--If documents are added to the follower, then the follower is no longer in sync with its leader. However, the follower will not undertake any action to put itself in sync, until the leader has new index data.--When a commit operation takes place on the leader, the index version of the leader becomes different from that of the follower. The follower then fetches the list of files and finds that some of the files present on the leader are also present in the local index but with different sizes and timestamps. This means that the leader and follower have incompatible indexes.--To correct this problem, the follower then copies all the index files from leader to a new index directory and asks the core to load the fresh index from the new directory.--== HTTP API Commands for the ReplicationHandler--You can use the HTTP commands below to control the ReplicationHandler's operations.--`enablereplication`::-Enable replication on the "leader" for all its followers.-+-[source,bash]-http://_leader_host:port_/solr/_core_name_/replication?command=enablereplication--`disablereplication`::-Disable replication on the leader for all its followers.-+-[source,bash]-http://_leader_host:port_/solr/_core_name_/replication?command=disablereplication--`indexversion`::-Return the version of the latest replicatable index on the specified leader or follower.-+-[source,bash]-http://_host:port_/solr/_core_name_/replication?command=indexversion--`fetchindex`::-Force the specified follower to fetch a copy of the index from its leader.-+-[source.bash]-http://_follower_host:port_/solr/_core_name_/replication?command=fetchindex-+-If you like, you can pass an extra attribute such as `leaderUrl` or `compression` (or any other parameter which is specified in the `<lst name="follower">` tag) to do a one time replication from a leader. This obviates the need for hard-coding the leader in the follower.--`abortfetch`::-Abort copying an index from a leader to the specified follower.-+-[source,bash]-http://_follower_host:port_/solr/_core_name_/replication?command=abortfetch--`enablepoll`::-Enable the specified follower to poll for changes on the leader.-+-[source,bash]-http://_follower_host:port_/solr/_core_name_/replication?command=enablepoll--`disablepoll`::-Disable the specified follower from polling for changes on the leader.-+-[source,bash]-http://_follower_host:port_/solr/_core_name_/replication?command=disablepoll--`details`::-Retrieve configuration details and current status.-+-[source,bash]-http://_follower_host:port_/solr/_core_name_/replication?command=details--`filelist`::-Retrieve a list of Lucene files present in the specified host's index.-+-[source,bash]-http://_host:port_/solr/_core_name_/replication?command=filelist&generation=<_generation-number_>-+-You can discover the generation number of the index by running the `indexversion` command.--`backup`::-Create a backup on leader if there are committed index data in the server; otherwise, does nothing.-+-[source,bash]-http://_leader_host:port_/solr/_core_name_/replication?command=backup-+-This command is useful for making periodic backups. There are several supported request parameters:-+-* `numberToKeep:`: This can be used with the backup command unless the `maxNumberOfBackups` initialization parameter has been specified on the handler – in which case `maxNumberOfBackups` is always used and attempts to use the `numberToKeep` request parameter will cause an error.-* `name`: (optional) Backup name. The snapshot will be created in a directory called `snapshot.<name>` within the data directory of the core. By default the name is generated using date in `yyyyMMddHHmmssSSS` format. If `location` parameter is passed, that would be used instead of the data directory-* `repository`: The name of the backup repository to use. When not specified, it defaults to local file system.-* `location`: Backup location. Value depends on the repository in use. For file system repository, location defaults to core's dataDir, and if specified, it needs to be within `SOLR_HOME`, `SOLR_DATA_HOME` or the paths specified by solr.xml `allowPaths`.--`restore`::-Restore a backup from a backup repository.-+-[source,bash]-http://_leader_host:port_/solr/_core_name_/replication?command=restore-+-This command is used to restore a backup. There are several supported request parameters:-+-* `name`: (optional) Backup name. The name of the backed up index snapshot to be restored. If the name is not provided, it looks for backups with snapshot.<timestamp> format in the location directory. It picks the latest timestamp backup in that case.-* `repository`: The name of the backup repository where the backup resides. When not specified, it defaults to local file system.-* `location`: Backup location. Value depends on the repository in use. For file system repository, location defaults to core's dataDir, and if specified, it needs to be within `SOLR_HOME`, `SOLR_DATA_HOME` or the paths specified by solr.xml `allowPaths`.--`restorestatus`::-Check the status of a running restore operation.-+-[source,bash]-http://_leader_host:port_/solr/_core_name_/replication?command=restorestatus-+-This command is used to check the status of a restore operation. This command takes no parameters.-+-The status value can be "In Progress", "success", or "failed". If it failed then an "exception" will also be sent in the response.--`deletebackup`::-Delete any backup created using the `backup` command.-+-[source,bash]-http://_leader_host:port_ /solr/_core_name_/replication?command=deletebackup-+-There are two supported parameters:--* `name`: The name of the snapshot. A snapshot with the name `snapshot._name_` must exist. If not, an error is thrown.-* `location`: Location where the snapshot is created.---== Distribution and Optimization--Optimizing an index is not something most users should generally worry about - but in particular users should be aware of the impacts of optimizing an index when using the `ReplicationHandler`.--The time required to optimize a leader index can vary dramatically. A small index may be optimized in minutes. A very large index may take hours. The variables include the size of the index and the speed of the hardware.--Distributing a newly optimized index may take only a few minutes or up to an hour or more, again depending on the size of the index and the performance capabilities of network connections and disks. During optimization the machine is under load and does not process queries very well. Given a schedule of updates being driven a few times an hour to the followers, we cannot run an optimize with every committed snapshot.--Copying an optimized index means that the *entire* index will need to be transferred during the next `snappull`. This is a large expense, but not nearly as huge as running the optimize everywhere.--Consider this example: on a three-follower one-leader configuration, distributing a newly-optimized index takes approximately 80 seconds _total_. Rolling the change across a tier would require approximately ten minutes per machine (or machine group). If this optimize were rolled across the query tier, and if each follower node being optimized were disabled and not receiving queries, a rollout would take at least twenty minutes and potentially as long as an hour and a half. Additionally, the files would need to be synchronized so that the _following_ the optimize, `snappull` would not think that the independently optimized files were different in any way. This would also leave the door open to independent corruption of indexes instead of each being a perfect copy of the leader.--Optimizing on the leader allows for a straight-forward optimization operation. No query followers need to be taken out of service. The optimized index can be distributed in the background as queries are being normally serviced. The optimization can occur at any time convenient to the application providing index updates.--While optimizing may have some benefits in some situations, a rapidly changing index will not retain those benefits for long, and since optimization is an intensive process, it may be better to consider other options, such as lowering the merge factor (discussed in the section on <<indexconfig-in-solrconfig.adoc#merge-factors,Index Configuration>>).--TIP: Do not elect to optimize your index unless you have tangible evidence that it will significantly improve your search performance. Recent changes in Solr/Lucene have dramatically lessened the need to optimize as discussed at the above link.diff --git a/solr/solr-ref-guide/src/index-segments-merging.adoc b/solr/solr-ref-guide/src/index-segments-merging.adocnew file mode 100644index 00000000000..7ecb000d0ab--- /dev/null+++ b/solr/solr-ref-guide/src/index-segments-merging.adoc@@ -0,0 +1,331 @@+= Index Segments and Merging+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Lucene indexes are stored in segments and Solr offers several parameters to control how new segments are written and when segments are merged.++Lucene indexes are "write-once" files: once a segment has been written to permanent storage (to disk), it is never altered.+This means that indexes are actually comprised of several files which are each subsets of the full index.+In order to prevent eternal fragmentation of the index, segments are periodically merged.++== <indexConfig> in solrconfig.xml++The `<indexConfig>` section of `solrconfig.xml` defines low-level behavior of the Lucene index writers.++By default, the settings are commented out in the sample `solrconfig.xml` included with Solr, which means the defaults are used.+In most cases, the defaults are fine.++[source,xml]+----+<indexConfig>+ ...+</indexConfig>+----++== Writing New Segments++The following elements can be defined under the `<indexConfig>` element and define when new segments are written ("flushed") to disk.++=== ramBufferSizeMB++Once accumulated document updates exceed this much memory space (defined in megabytes), then the pending updates are flushed.+This can also create new segments or trigger a merge.+Using this setting is generally preferable to `maxBufferedDocs`.+If both `maxBufferedDocs` and `ramBufferSizeMB` are set in `solrconfig.xml`, then a flush will occur when either limit is reached.+The default is `100` MB.++[source,xml]+----+<ramBufferSizeMB>100</ramBufferSizeMB>+----++=== maxBufferedDocs++Sets the number of document updates to buffer in memory before they are flushed as a new segment.+This may also trigger a merge.+The default Solr configuration sets to flush by RAM usage (`ramBufferSizeMB`).++[source,xml]+----+<maxBufferedDocs>1000</maxBufferedDocs>+----++=== useCompoundFile++Controls whether newly written (and not yet merged) index segments should use the <<Compound File Segments>> format.+The default is `false`.++[source,xml]+----+<useCompoundFile>false</useCompoundFile>+----++=== ramPerThreadHardLimitMB++Sets the maximum memory (defined in megabytes) consumption per thread triggering a forced flush if exceeded.+The given value must be greater than `0` and less than `2048` MB (2GB).++[source,xml]+----+<ramPerThreadHardLimitMB>1945</ramPerThreadHardLimitMB>+----++NOTE: This is an expert level parameter as it triggers forced flush even if `ramBufferSizeMB` has not been exceeded.++== Merging Index Segments++The following settings define when segments are merged.++=== mergePolicyFactory++Defines how merging segments is done.++The default in Solr is to use `TieredMergePolicy`, which merges segments of approximately equal size, subject to an allowed number of segments per tier.++Other policies available are the `LogByteSizeMergePolicy` and `LogDocMergePolicy`.+For more information on these policies, please see {lucene-javadocs}/core/org/apache/lucene/index/MergePolicy.html[the MergePolicy javadocs].++[source,xml]+----+<mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">+ <int name="maxMergeAtOnce">10</int>+ <int name="segmentsPerTier">10</int>+ <double name="forceMergeDeletesPctAllowed">10.0</double>+ <double name="deletesPctAllowed">33.0</double>+</mergePolicyFactory>+----++[[merge-factors]]+=== Controlling Segment Sizes++The most common adjustment users make to the configuration of `TieredMergePolicy` (or `LogByteSizeMergePolicy`) are the "merge factors" to change how many segments should be merged at one time and, in the `TieredMergePolicy` case, the maximum size of an merged segment.++For `TieredMergePolicy`, this is controlled by setting the `maxMergeAtOnce` (default `10`), `segmentsPerTier` (default `10`) and `maxMergedSegmentMB` (default `5000`) options.++`LogByteSizeMergePolicy` has a single `mergeFactor` option (default `10`).++To understand why these options are important, consider what happens when an update is made to an index using `LogByteSizeMergePolicy`: Documents are always added to the most recently opened segment.+When a segment fills up, a new segment is created and subsequent updates are placed there.++If creating a new segment would cause the number of lowest-level segments to exceed the `mergeFactor` value, then all those segments are merged together to form a single large segment.+Thus, if the merge factor is `10`, each merge results in the creation of a single segment that is roughly ten times larger than each of its ten constituents.+When there are 10 of these larger segments, then they in turn are merged into an even larger single segment.+This process can continue indefinitely.++When using `TieredMergePolicy`, the process is the same, but instead of a single `mergeFactor` value, the `segmentsPerTier` setting is used as the threshold to decide if a merge should happen, and the `maxMergeAtOnce` setting determines how many segments should be included in the merge.++Choosing the best merge factors is generally a trade-off of indexing speed vs. searching speed.+Having fewer segments in the index generally accelerates searches, because there are fewer places to look.+It also can also result in fewer physical files on disk.+But to keep the number of segments low, merges will occur more often, which can add load to the system and slow down updates to the index.++Conversely, keeping more segments can accelerate indexing, because merges happen less often, making an update is less likely to trigger a merge.+But searches become more computationally expensive and will likely be slower, because search terms must be looked up in more index segments.+Faster index updates also means shorter commit turnaround times, which means more timely search results.++=== Controlling Deleted Document Percentages++When a document is deleted or updated, the document is marked as deleted but it not removed from the index until the segment is merged.+There are two parameters that can can be adjusted when using the default TieredMergePolicy that influence the number of deleted documents in an index.++`forceMergeDeletesPctAllowed`::+(default `10.0`) When the external `expungeDeletes` command is issued, any segment that has more than this percent deleted documents will be merged into a new segment and the data associated with the deleted documents will be purged.+A value of `0.0` will make expungeDeletes behave essentially identically to `optimize`.++`deletesPctAllowed`::+(default `33.0`). During normal segment merging, a best effort is made to insure that the total percentage of deleted documents in the index is below this threshold.+Valid settings are between 20% and 50%.+33% was chosen as the default because as this setting approaches 20%, considerable load is added to the system.++=== Customizing Merge Policies++If the configuration options for the built-in merge policies do not fully suit your use case, you can customize them either by creating a custom merge policy factory that you specify in your configuration, or by configuring a {solr-javadocs}/core/org/apache/solr/index/WrapperMergePolicyFactory.html[merge policy wrapper] which uses a `wrapped.prefix` configuration option to control how the factory it wraps will be configured:++[source,xml]+----+<mergePolicyFactory class="org.apache.solr.index.SortingMergePolicyFactory">+ <str name="sort">timestamp desc</str>+ <str name="wrapped.prefix">inner</str>+ <str name="inner.class">org.apache.solr.index.TieredMergePolicyFactory</str>+ <int name="inner.maxMergeAtOnce">10</int>+ <int name="inner.segmentsPerTier">10</int>+</mergePolicyFactory>+----++The example above shows Solr's {solr-javadocs}/core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] being configured to sort documents in merged segments by `"timestamp desc"`, and wrapped around a `TieredMergePolicyFactory` configured to use the values `maxMergeAtOnce=10` and `segmentsPerTier=10` via the `inner` prefix defined by `SortingMergePolicyFactory` 's `wrapped.prefix` option.+For more information on using `SortingMergePolicyFactory`, see <<common-query-parameters.adoc#segmentterminateearly-parameter,the segmentTerminateEarly parameter>>.++=== mergeScheduler++The merge scheduler controls how merges are performed.+The default `ConcurrentMergeScheduler` performs merges in the background using separate threads.+The alternative, `SerialMergeScheduler`, does not perform merges with separate threads.++The `ConcurrentMergeScheduler` has the following configurable attributes:++`maxMergeCount`::+The maximum number of simultaneous merges that are allowed.+If a merge is necessary yet we already have this many threads running, the indexing thread will block until a merge thread has completed.+Note that Solr will only run the smallest `maxThreadCount` merges at a time.++`maxThreadCount`::+The maximum number of simultaneous merge threads that should be running at once. This must be less than `maxMergeCount`.++`ioThrottle`::+A Boolean value (`true` or `false`) to explicitly control I/O throttling.+By default throttling is enabled and the CMS will limit I/O throughput when merging to leave other (search, indexing) some room.++The defaults for the above attributes are dynamically set based on whether the underlying disk drive is rotational disk or not.+Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.++.Example: Dynamic defaults+[source,xml]+----+<mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>+----++.Example: Explicit defaults+[source,xml]+----+<mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler">+ <int name="maxMergeCount">9</int>+ <int name="maxThreadCount">4</int>+</mergeScheduler>+----++=== mergedSegmentWarmer++When using Solr for <<solrcloud-distributed-requests.adoc#near-real-time-nrt-use-cases,Near Real Time Use Cases>>, a merged segment warmer can be configured to warm the reader on the newly merged segment, before the merge commits.+This is not required for near real-time search, but will reduce search latency on opening a new near real-time reader after a merge completes.++[source,xml]+----+<mergedSegmentWarmer class="org.apache.lucene.index.SimpleMergedSegmentWarmer"/>+----++== Compound File Segments++Each Lucene segment is typically comprised of a dozen or so files.+Lucene can be configured to bundle all of the files for a segment into a single compound file using a file extension of `.cfs`, for "Compound File Segment".++CFS segments may incur a minor performance hit for various reasons, depending on the runtime environment.+For example, filesystem buffers are typically associated with open file descriptors, which may limit the total cache space available to each index.++On systems where the number of open files allowed per process is limited, CFS may avoid hitting that limit.+The open files limit might also be tunable for your OS with the Linux/Unix `ulimit` command, or something similar for other operating systems.++.CFS: New Segments vs Merged Segments+[NOTE]+====+To configure whether _newly written segments_ should use CFS, see the <<usecompoundfile,`useCompoundFile`>> setting described above.+To configure whether _merged segments_ use CFS, review the Javadocs for your <<mergepolicyfactory,`mergePolicyFactory`>>.++Many <<Merging Index Segments,Merge Policy>> implementations support `noCFSRatio` and `maxCFSSegmentSizeMB` settings with default values that prevent compound files from being used for large segments, but do use compound files for small segments.+====++== Segments Info Screen++The Segments Info screen in the Admin UI lets you see a visualization of the various segments in the underlying Lucene index for this core, with information about the size of each segment – both bytes and in number of documents – as well as other basic metadata about those segments.+Most visible is the the number of deleted documents, but you can hover your mouse over the segments to see additional numeric details.++image::images/segments-info/segments_info.png[image,width=486,height=250]++This information may be useful for people to help make decisions about the optimal <<merging-index-segments,merge settings>> for their data.++== Index Locks++=== lockType++The LockFactory options specify the locking implementation to use.++The set of valid lock type options depends on the <<index-location-format.adoc#,DirectoryFactory>> you have configured.++The values listed below are are supported by `StandardDirectoryFactory` (the default):++* `native` (default) uses `NativeFSLockFactory` to specify native OS file locking.+If a second Solr process attempts to access the directory, it will fail.+Do not use when multiple Solr web applications are attempting to share a single index.+See also the {lucene-javadocs}/core/org/apache/lucene/store/NativeFSLockFactory.html[NativeFSLockFactory javadocs].++* `simple` uses `SimpleFSLockFactory` to specify a plain file for locking.+See also the {lucene-javadocs}/core/org/apache/lucene/store/SimpleFSLockFactory.html[SimpleFSLockFactory javadocs].++* `single` (expert) uses `SingleInstanceLockFactory`.+Use for special situations of a read-only index directory, or when there is no possibility of more than one process trying to modify the index (even sequentially).+This type will protect against multiple cores within the _same_ JVM attempting to access the same index.+++WARNING: If multiple Solr instances in different JVMs modify an index, this type will _not_ protect against index corruption.++++See also the {lucene-javadocs}/core/org/apache/lucene/store/SingleInstanceLockFactory.html[SingleInstanceLockFactory javadocs].++* `hdfs` uses `HdfsLockFactory` to support reading and writing index and transaction log files to a HDFS filesystem.+See the section <<solr-on-hdfs.adoc#,Solr on HDFS>> for more details on using this feature.++[source,xml]+----+<lockType>native</lockType>+----++=== writeLockTimeout++The maximum time to wait for a write lock on an IndexWriter.+The default is `1000`, expressed in milliseconds.++[source,xml]+----+<writeLockTimeout>1000</writeLockTimeout>+----++== Other Indexing Settings++There are a few other parameters that may be important to configure for your implementation. These settings affect how or when updates are made to an index.++=== deletionPolicy++Controls how commits are retained in case of rollback.+The default is `SolrDeletionPolicy`, which take ths following parameters:++`maxCommitsToKeep`::+The maximum number of commits to keep.++`maxOptimizedCommitsToKeep`::+The maximum number of optimized commits to keep.++`maxCommitAge`::+The maximum age of any commit to keep.+This supports `DateMathParser` syntax.++[source,xml]+----+<deletionPolicy class="solr.SolrDeletionPolicy">+ <str name="maxCommitsToKeep">1</str>+ <str name="maxOptimizedCommitsToKeep">0</str>+ <str name="maxCommitAge">1DAY</str>+</deletionPolicy>+----++=== infoStream++The InfoStream setting instructs the underlying Lucene classes to write detailed debug information from the indexing process as Solr log messages.+Note that enabling this may substantially increase the size of your logs and can cause some performance lags in high traffic systems.+The default is `false`.++[source,xml]+----+<infoStream>false</infoStream>+----diff --git a/solr/solr-ref-guide/src/index.adoc b/solr/solr-ref-guide/src/index.adocindex 108d1220086..2805c849dd2 100644--- a/solr/solr-ref-guide/src/index.adoc+++ b/solr/solr-ref-guide/src/index.adoc@@ -1,26 +1,10 @@= Apache Solr Reference Guide-:page-children: about-this-guide, \- getting-started, \- deployment-and-operations, \- using-the-solr-administration-user-interface, \- documents-fields-and-schema-design, \- understanding-analyzers-tokenizers-and-filters, \- indexing-and-basic-data-operations, \- searching, \- streaming-expressions, \- solrcloud, \- legacy-scaling-and-distribution, \- circuit-breakers, \- rate-limiters, \- task-management, \- solr-plugins, \- the-well-configured-solr-instance, \- monitoring-solr, \- securing-solr, \- client-apis, \- further-assistance, \- solr-glossary, \- errata+:page-children: getting-started, \+ deployment-guide, \+ configuration-guide, \+ schema-indexing-guide, \+ query-guide, \+ solr-upgrade-notes:page-notitle::page-show-toc: false:page-layout: home@@ -52,90 +36,25 @@ Welcome to Apache Solr(TM), the open source solution for search and analytics.Solr is the fast open source search platform built on Apache Lucene(TM) that provides scalable indexing and search, as well as faceting, hit highlighting and advanced analysis/tokenization capabilities. Solr and Lucene are managed by the http://www.apache.org/[Apache Software Foundation].This Reference Guide is the official Solr documentation, written and published by Lucene/Solr committers.--++++-<p><a class="btn btn-home btn-lg" href="https://solr.apache.org/downloads.html" role="button">Download Solr</a></p>-++++****The Guide includes the following sections:-[.container]----.Getting Started with Solr-[sidebar.col]-****--The *<<getting-started.adoc#,Getting Started>>* section guides you through the installation and setup of Solr. A detailed tutorial for first-time users shows many of Solr's features.--*<<using-the-solr-administration-user-interface.adoc#,Using the Solr Administration User Interface>>*: This section introduces the Web-based interface for administering Solr. From your browser you can view configuration files, submit queries, view logfile settings and Java environment settings, and monitor and control distributed configurations.-****--.Deploying Solr-[sidebar.col]-****--*<<deployment-and-operations.adoc#,Deployment and Operations>>*: Once you have Solr configured, you want to deploy it to production and keep it up to date. This section includes information about how to take Solr to production, run it in HDFS or AWS, and information about upgrades and managing Solr from the command line.--*<<monitoring-solr.adoc#,Monitoring Solr>>*: Solr includes options for keeping an eye on the performance of your Solr cluster with the web-based administration console, through the command line interface, or using REST APIs.-****---------'''---------.Indexing Documents-[sidebar.col]-****-*<<indexing-and-basic-data-operations.adoc#,Indexing and Basic Data Operations>>*: This section describes the indexing process and basic index operations, such as commit, optimize, and rollback.--*<<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>>*: This section describes how Solr organizes data in the index. It explains how a Solr schema defines the fields and field types which Solr uses to organize data within the document files it indexes.--*<<understanding-analyzers-tokenizers-and-filters.adoc#,Understanding Analyzers, Tokenizers, and Filters>>*: This section explains how Solr prepares text for indexing and searching. Analyzers parse text and produce a stream of tokens, lexical units used for indexing and searching. Tokenizers break field data down into tokens. Filters perform other transformational or selective work on token streams.-****--.Searching Documents-[sidebar.col]****+[.home-table,cols="1,1,1",frame=none,grid=none,stripes=none]+|===+^| <<getting-started.adoc#,Getting Started>> ^| <<deployment-guide.adoc#,Deployment Guide>> ^| <<configuration-guide.adoc#,Configuration Guide>>-*<<searching.adoc#,Searching>>*: This section presents an overview of the search process in Solr. It describes the main components used in searches, including request handlers, query parsers, and response writers. It lists the query parameters that can be passed to Solr, and it describes features such as boosting and faceting, which can be used to fine-tune search results.+| Introductory concepts and tutorials. | Installation, monitoring, scaling, and deploying to production. | Tuning Solr's configuration files for your use case.-*<<streaming-expressions.adoc#,Streaming Expressions>>*: A stream processing language for Solr, with a suite of functions to perform many types of queries and parallel execution tasks.+|===-*<<client-apis.adoc#,Client APIs>>*: This section tells you how to access Solr through various client APIs, including JavaScript, JSON, and Ruby.-****-------+[.home-table,cols="1,1,1",frame=none,grid=none,stripes=none]+|===+^| <<schema-indexing-guide.adoc#,Schema & Indexing Guide>> ^| <<query-guide.adoc#,Query Guide>> ^| <<solr-upgrade-notes.adoc#,Solr Upgrade Notes>>-'''+| Configuring Solr's schema and indexing documents. | All aspects of Solr queries. | Change notes for Solr releases.--------.Scaling Solr-[sidebar.col]-****-*<<solrcloud.adoc#,SolrCloud>>*: This section describes SolrCloud, which provides comprehensive distributed capabilities.--*<<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>*: This section tells you how to grow a Solr distribution by dividing a large index into sections called shards, which are then distributed across multiple servers, or by replicating a single index across multiple services.--*<<circuit-breakers.adoc#,Circuit Breakers>>*: This section talks about circuit breakers, a way of allowing a higher stability of Solr nodes and increased service level guarantees of requests that are accepted by Solr.--*<<rate-limiters.adoc#,Request Rate Limiters>>*: This section talks about request rate limiters, a way of guaranteeing throughput per request type and dedicating resource quotas by resource type. Rate limiter configurations are per instance/JVM and applied to the entire JVM, not at a core/collection level.--*<<task-management.adoc#,Task Management>>*: This section talks about task management interface, which can be used to control cancellable tasks.-****--.Advanced Configuration-[sidebar.col]-****-*<<securing-solr.adoc#,Securing Solr>>*: When planning how to secure Solr, you should consider which of the available features or approaches are right for you.+|===-*<<the-well-configured-solr-instance.adoc#,The Well-Configured Solr Instance>>*: This section discusses performance tuning for Solr. It begins with an overview of the `solrconfig.xml` file, then tells you how to configure cores with `solr.xml`, how to configure the Lucene index writer, and more.****---diff --git a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc b/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adocdeleted file mode 100644index 5d0f798bec2..00000000000--- a/solr/solr-ref-guide/src/indexconfig-in-solrconfig.adoc+++ /dev/null@@ -1,242 +0,0 @@-= IndexConfig in SolrConfig-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The `<indexConfig>` section of `solrconfig.xml` defines low-level behavior of the Lucene index writers.--By default, the settings are commented out in the sample `solrconfig.xml` included with Solr, which means the defaults are used. In most cases, the defaults are fine.--[source,xml]------<indexConfig>- ...-</indexConfig>-------== Writing New Segments--=== ramBufferSizeMB--Once accumulated document updates exceed this much memory space (defined in megabytes), then the pending updates are flushed. This can also create new segments or trigger a merge. Using this setting is generally preferable to `maxBufferedDocs`. If both `maxBufferedDocs` and `ramBufferSizeMB` are set in `solrconfig.xml`, then a flush will occur when either limit is reached. The default is 100Mb.--[source,xml]------<ramBufferSizeMB>100</ramBufferSizeMB>-------=== maxBufferedDocs--Sets the number of document updates to buffer in memory before they are flushed as a new segment. This may also trigger a merge. The default Solr configuration sets to flush by RAM usage (`ramBufferSizeMB`).--[source,xml]------<maxBufferedDocs>1000</maxBufferedDocs>-------=== useCompoundFile--Controls whether newly written (and not yet merged) index segments should use the <<Compound File Segments>> format. The default is false.--[source,xml]------<useCompoundFile>false</useCompoundFile>-------=== ramPerThreadHardLimitMB--Sets the maximum memory (defined in megabytes) consumption per thread triggering a forced flush if exceeded. The given value must be greater than 0 and less than 2GB (2048 MB).--[source,xml]------<ramPerThreadHardLimitMB>1945</ramPerThreadHardLimitMB>-------NOTE: This is an expert level parameter as it triggers forced flush even if <<ramBufferSizeMB>> has not been exceeded.--== Merging Index Segments--=== mergePolicyFactory--Defines how merging segments is done.--The default in Solr is to use a `TieredMergePolicy`, which merges segments of approximately equal size, subject to an allowed number of segments per tier.--Other policies available are the `LogByteSizeMergePolicy` and `LogDocMergePolicy`.-For more information on these policies, please see {lucene-javadocs}/core/org/apache/lucene/index/MergePolicy.html[the MergePolicy javadocs].--[source,xml]------<mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory">- <int name="maxMergeAtOnce">10</int>- <int name="segmentsPerTier">10</int>- <double name="forceMergeDeletesPctAllowed">10.0</double>- <double name="deletesPctAllowed">33.0</double>--</mergePolicyFactory>-------[[merge-factors]]-=== Controlling Segment Sizes--The most common adjustment users make to the configuration of `TieredMergePolicy` (or `LogByteSizeMergePolicy`) are the "merge factors" to change how many segments should be merged at one time and, in the `TieredMergePolicy` case, the maximum size of an merged segment.--For `TieredMergePolicy`, this is controlled by setting the `maxMergeAtOnce` (default `10`), `segmentsPerTier` (default `10`) and `maxMergedSegmentMB` (default `5000`) options.--`LogByteSizeMergePolicy` has a single `mergeFactor` option (default `10`).--To understand why these options are important, consider what happens when an update is made to an index using `LogByteSizeMergePolicy`: Documents are always added to the most recently opened segment. When a segment fills up, a new segment is created and subsequent updates are placed there.--If creating a new segment would cause the number of lowest-level segments to exceed the `mergeFactor` value, then all those segments are merged together to form a single large segment. Thus, if the merge factor is `10`, each merge results in the creation of a single segment that is roughly ten times larger than each of its ten constituents. When there are 10 of these larger segments, then they in turn are merged into an even larger single segment. This process can continue indefinitely.--When using `TieredMergePolicy`, the process is the same, but instead of a single `mergeFactor` value, the `segmentsPerTier` setting is used as the threshold to decide if a merge should happen, and the `maxMergeAtOnce` setting determines how many segments should be included in the merge.--Choosing the best merge factors is generally a trade-off of indexing speed vs. searching speed. Having fewer segments in the index generally accelerates searches, because there are fewer places to look. It also can also result in fewer physical files on disk. But to keep the number of segments low, merges will occur more often, which can add load to the system and slow down updates to the index.--Conversely, keeping more segments can accelerate indexing, because merges happen less often, making an update is less likely to trigger a merge. But searches become more computationally expensive and will likely be slower, because search terms must be looked up in more index segments. Faster index updates also means shorter commit turnaround times, which means more timely search results.--=== Controlling Deleted Document Percentages--When a document is deleted or updated, the document is marked as deleted but it not removed from the index until the segment is merged. There are two parameters that can can be adjusted when using the default TieredMergePolicy that influence the number of deleted documents in an index.--`forceMergeDeletesPctAllowed`::-(default 10.0) When the external expungeDeletes command is issued, any segment that has more than this percent deleted documents will be merged into a new segment and the data associated with the deleted documents will be purged. A value of 0.0 will make expungeDeletes behave essentially identically to `optimize`.--`deletesPctAllowed`::-(default 33.0). During normal segment merging, a "best effort" is made to insure that the total percentage of deleted documents in the index is below this threshold. Valid settings are between 20% and 50%. 33% was chosen as the default because as this setting approaches 20%, considerable load is added to the system.--=== Customizing Merge Policies--If the configuration options for the built-in merge policies do not fully suit your use case, you can customize them: either by creating a custom merge policy factory that you specify in your configuration, or by configuring a {solr-javadocs}/core/org/apache/solr/index/WrapperMergePolicyFactory.html[merge policy wrapper] which uses a `wrapped.prefix` configuration option to control how the factory it wraps will be configured:--[source,xml]------<mergePolicyFactory class="org.apache.solr.index.SortingMergePolicyFactory">- <str name="sort">timestamp desc</str>- <str name="wrapped.prefix">inner</str>- <str name="inner.class">org.apache.solr.index.TieredMergePolicyFactory</str>- <int name="inner.maxMergeAtOnce">10</int>- <int name="inner.segmentsPerTier">10</int>-</mergePolicyFactory>-------The example above shows Solr's {solr-javadocs}/core/org/apache/solr/index/SortingMergePolicyFactory.html[`SortingMergePolicyFactory`] being configured to sort documents in merged segments by `"timestamp desc"`, and wrapped around a `TieredMergePolicyFactory` configured to use the values `maxMergeAtOnce=10` and `segmentsPerTier=10` via the `inner` prefix defined by `SortingMergePolicyFactory` 's `wrapped.prefix` option. For more information on using `SortingMergePolicyFactory`, see <<common-query-parameters.adoc#segmentterminateearly-parameter,the segmentTerminateEarly parameter>>.--=== mergeScheduler--The merge scheduler controls how merges are performed. The default `ConcurrentMergeScheduler` performs merges in the background using separate threads. The alternative, `SerialMergeScheduler`, does not perform merges with separate threads.--The `ConcurrentMergeScheduler` has the following configurable attributes:--`maxMergeCount`::-The maximum number of simultaneous merges that are allowed. If a merge is necessary yet we already have this many threads running, the indexing thread will block until a merge thread has completed. Note that Solr will only run the smallest `maxThreadCount` merges at a time.--`maxThreadCount`::-The maximum number of simultaneous merge threads that should be running at once. This must be less than `maxMergeCount`.--`ioThrottle`::-A Boolean value (true/ false) to explicitly control I/O throttling. By default throttling is enabled and the CMS will limit I/O throughput when merging to leave other (search, indexing) some room.--The defaults for the above attributes are dynamically set based on whether the underlying disk drive is rotational disk or not. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.--.Example: Dynamic defaults-[source,xml]------<mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/>-------.Example: Explicit defaults-[source,xml]------<mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler">- <int name="maxMergeCount">9</int>- <int name="maxThreadCount">4</int>-</mergeScheduler>-------=== mergedSegmentWarmer--When using Solr in for <<near-real-time-searching.adoc#,Near Real Time Searching>> a merged segment warmer can be configured to warm the reader on the newly merged segment, before the merge commits. This is not required for near real-time search, but will reduce search latency on opening a new near real-time reader after a merge completes.--[source,xml]------<mergedSegmentWarmer class="org.apache.lucene.index.SimpleMergedSegmentWarmer"/>-------== Compound File Segments--Each Lucene segment is typically comprised of a dozen or so files. Lucene can be configured to bundle all of the files for a segment into a single compound file using a file extension of `.cfs`; it's an abbreviation for Compound File Segment.--CFS segments may incur a minor performance hit for various reasons, depending on the runtime environment. For example, filesystem buffers are typically associated with open file descriptors, which may limit the total cache space available to each index.--On systems where the number of open files allowed per process is limited, CFS may avoid hitting that limit. The open files limit might also be tunable for your OS with the Linux/Unix `ulimit` command, or something similar for other operating systems.--.CFS: New Segments vs Merged Segments-[NOTE]-====-To configure whether _newly written segments_ should use CFS, see the <<useCompoundFile,`useCompoundFile`>> setting described above. To configure whether _merged segments_ use CFS, review the Javadocs for your <<mergePolicyFactory,`mergePolicyFactory`>>.--Many <<Merging Index Segments,Merge Policy>> implementations support `noCFSRatio` and `maxCFSSegmentSizeMB` settings with default values that prevent compound files from being used for large segments, but do use compound files for small segments.--====--== Index Locks--=== lockType--The LockFactory options specify the locking implementation to use.--The set of valid lock type options depends on the <<datadir-and-directoryfactory-in-solrconfig.adoc#,DirectoryFactory>> you have configured.--The values listed below are are supported by `StandardDirectoryFactory` (the default):--* `native` (default) uses NativeFSLockFactory to specify native OS file locking. If a second Solr process attempts to access the directory, it will fail. Do not use when multiple Solr web applications are attempting to share a single index. See also the {lucene-javadocs}/core/org/apache/lucene/store/NativeFSLockFactory.html[Javadocs].-* `simple` uses SimpleFSLockFactory to specify a plain file for locking. See also the {lucene-javadocs}/core/org/apache/lucene/store/SimpleFSLockFactory.html[Javadocs].-* `single` (expert) uses SingleInstanceLockFactory. Use for special situations of a read-only index directory, or when there is no possibility of more than one process trying to modify the index (even sequentially). This type will protect against multiple cores within the _same_ JVM attempting to access the same index. WARNING! If multiple Solr instances in different JVMs modify an index, this type will _not_ protect against index corruption. See also the {lucene-javadocs}/core/org/apache/lucene/store/SingleInstanceLockFactory.html[Javadocs].-* `hdfs` uses HdfsLockFactory to support reading and writing index and transaction log files to a HDFS filesystem. See the section <<running-solr-on-hdfs.adoc#,Running Solr on HDFS>> for more details on using this feature.--[source,xml]------<lockType>native</lockType>-------=== writeLockTimeout--The maximum time to wait for a write lock on an IndexWriter. The default is 1000, expressed in milliseconds.--[source,xml]------<writeLockTimeout>1000</writeLockTimeout>-------== Other Indexing Settings--There are a few other parameters that may be important to configure for your implementation. These settings affect how or when updates are made to an index.--`deletionPolicy`:: Controls how commits are retained in case of rollback. The default is `SolrDeletionPolicy`, which has sub-parameters for the maximum number of commits to keep (`maxCommitsToKeep`), the maximum number of optimized commits to keep (`maxOptimizedCommitsToKeep`), and the maximum age of any commit to keep (`maxCommitAge`), which supports `DateMathParser` syntax.--`infoStream`:: The InfoStream setting instructs the underlying Lucene classes to write detailed debug information from the indexing process as Solr log messages.---[source,xml]------<deletionPolicy class="solr.SolrDeletionPolicy">- <str name="maxCommitsToKeep">1</str>- <str name="maxOptimizedCommitsToKeep">0</str>- <str name="maxCommitAge">1DAY</str>-</deletionPolicy>-<infoStream>false</infoStream>-----diff --git a/solr/solr-ref-guide/src/indexing-and-basic-data-operations.adoc b/solr/solr-ref-guide/src/indexing-and-basic-data-operations.adocdeleted file mode 100644index 204a978337d..00000000000--- a/solr/solr-ref-guide/src/indexing-and-basic-data-operations.adoc+++ /dev/null@@ -1,56 +0,0 @@-= Indexing and Basic Data Operations-:page-children: introduction-to-solr-indexing, +- post-tool, +- uploading-data-with-index-handlers, +- indexing-nested-documents, +- uploading-data-with-solr-cell-using-apache-tika, +- updating-parts-of-documents, +- detecting-languages-during-indexing, +- de-duplication, +- content-streams, +- reindexing--// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section describes how Solr adds data to its index. It covers the following topics:--* *<<introduction-to-solr-indexing.adoc#,Introduction to Solr Indexing>>*: An overview of Solr's indexing process.--* *<<post-tool.adoc#,Post Tool>>*: Information about using `post.jar` to quickly upload some content to your system.--* *<<uploading-data-with-index-handlers.adoc#,Uploading Data with Index Handlers>>*: Information about using Solr's Index Handlers to upload XML/XSLT, JSON and CSV data.--* *<<transforming-and-indexing-custom-json.adoc#,Transforming and Indexing Custom JSON>>*: Index any JSON of your choice--* *<<indexing-nested-documents.adoc#,Indexing Nested Documents>>*: Detailed information about indexing and schema configuration for nested documents.--* *<<uploading-data-with-solr-cell-using-apache-tika.adoc#,Uploading Data with Solr Cell using Apache Tika>>*: Information about using the Solr Cell framework to upload data for indexing.--* *<<updating-parts-of-documents.adoc#,Updating Parts of Documents>>*: Information about how to use atomic updates and optimistic concurrency with Solr.--* *<<detecting-languages-during-indexing.adoc#,Detecting Languages During Indexing>>*: Information about using language identification during the indexing process.--* *<<de-duplication.adoc#,De-Duplication>>*: Information about configuring Solr to mark duplicate documents as they are indexed.--* *<<content-streams.adoc#,Content Streams>>*: Information about streaming content to Solr Request Handlers.--* *<<reindexing.adoc#,Reindexing>>*: Details about when reindexing is required or recommended, and some strategies for completely reindexing your documents.--== Indexing Using Client APIs--Using client APIs, such as <<using-solrj.adoc#,SolrJ>>, from your applications is an important option for updating Solr indexes. See the <<client-apis.adoc#,Client APIs>> section for more information.diff --git a/solr/solr-ref-guide/src/indexing-data-operations.adoc b/solr/solr-ref-guide/src/indexing-data-operations.adocnew file mode 100644index 00000000000..6245d113460--- /dev/null+++ b/solr/solr-ref-guide/src/indexing-data-operations.adoc@@ -0,0 +1,55 @@+= Indexing and Data Operations+:page-children: indexing-with-update-handlers, \+ indexing-with-tika, \+ indexing-nested-documents, \+ post-tool, \+ documents-screen, \+ partial-document-updates, \+ reindexing, \+ language-detection, \+ de-duplication, \+ content-streams+:page-show-toc: false+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++This section describes how Solr adds data to its index. It covers the following topics:++****+// This tags the below list so it can be used in the parent page section list+// tag::indexing-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>: Index XML/XSLT, JSON and CSV data.+| <<indexing-with-tika.adoc#,Indexing with Solr Cell and Apache Tika>>: Use Tika to index data.+| <<transforming-and-indexing-custom-json.adoc#,Transforming and Indexing Custom JSON>>: Index custom JSON data.+| <<indexing-nested-documents.adoc#,Indexing Nested Documents>>: Indexing and schema configuration for nested documents.+| <<post-tool.adoc#,Post Tool>>: Quickly upload content to your system.+| <<documents-screen.adoc#,Documents Screen>>: Admin UI for form-based document updates.+| <<partial-document-updates.adoc#,Partial Document Updates>>: Atomic updates and optimistic concurrency.+| <<reindexing.adoc#,Reindexing>>: When and how to reindex.+| <<language-detection.adoc#,Language detection>>: Language identification during indexing.+| <<de-duplication.adoc#,De-Duplication>>: Mark duplicate documents as they are indexed.+| <<content-streams.adoc#,Content Streams>>: Stream content to request handlers.+|+|===+// end::indexing-sections[]+****++== Indexing Using Client APIs++Using client APIs, such as <<solrj.adoc#,SolrJ>>, from your applications is an important option for updating Solr indexes. See the <<client-apis.adoc#,Client APIs>> section for more information.diff --git a/solr/solr-ref-guide/src/indexing-nested-documents.adoc b/solr/solr-ref-guide/src/indexing-nested-documents.adocindex 75a9e39d604..21f02938db4 100644--- a/solr/solr-ref-guide/src/indexing-nested-documents.adoc+++ b/solr/solr-ref-guide/src/indexing-nested-documents.adoc@@ -1,4 +1,4 @@-= Indexing Nested Child Documents+= Indexing Nested Documents:solr-root-path: ../../:example-source-dir: {solr-root-path}solrj/src/test/org/apache/solr/client/ref_guide_examples/// Licensed to the Apache Software Foundation (ASF) under one@@ -27,27 +27,26 @@ with multiple types of child documents representing individual SKUs (with uniqueThe "top most" parent with all children is referred to as a "root" document or formerly "blockdocument" and it explains some of the nomenclature of related features.-At query time, the <<other-parsers.adoc#block-join-query-parsers,Block Join Query Parsers>> can search these relationships,- and the `<<transforming-result-documents.adoc#child-childdoctransformerfactory,[child]>>` Document Transformer can attach child (or other "descendent") documents to the result documents.+At query time, the <<block-join-query-parser.adoc#,Block Join Query Parsers>> can search these relationships,+ and the `<<document-transformers.adoc#child-childdoctransformerfactory,[child]>>` Document Transformer can attach child (or other "descendent") documents to the result documents.In terms of performance, indexing the relationships between documents usually yields much faster queries than an equivalent "<<other-parsers#join-query-parser,query time join>>",since the relationships are already stored in the index and do not need to be computed.However, nested documents are less flexible than query time joins as it imposes rules that some applications may not be able to accept.-Nested documents may be indexed via either the XML or JSON data syntax, and is also supported by <<using-solrj.adoc#,SolrJ>> with javabin.+Nested documents may be indexed via either the XML or JSON data syntax, and is also supported by <<solrj.adoc#,SolrJ>> with javabin.[CAUTION]====.Reindexing Considerations-With the exception of in-place updates, Solr must internally reindex an entire nested document tree-if there are updates to it. For some applications this may-result in a lot of extra indexing overhead that may not be worth the performance gains at query-time versus other modeling approaches.+With the exception of in-place updates, Solr must internally reindex an entire nested document tree if there are updates to it.+For some applications this may result in a lot of extra indexing overhead that may not be worth the performance gains at query time versus other modeling approaches.====-In the examples on this page, the IDs of child documents are always provided. However, you need not-generate such IDs; you can let Solr populate them automatically. It will concatenate the ID of its-parent with a separator and path information that should be unique. Try it out for yourself!+In the examples on this page, the IDs of child documents are always provided.+However, you need not generate such IDs; you can let Solr populate them automatically.+It will concatenate the ID of its parent with a separator and path information that should be unique.+Try it out for yourself![#example-indexing-syntax]== Example Indexing Syntax: Pseudo-Fields@@ -57,7 +56,9 @@ different types of child documents specified in "pseudo-fields": "skus" and "man[NOTE]====-Even though the child documents in these examples are provided syntactically as field values, this is simply a matter of syntax and as such `skus` and `manuals` are not actual fields in the documents. Consequently, these field names need not be defined in the schema and probably shouldn't be as it would be confusing. There is no "child document" field type.+Even though the child documents in these examples are provided syntactically as field values, this is simply a matter of syntax and as such `skus` and `manuals` are not actual fields in the documents.+Consequently, these field names need not be defined in the schema and probably shouldn't be as it would be confusing.+There is no "child document" field type.====//@@ -126,7 +127,7 @@ Even though the child documents in these examples are provided syntactically as[CAUTION]=====-The <<uploading-data-with-index-handlers#json-update-convenience-paths,`/update/json/docs` convenience path>> will automatically flatten complex JSON documents by default -- so to index nested JSON documents make sure to use `/update`.+The <<indexing-with-update-handlers#json-update-convenience-paths,`/update/json/docs` convenience path>> will automatically flatten complex JSON documents by default -- so to index nested JSON documents make sure to use `/update`.=========@@ -238,9 +239,8 @@ Indexing nested documents _requires_ an indexed field named `\_root_`:* Solr automatically populates this field in _all_ documents with the `id` value of it's root document-- it's highest ancestor, possibly itself.* This field must be indexed (`indexed="true"`) but doesn't need to-be either stored (`stored="true"`) or use doc values (`docValues="true"`), however you are free-to do so if you find it useful. If you want to use `uniqueBlock(\_root_)`-<<json-facet-api#stat-facet-functions,field type limitation>>, then you should enable docValues.+be either stored (`stored="true"`) or use doc values (`docValues="true"`), however you are free to do so if you find it useful.+If you want to use `uniqueBlock(\_root_)` <<json-facet-api#stat-facet-functions,field type limitation>>, then you should enable docValues.Preferably, you will also define `\_nest_path_` which adds features and ease-of-use:@@ -255,7 +255,8 @@ Preferably, you will also define `\_nest_path_` which adds features and ease-of-when using the `<<searching-nested-documents.adoc#child-doc-transformer,[child]>>` doc transformer.** If this field does not exist, the `[child]` transformer will return all descendent child documents as a flattened list -- just as if they had been <<#indexing-anonymous-children,indexed as anonymous children>>.* If you do not use `\_nest_path_` it is strongly recommended that every document have some-field that differentiates root documents from their nested children -- and differentiates different "types" of child documents. This is not strictly necessary, so long as it's possible to write a "filter" query that can be used to isolate and select only parent documents for use in the <<other-parsers.adoc#block-join-query-parsers,block join query parsers>> and <<searching-nested-documents.adoc#child-doc-transformer,[child]>> doc transformer+field that differentiates root documents from their nested children -- and differentiates different "types" of child documents.+This is not strictly necessary, so long as it's possible to write a "filter" query that can be used to isolate and select only parent documents for use in the <<block-join-query-parser#,block join query parsers>> and <<searching-nested-documents.adoc#child-doc-transformer,[child]>> doc transformer* It's possible to query on this field, although at present it's only documented how to in thecontext of `[child]`'s `childFilter` parameter.@@ -280,33 +281,33 @@ documents.[TIP]====When using SolrCloud it is a _VERY_ good idea to use-<<shards-and-indexing-data-in-solrcloud#document-routing,prefix based compositeIds>> with a-common prefix for all documents in the nested document tree. This makes it much easier to apply-<<updating-parts-of-documents#updating-child-documents,atomic updates to individual child documents>>+<<solrcloud-shards-indexing.adoc#document-routing,prefix based compositeIds>> with a+common prefix for all documents in the nested document tree.+This makes it much easier to apply+<<partial-document-updates#updating-child-documents,atomic updates to individual child documents>>====== Maintaining Integrity with Updates and DeletesNested document trees can be modified with Solr's-<<updating-parts-of-documents#updating-child-documents,atomic/partial update>> feature to+<<partial-document-updates#updating-child-documents,atomic update>> feature tomanipulate any document in a nested tree, and even to add new child documents.This aspect isn't different than updating any normal document -- Solr internally deletes the oldnested document tree and it adds the newly modified one.Just be mindful to add a `_root_` field if the partial update is to a child doc so that Solrknows which Root doc it's related to.-Solr demands that the `id` of _all_ documents in a collection be unique. Solr enforces this for-root documents within a shard but it doesn't for child documents to avoid the expense of checking.+Solr demands that the `id` of _all_ documents in a collection be unique.+Solr enforces this for root documents within a shard but it doesn't for child documents to avoid the expense of checking.Clients should be very careful to *never* violate this.-To delete an entire nested document tree, you can simply delete-by-ID using the `id` of the root-document. Delete-by-ID will not work with the `id` of a child document, since only root document-IDs are considered. Instead, use delete-by-query (most efficient) or-<<updating-parts-of-documents#updating-child-documents,atomic updates>> to remove the child document from it's parent.+To delete an entire nested document tree, you can simply delete-by-ID using the `id` of the root document.+Delete-by-ID will not work with the `id` of a child document, since only root document IDs are considered.+Instead, use delete-by-query (most efficient) or <<partial-document-updates#,atomic updates>> to remove the child document from it's parent.-If you use Solr's delete-by-query APIs, you *MUST* be careful to ensure that any deletion query-is structured to ensure no descendent children remain of any documents that are being deleted. *_Doing otherwise will violate integrity assumptions that Solr expects._*+If you use Solr's delete-by-query APIs, you *MUST* be careful to ensure that any deletion query is structured to ensure no descendent children remain of any documents that are being deleted.+*_Doing otherwise will violate integrity assumptions that Solr expects._*diff --git a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc b/solr/solr-ref-guide/src/indexing-with-tika.adocsimilarity index 98%rename from solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adocrename to solr/solr-ref-guide/src/indexing-with-tika.adocindex 33acbbeed63..e3c97239995 100644--- a/solr/solr-ref-guide/src/uploading-data-with-solr-cell-using-apache-tika.adoc+++ b/solr/solr-ref-guide/src/indexing-with-tika.adoc@@ -1,4 +1,4 @@-= Uploading Data with Solr Cell using Apache Tika+= Indexing with Solr Cell and Apache Tika// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -70,7 +70,7 @@ that have a lot of rich media embedded in them.For these reasons, Solr Cell is not recommended for use in a production system.It is a best practice to use Solr Cell as a proof-of-concept tool during development and then run Tika as an external-process that sends the extracted documents to Solr (via <<using-solrj.adoc#,SolrJ>>) for indexing.+process that sends the extracted documents to Solr (via <<solrj.adoc#,SolrJ>>) for indexing.This way, any extraction failures that occur are isolated from Solr itself and can be handled gracefully.For a few examples of how this could be done, see this blog post by Erick Erickson, https://lucidworks.com/2012/02/14/indexing-with-solrj/[Indexing with SolrJ].@@ -115,7 +115,7 @@ bin/post -c gettingstarted example/exampledocs/solr-word.pdf -params "literal.idNow you can execute a query and find that document with a request like `\http://localhost:8983/solr/gettingstarted/select?q=pdf`. The document will look something like this:-image:images/solr-cell/sample-pdf-query.png[float="right",width=50%,pdfwidth=60%]+image::images/indexing-with-tika/sample-pdf-query.png[float="right",width=50%,pdfwidth=60%]You may notice there are many metadata fields associated with this document.Solr's configuration is by default in "schemaless" (data driven) mode, and thus all metadata fields extracted get their own field.@@ -365,8 +365,6 @@ myFileName = myPasswordFor a multi-core configuration, you can specify `sharedLib='lib'` in the `<solr/>` section of `solr.xml` and place the necessary jar files there.-For more information about Solr cores, see <<the-well-configured-solr-instance.adoc#,The Well-Configured Solr Instance>>.-=== Extending the ExtractingRequestHandlerIf you want to supply your own `ContentHandler` for Solr to use, you can extend the `ExtractingRequestHandler` and override the `createFactory()` method. This factory is responsible for constructing the `SolrContentHandler` that interacts with Tika, and allows literals to override Tika-parsed values. Set the parameter `literalsOverride`, which normally defaults to `true`, to `false` to append Tika-parsed values to literal values.@@ -466,7 +464,8 @@ curl "http://localhost:8983/solr/gettingstarted/update/extract?literal.id=doc6&d== Using Solr Cell with SolrJ-SolrJ is a Java client that you can use to add documents to the index, update the index, or query the index. You'll find more information on SolrJ in <<using-solrj.adoc#,Using SolrJ>>.+SolrJ is a Java client that you can use to add documents to the index, update the index, or query the index.+You'll find more information on SolrJ in <<solrj.adoc#,SolrJ>>.Here's an example of using Solr Cell and SolrJ to add documents to a Solr index.diff --git a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc b/solr/solr-ref-guide/src/indexing-with-update-handlers.adocsimilarity index 95%rename from solr/solr-ref-guide/src/uploading-data-with-index-handlers.adocrename to solr/solr-ref-guide/src/indexing-with-update-handlers.adocindex 80e1209a69c..73d88c88ca5 100644--- a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc+++ b/solr/solr-ref-guide/src/indexing-with-update-handlers.adoc@@ -1,4 +1,4 @@-= Uploading Data with Index Handlers+= Indexing with Update Handlers:page-children: transforming-and-indexing-custom-json// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file@@ -17,9 +17,11 @@// specific language governing permissions and limitations// under the License.-Index Handlers are Request Handlers designed to add, delete and update documents to the index. In addition to having plugins for importing rich documents <<uploading-data-with-solr-cell-using-apache-tika.adoc#,using Tika>>, Solr natively supports indexing structured documents in XML, CSV and JSON.+Update handlers are request handlers designed to add, delete and update documents to the index.+In addition to having plugins for importing rich documents <<indexing-with-tika.adoc#,Indexing with Solr Cell and Apache Tika>>, Solr natively supports indexing structured documents in XML, CSV, and JSON.-The recommended way to configure and use request handlers is with path based names that map to paths in the request url. However, request handlers can also be specified with the `qt` (query type) parameter if the <<requestdispatcher-in-solrconfig.adoc#,`requestDispatcher`>> is appropriately configured. It is possible to access the same handler using more than one name, which can be useful if you wish to specify different sets of default options.+The recommended way to configure and use request handlers is with path based names that map to paths in the request URL.+However, request handlers can also be specified with the `qt` (query type) parameter if the <<requestdispatcher.adoc#,`requestDispatcher`>> is appropriately configured. It is possible to access the same handler using more than one name, which can be useful if you wish to specify different sets of default options.A single unified update request handler supports XML, CSV, JSON, and javabin update requests, delegating to the appropriate `ContentStreamLoader` based on the `Content-Type` of the <<content-streams.adoc#,ContentStream>>.@@ -190,14 +192,16 @@ For posting XML messages contained in a file, you can use the alternative form:curl http://localhost:8983/solr/my_collection/update -H "Content-Type: text/xml" --data-binary @myfile.xml-----The approach above works well, but using the `--data-binary` option causes `curl` to load the whole `myfile.xml` into memory before posting it to server. This may be problematic when dealing with multi-gigabyte files. This alternative `curl` command performs equivalent operations but with minimal `curl` memory usage:+The approach above works well, but using the `--data-binary` option causes `curl` to load the whole `myfile.xml` into memory before posting it to server.+This may be problematic when dealing with multi-gigabyte files.+This alternative `curl` command performs equivalent operations but with minimal `curl` memory usage:[source,bash]----curl http://localhost:8983/solr/my_collection/update -H "Content-Type: text/xml" -T "myfile.xml" -X POST-----Short requests can also be sent using a HTTP GET command, if enabled in <<requestdispatcher-in-solrconfig.adoc#requestparsers-element,RequestDispatcher in SolrConfig>> element, URL-encoding the request, as in the following. Note the escaping of "<" and ">":+Short requests can also be sent using a HTTP GET command, if enabled in <<requestdispatcher.adoc#requestparsers-element,`requestParsers`>> element of `solrconfig.xml`, URL-encoding the request, as in the following. Note the escaping of "<" and ">":[source,bash]----diff --git a/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc b/solr/solr-ref-guide/src/initparams.adocsimilarity index 68%rename from solr/solr-ref-guide/src/initparams-in-solrconfig.adocrename to solr/solr-ref-guide/src/initparams.adocindex 3bc4282af84..fee57c5126f 100644--- a/solr/solr-ref-guide/src/initparams-in-solrconfig.adoc+++ b/solr/solr-ref-guide/src/initparams.adoc@@ -1,4 +1,5 @@-= InitParams in SolrConfig+= InitParams+:page-show-toc: false// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -20,12 +21,15 @@ The `<initParams>` section of `solrconfig.xml` allows you to define request handThere are a couple of use cases where this might be desired:-* Some handlers are implicitly defined in code - see <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>> - and there should be a way to add/append/override some of the implicitly defined properties.-* There are a few properties that are used across handlers. This helps you keep only a single definition of those properties and apply them over multiple handlers.+* Some handlers are implicitly defined in code (see <<implicit-requesthandlers.adoc#,Implicit Request Handlers>>) and there should be a way to add/append/override some of the implicitly defined properties.+* There are a few properties that are used across handlers.+This helps you keep only a single definition of those properties and apply them over multiple handlers.-For example, if you want several of your search handlers to return the same list of fields, you can create an `<initParams>` section without having to define the same set of parameters in each request handler definition. If you have a single request handler that should return different fields, you can define the overriding parameters in individual `<requestHandler>` sections as usual.+For example, if you want several of your search handlers to return the same list of fields, you can create an `<initParams>` section without having to define the same set of parameters in each request handler definition.+If you have a single request handler that should return different fields, you can define the overriding parameters in individual `<requestHandler>` sections as usual.-The properties and configuration of an `<initParams>` section mirror the properties and configuration of a request handler. It can include sections for defaults, appends, and invariants, the same as any request handler.+The properties and configuration of an `<initParams>` section mirror the properties and configuration of a request handler.+It can include sections for defaults, appends, and invariants, the same as any request handler.For example, here is one of the `<initParams>` sections defined by default in the `_default` example:@@ -38,15 +42,19 @@ For example, here is one of the `<initParams>` sections defined by default in th</initParams>-----This sets the default search field ("df") to be "_text_" for all of the request handlers named in the path section. If we later want to change the `/query` request handler to search a different field by default, we could override the `<initParams>` by defining the parameter in the `<requestHandler>` section for `/query`.+This sets the default search field ("df") to be "_text_" for all of the request handlers named in the path section.+If we later want to change the `/query` request handler to search a different field by default, we could override the `<initParams>` by defining the parameter in the `<requestHandler>` section for `/query`.The syntax and semantics are similar to that of a `<requestHandler>`. The following are the attributes:`path`::-A comma-separated list of paths which will use the parameters. Wildcards can be used in paths to define nested paths, as described below.+A comma-separated list of paths which will use the parameters.+Wildcards can be used in paths to define nested paths, as described below.`name`::-The name of this set of parameters. The name can be used directly in a requestHandler definition if a path is not explicitly named. If you give your `<initParams>` a name, you can refer to the parameters in a `<requestHandler>` that is not defined as a path.+The name of this set of parameters.+The name can be used directly in a requestHandler definition if a path is not explicitly named.+If you give your `<initParams>` a name, you can refer to the parameters in a `<requestHandler>` that is not defined as a path.+For example, if an `<initParams>` section has the name "myParams", you can call the name when defining your request handler:+@@ -55,7 +63,9 @@ For example, if an `<initParams>` section has the name "myParams", you can call== Wildcards in initParams-An `<initParams>` section can support wildcards to define nested paths that should use the parameters defined. A single asterisk (\*) denotes that a nested path one level deeper should use the parameters. Double asterisks (**) denote all nested paths no matter how deep should use the parameters.+An `<initParams>` section can support wildcards to define nested paths that should use the parameters defined.+A single asterisk (`\*`) denotes that a nested path one level deeper should use the parameters.+Double asterisks (`**`) denote all nested paths no matter how deep should use the parameters.For example, if we have an `<initParams>` that looks like this:@@ -96,7 +106,8 @@ Next we have a request handler named `/root/search5`:<requestHandler name="/root/search5" class="SearchHandler"/>-----We defined a wildcard for nested paths that are one level deeper than `/root`, so this request handler will use the parameters. This one, however, will not, because `/root/search5/test` is more than one level deep from `/root`:+We defined a wildcard for nested paths that are one level deeper than `/root`, so this request handler will use the parameters.+This one, however, will not, because `/root/search5/test` is more than one level deep from `/root`:[source,xml]----diff --git a/solr/solr-ref-guide/src/installation-deployment.adoc b/solr/solr-ref-guide/src/installation-deployment.adocnew file mode 100644index 00000000000..93a4811e851--- /dev/null+++ b/solr/solr-ref-guide/src/installation-deployment.adoc@@ -0,0 +1,45 @@+= Installation and Deployment+:page-children: system-requirements, \+ installing-solr, \+ taking-solr-to-production, \+ jvm-settings, \+ upgrading-a-solr-cluster, \+ backup-restore, \+ solr-in-docker, \+ solr-on-hdfs+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+This section covers system requirements, installation, upgrades, and deployments.++****+// This tags the below list so it can be used in the parent page section list+// tag::install-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<system-requirements.adoc#,System Requirements>>: Solr system requirements.+| <<installing-solr.adoc#,Installing Solr>>: Solr installation process.+| <<taking-solr-to-production.adoc#,Taking Solr to Production>>: Solr as a service and production considerations.+| <<jvm-settings.adoc#,JVM Settings>>: Java Virtual Machines best practices.+| <<upgrading-a-solr-cluster.adoc#,Upgrading a Solr Cluster>>: SolrCloud cluster upgrades.+| <<backup-restore.adoc#,Backup and Restore>>: Solr backup strategies.+| <<solr-in-docker#,Solr in Docker>>: Solr's Docker image.+| <<solr-on-hdfs.adoc#,Solr on HDFS>>: Store Solr indexes and transaction logs in HDFS.+|===+// end::install-sections[]+****diff --git a/solr/solr-ref-guide/src/installing-solr.adoc b/solr/solr-ref-guide/src/installing-solr.adocindex 2d66fd7e506..09d56c22d84 100644--- a/solr/solr-ref-guide/src/installing-solr.adoc+++ b/solr/solr-ref-guide/src/installing-solr.adoc@@ -19,7 +19,7 @@Installation of Solr on Unix-compatible or Windows servers generally requires simply extracting (or, unzipping) the download package.-Please be sure to review the <<solr-system-requirements.adoc#,Solr System Requirements>> before starting Solr.+Please be sure to review the <<system-requirements.adoc#,System Requirements>> before starting Solr.== Available Solr Packages@@ -47,7 +47,7 @@ How to size your Solr installation is a complex question that relies on a numberIt's highly recommended that you spend a bit of time thinking about the factors that will impact hardware sizing for your Solr implementation. A very good blog post that discusses the issues to consider is https://lucidworks.com/2012/07/23/sizing-hardware-in-the-abstract-why-we-dont-have-a-definitive-answer/[Sizing Hardware in the Abstract: Why We Don't have a Definitive Answer].====-One thing to note when planning your installation is that a hard limit exists in Lucene for the number of documents in a single index: approximately 2.14 billion documents (2,147,483,647 to be exact). In practice, it is highly unlikely that such a large number of documents would fit and perform well in a single index, and you will likely need to distribute your index across a cluster before you ever approach this number. If you know you will exceed this number of documents in total before you've even started indexing, it's best to plan your installation with <<solrcloud.adoc#,SolrCloud>> as part of your design from the start.+One thing to note when planning your installation is that a hard limit exists in Lucene for the number of documents in a single index: approximately 2.14 billion documents (2,147,483,647 to be exact). In practice, it is highly unlikely that such a large number of documents would fit and perform well in a single index, and you will likely need to distribute your index across a cluster before you ever approach this number. If you know you will exceed this number of documents in total before you've even started indexing, it's best to plan your installation with <<cluster-types.adoc#solrcloud-mode,SolrCloud>> as part of your design from the start.== Package Installation@@ -151,7 +151,7 @@ bin/solr -e techproductsCurrently, the available examples you can run are: techproducts, schemaless, and cloud. See the section <<solr-control-script-reference.adoc#running-with-example-configurations,Running with Example Configurations>> for details on each example..Getting Started with SolrCloud-NOTE: Running the `cloud` example starts Solr in <<solrcloud.adoc#,SolrCloud>> mode. For more information on starting Solr in SolrCloud mode, see the section <<getting-started-with-solrcloud.adoc#,Getting Started with SolrCloud>>.+NOTE: Running the `cloud` example starts Solr in <<cluster-types.adoc#solrcloud-mode,SolrCloud>> mode. For more information on starting Solr in SolrCloud mode, see the section <<tutorial-solrcloud.adoc#,Getting Started with SolrCloud>>.=== Check if Solr is Running@@ -169,7 +169,7 @@ That's it! Solr is running. If you need convincing, use a Web browser to see the`\http://localhost:8983/solr/`.The Solr Admin interface.-image::images/running-solr/SolrAdminDashboard.png[Solr's Admin UI,pdfwidth=75%]+image::images/installing-solr/SolrAdminDashboard.png[Solr's Admin UI,pdfwidth=75%]If Solr is not running, your browser will complain that it cannot connect to the server. Check your port number and try again.diff --git a/solr/solr-ref-guide/src/introduction-to-client-apis.adoc b/solr/solr-ref-guide/src/introduction-to-client-apis.adocdeleted file mode 100644index ecb59d6d7c4..00000000000--- a/solr/solr-ref-guide/src/introduction-to-client-apis.adoc+++ /dev/null@@ -1,29 +0,0 @@-= Introduction to Client APIs-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--At its heart, Solr is a Web application, but because it is built on open protocols, any type of client application can use Solr.--HTTP is the fundamental protocol used between client applications and Solr. The client makes a request and Solr does some work and provides a response. Clients use requests to ask Solr to do things like perform queries or index documents.--Client applications can reach Solr by creating HTTP requests and parsing the HTTP responses. Client APIs encapsulate much of the work of sending requests and parsing responses, which makes it much easier to write client applications.--Clients use Solr's five fundamental operations to work with Solr. The operations are query, index, delete, commit, and optimize.--Queries are executed by creating a URL that contains all the query parameters. Solr examines the request URL, performs the query, and returns the results. The other operations are similar, although in certain cases the HTTP request is a POST operation and contains information beyond whatever is included in the request URL. An index operation, for example, may contain a document in the body of the request.--Solr also features an EmbeddedSolrServer that offers a Java API without requiring an HTTP connection. For details, see <<using-solrj.adoc#,Using SolrJ>>.diff --git a/solr/solr-ref-guide/src/introduction-to-scaling-and-distribution.adoc b/solr/solr-ref-guide/src/introduction-to-scaling-and-distribution.adocdeleted file mode 100644index 19a1941b318..00000000000--- a/solr/solr-ref-guide/src/introduction-to-scaling-and-distribution.adoc+++ /dev/null@@ -1,43 +0,0 @@-= Introduction to Scaling and Distribution-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Both Lucene and Solr were designed to scale to support large implementations with minimal custom coding.--This section covers:--* <<distributed-search-with-index-sharding.adoc#,distributing>> an index across multiple servers-* <<index-replication.adoc#,replicating>> an index on multiple servers-* <<merging-indexes.adoc#,merging indexes>>--If you need full scale distribution of indexes and queries, as well as replication, load balancing and failover, you may want to use SolrCloud. Full details on configuring and using SolrCloud is available in the section <<solrcloud.adoc#,SolrCloud>>.--== What Problem Does Distribution Solve?--If searches are taking too long or the index is approaching the physical limitations of its machine, you should consider distributing the index across two or more Solr servers.--To distribute an index, you divide the index into partitions called shards, each of which runs on a separate machine. Solr then partitions searches into sub-searches, which run on the individual shards, reporting results collectively.--The architectural details underlying index sharding are invisible to end users, who simply experience faster performance on queries against very large indexes.--== What Problem Does Replication Solve?--Replicating an index is useful when:--* You have a large search volume which one machine cannot handle, so you need to distribute searches across multiple read-only copies of the index.-* There is a high volume/high rate of indexing which consumes machine resources and reduces search performance on the indexing machine, so you need to separate indexing and searching.-* You want to make a backup of the index (see <<making-and-restoring-backups.adoc#,Making and Restoring Backups>>).diff --git a/solr/solr-ref-guide/src/introduction.adoc b/solr/solr-ref-guide/src/introduction.adocnew file mode 100644index 00000000000..28fe32d7ef3--- /dev/null+++ b/solr/solr-ref-guide/src/introduction.adoc@@ -0,0 +1,43 @@+= Introduction to Solr+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr is a search server built on top of https://lucene.apache.org[Apache Lucene], an open source, Java-based, information retrieval library.+It is designed to drive powerful document retrieval applications - wherever you need to serve data to users based on their queries, Solr can work for you.++Solr is based on open standards and it is highly extensible.+Solr queries are simple HTTP request URLs and the response is a structured document: mainly JSON, but it could also be XML, CSV, or other formats.+This means that a wide variety of clients will be able to use Solr, from other web applications to browser clients, rich client applications, and mobile devices.+Any platform capable of HTTP can talk to Solr.+See <<client-apis.adoc#,Client APIs>> for details on client APIs.++Flexible schema configurations allow nearly any type of data to be stored in Solr.+The <<schema-indexing-guide.adoc#,Schema and Indexing Guide>> has more details on these options.++Solr offers support for the simplest keyword searching through to complex queries on multiple fields and faceted search results.+Collapsing and clustering results offer compelling features for e-commerce and storefronts.+Powerful math expressions provide the backbone for advanced analytics use cases.+The <<query-guide.adoc#,Query Guide>> has more information about searching and queries.++If Solr's capabilities are not impressive enough, its ability to handle very high-volume applications should do the trick.++A relatively common scenario is that you have so much data, or so many queries, that a single Solr server is unable to handle your entire workload.+In this case, you can scale up the capabilities of your application using <<scaling-solr.adoc#,SolrCloud>> to better distribute the data, and the processing of requests, across many servers.+Multiple options can be mixed and matched depending on the scalability you need.++Best of all, this talk about high-volume applications is not just hypothetical: some of the famous Internet sites that use Solr today are Macy's, EBay, and Zappo's.+For more examples, take a look at https://cwiki.apache.org/confluence/display/solr/PublicServers.diff --git a/solr/solr-ref-guide/src/java-properties.adoc b/solr/solr-ref-guide/src/java-properties.adocdeleted file mode 100644index f6afb7dd610..00000000000--- a/solr/solr-ref-guide/src/java-properties.adoc+++ /dev/null@@ -1,22 +0,0 @@-= Java Properties-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Java Properties screen provides easy access to one of the most essential components of a top-performing Solr systems. With the Java Properties screen, you can see all the properties of the JVM running Solr, including the class paths, file encodings, JVM memory settings, operating system, and more.--.Java Properties Screen-image::images/java-properties/javaproperties.png[image,width=593,height=250]diff --git a/solr/solr-ref-guide/src/using-javascript.adoc b/solr/solr-ref-guide/src/javascript.adocsimilarity index 81%rename from solr/solr-ref-guide/src/using-javascript.adocrename to solr/solr-ref-guide/src/javascript.adocindex 64c8894f7a1..b4e90574445 100644--- a/solr/solr-ref-guide/src/using-javascript.adoc+++ b/solr/solr-ref-guide/src/javascript.adoc@@ -1,4 +1,4 @@-= Using JavaScript+= JavaScript// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,11 +16,14 @@// specific language governing permissions and limitations// under the License.-Using Solr from JavaScript clients is so straightforward that it deserves a special mention. In fact, it is so straightforward that there is no client API. You don't need to install any packages or configure anything.+Using Solr from JavaScript clients is so straightforward that it deserves a special mention.+In fact, it is so straightforward that there is no client API.+You don't need to install any packages or configure anything.HTTP requests can be sent to Solr using the standard `XMLHttpRequest` mechanism.-By default, Solr sends <<response-writers.adoc#json-response-writer,JavaScript Object Notation (JSON) responses>>, which are easily interpreted in JavaScript. You don't need to add anything to the request URL to have responses sent as JSON.+By default, Solr sends <<response-writers.adoc#json-response-writer,JavaScript Object Notation (JSON) responses>>, which are easily interpreted in JavaScript.+You don't need to add anything to the request URL to have responses sent as JSON.For more information and an excellent example, take a look at the SolJSON page on the Solr Wiki:diff --git a/solr/solr-ref-guide/src/jdbc-dbvisualizer.adoc b/solr/solr-ref-guide/src/jdbc-dbvisualizer.adocnew file mode 100644index 00000000000..fba53e8e4a5--- /dev/null+++ b/solr/solr-ref-guide/src/jdbc-dbvisualizer.adoc@@ -0,0 +1,126 @@+= JDBC with DbVisualizer+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr's JDBC driver supports https://www.dbvis.com/[DbVisualizer] for querying Solr.++For DbVisualizer, you will need to create a new driver for Solr using the DbVisualizer Driver Manager.+This will add several SolrJ client .jars to the DbVisualizer classpath.+The files required are:++* all .jars found in `$SOLR_HOME/dist/solrj-lib`+* the SolrJ .jar found at `$SOLR_HOME/dist/solr-solrj-<version>.jar`++Once the driver has been created, you can create a connection to Solr with the connection string format outlined in the generic section and use the SQL Commander to issue queries.++== Setup Driver++=== Open Driver Manager++From the Tools menu, choose Driver Manager to add a driver.++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.png[image,width=673,height=400]+++=== Create a New Driver++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.png[image,width=532,height=400]+++=== Name the Driver in Driver Manager++Provide a name for the driver, and provide the URL format: `jdbc:solr://<zk_connection_string>/?collection=<collection>`.+Do not fill in values for the variables "```zk_connection_string```" and "```collection```", those will be provided later when the connection to Solr is configured.+The Driver Class will also be automatically added when the driver .jars are added.++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.png[image,width=532,height=400]+++=== Add Driver Files to Classpath++The driver files to be added are:++* all .jars in `$SOLR_HOME/dist/solrj-lib`+* the SolrJ .jar found in `$SOLR_HOME/dist/solr-solrj-<version>.jar`++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.png[image,width=535,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.png[image,width=664,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.png[image,width=653,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.png[image,width=655,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.png[image,width=651,height=400]+++=== Review and Close Driver Manager++Once the driver files have been added, you can close the Driver Manager.++== Create a Connection++Next, create a connection to Solr using the driver just created.++=== Use the Connection Wizard++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.png[image,width=763,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.png[image,width=807,height=400]+++=== Name the Connection++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.png[image,width=402,height=400]+++=== Select the Solr driver++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.png[image,width=399,height=400]+++=== Specify the Solr URL++Provide the Solr URL, using the ZooKeeper host and port and the collection.+For example, `jdbc:solr://localhost:9983?collection=test`++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.png[image,width=401,height=400]+++== Open and Connect to Solr++Once the connection has been created, double-click on it to open the connection details screen and connect to Solr.++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.png[image,width=625,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.png[image,width=592,height=400]+++== Open SQL Commander to Enter Queries++When the connection is established, you can use the SQL Commander to issue queries and view data.++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.png[image,width=577,height=400]+++image::images/jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.png[image,width=556,height=400]diff --git a/solr/solr-ref-guide/src/solr-jdbc-python-jython.adoc b/solr/solr-ref-guide/src/jdbc-python-jython.adocsimilarity index 91%rename from solr/solr-ref-guide/src/solr-jdbc-python-jython.adocrename to solr/solr-ref-guide/src/jdbc-python-jython.adocindex b75157f4d7a..a432deec70d 100644--- a/solr/solr-ref-guide/src/solr-jdbc-python-jython.adoc+++ b/solr/solr-ref-guide/src/jdbc-python-jython.adoc@@ -1,4 +1,4 @@-= Solr JDBC - Python/Jython+= JDBC with Python/Jython// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -20,7 +20,8 @@ Solr's JDBC driver supports Python and Jython.== Python-Python supports accessing JDBC using the https://pypi.python.org/pypi/JayDeBeApi/[JayDeBeApi] library. The CLASSPATH variable must be configured to contain the solr-solrj jar and the supporting solrj-lib jars.+Python supports accessing JDBC using the https://pypi.python.org/pypi/JayDeBeApi/[JayDeBeApi] library.+The CLASSPATH variable must be configured to contain the `solr-solrj` jar and the supporting `solrj-lib` jars.=== JayDeBeApi@@ -64,7 +65,8 @@ if __name__ == '__main__':== Jython-Jython supports accessing JDBC natively with Java interfaces or with the zxJDBC library. The CLASSPATH variable must be configured to contain the solr-solrj jar and the supporting solrj-lib jars.+Jython supports accessing JDBC natively with Java interfaces or with the zxJDBC library.+The CLASSPATH variable must be configured to contain the solr-solrj jar and the supporting solrj-lib jars..run.sh[source,bash]diff --git a/solr/solr-ref-guide/src/solr-jdbc-r.adoc b/solr/solr-ref-guide/src/jdbc-r.adocsimilarity index 99%rename from solr/solr-ref-guide/src/solr-jdbc-r.adocrename to solr/solr-ref-guide/src/jdbc-r.adocindex 90371359228..4305e99a008 100644--- a/solr/solr-ref-guide/src/solr-jdbc-r.adoc+++ b/solr/solr-ref-guide/src/jdbc-r.adoc@@ -1,4 +1,4 @@-= Solr JDBC - R+= JDBC with R// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional informationdiff --git a/solr/solr-ref-guide/src/solr-jdbc-squirrel-sql.adoc b/solr/solr-ref-guide/src/jdbc-squirrel.adocsimilarity index 51%rename from solr/solr-ref-guide/src/solr-jdbc-squirrel-sql.adocrename to solr/solr-ref-guide/src/jdbc-squirrel.adocindex acaa0d72d62..48017cf1d81 100644--- a/solr/solr-ref-guide/src/solr-jdbc-squirrel-sql.adoc+++ b/solr/solr-ref-guide/src/jdbc-squirrel.adoc@@ -1,4 +1,4 @@-= Solr JDBC - SQuirreL SQL+= JDBC with SQuirreL SQL// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,7 +16,11 @@// specific language governing permissions and limitations// under the License.-For http://squirrel-sql.sourceforge.net[SQuirreL SQL], you will need to create a new driver for Solr. This will add several SolrJ client .jars to the SQuirreL SQL classpath. The files required are:+Solr's JDBC driver supports http://squirrel-sql.sourceforge.net[SQuirreL SQL].++To configure this, you will need to create a new driver for Solr.+This will add several SolrJ client .jars to the SQuirreL SQL classpath.+The files required are:* all .jars found in `$SOLR_HOME/dist/solrj-libs`* the SolrJ .jar found at `$SOLR_HOME/dist/solr-solrj-<version>.jar`@@ -27,40 +31,41 @@ Once the driver has been created, you can create a connection to Solr with the c=== Open Drivers-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_1.png[image,width=900,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_1.png[image,width=900,height=400]=== Add Driver-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_2.png[image,width=892,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_2.png[image,width=892,height=400]=== Name the Driver-Provide a name for the driver, and provide the URL format: `jdbc:solr://<zk_connection_string>/?collection=<collection>`. Do not fill in values for the variables "```zk_connection_string```" and "```collection```", those will be defined later when the connection to Solr is configured.+Provide a name for the driver, and provide the URL format: `jdbc:solr://<zk_connection_string>/?collection=<collection>`.+Do not fill in values for the variables "```zk_connection_string```" and "```collection```", those will be defined later when the connection to Solr is configured.-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_3.png[image,width=467,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_3.png[image,width=467,height=400]=== Add Solr JDBC jars to Classpath-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_4.png[image,width=467,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_4.png[image,width=467,height=400]-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_9.png[image,width=469,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_9.png[image,width=469,height=400]-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_5.png[image,width=469,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_5.png[image,width=469,height=400]-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_7.png[image,width=467,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_7.png[image,width=467,height=400]=== Add the Solr JDBC driver class nameAfter adding the .jars, you will need to additionally define the Class Name `org.apache.solr.client.solrj.io.sql.DriverImpl`.-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_11.png[image,width=470,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_11.png[image,width=470,height=400]== Create an Alias@@ -69,26 +74,26 @@ To define a JDBC connection, you must define an alias.=== Open Aliases-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_10.png[image,width=840,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_10.png[image,width=840,height=400]=== Add an Alias-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_12.png[image,width=959,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_12.png[image,width=959,height=400]=== Configure the Alias-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_14.png[image,width=470,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_14.png[image,width=470,height=400]=== Connect to the Alias-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_13.png[image,width=522,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_13.png[image,width=522,height=400]== QueryingOnce you've successfully connected to Solr, you can use the SQL interface to enter queries and work with data.-image::images/solr-jdbc-squirrel-sql/squirrelsql_solrjdbc_15.png[image,width=655,height=400]+image::images/jdbc-squirrel/squirrelsql_solrjdbc_15.png[image,width=655,height=400]diff --git a/solr/solr-ref-guide/src/jdbc-zeppelin.adoc b/solr/solr-ref-guide/src/jdbc-zeppelin.adocnew file mode 100644index 00000000000..672a63e8b53--- /dev/null+++ b/solr/solr-ref-guide/src/jdbc-zeppelin.adoc@@ -0,0 +1,91 @@+= JDBC with Apache Zeppelin+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++The Solr JDBC driver can support http://zeppelin.apache.org/[Apache Zeppelin].++[NOTE]+====+A Solr-specific interpreter is available as an alternative to using the JDBC driver.+See the section <<math-start.adoc#zeppelin-solr-interpreter,Zeppelin-Solr Interpreter>> for details.++If you choose to use Zeppelin's JDBC interpreter instead, you must use Zeppelin 0.6.0 or greater to get JDBC support.+====++To use Apache Zeppelin with Solr, you will need to create a JDBC interpreter for Solr.+This will add SolrJ to the interpreter classpath.+Once the interpreter has been created, you can create a notebook to issue queries.+The http://zeppelin.apache.org/docs/latest/interpreter/jdbc.html[Zeppelin JDBC interpreter documentation] provides additional information about JDBC prefixes and other features.++== Create the Apache Solr JDBC Interpreter++.Click "Interpreter" in the top navigation+image::images/jdbc-zeppelin/zeppelin_solrjdbc_1.png[image,height=400]++.Click "Create"+image::images/jdbc-zeppelin/zeppelin_solrjdbc_2.png[image,height=400]++.Enter information about your Solr installation+image::images/jdbc-zeppelin/zeppelin_solrjdbc_3.png[image,height=400]++[NOTE]+====+For most installations, Apache Zeppelin configures PostgreSQL as the JDBC interpreter default driver.+The default driver can either be replaced by the Solr driver as outlined above or you can add a separate JDBC interpreter prefix as outlined in the http://zeppelin.apache.org/docs/latest/interpreter/jdbc.html[Apache Zeppelin JDBC interpreter documentation].+====++== Create a Notebook++.Click Notebook -> Create new note+image::images/jdbc-zeppelin/zeppelin_solrjdbc_4.png[image,width=517,height=400]++.Provide a name and click "Create Note"+image::images/jdbc-zeppelin/zeppelin_solrjdbc_5.png[image,width=839,height=400]++== JDBC Interpreter Copy Sheet++To facilitate easy copying the parameters mentioned in the screenshots, here is a consolidated list of the parameters:++[source,text,subs=attributes]+----+Name : Solr+Interpreter : jdbc+default.url : jdbc:solr://SOLR_ZK_CONNECTION_STRING?collection=<collection_name>+default.driver : org.apache.solr.client.solrj.io.sql.DriverImpl+default.user : solr+dependency : org.apache.solr:solr-solrj:{solr-docs-version}.0+----++== Query with the Notebook++[IMPORTANT]+====+For some notebooks, the JDBC interpreter will not be bound to the notebook by default.+Instructions on how to bind the JDBC interpreter to a notebook are available https://zeppelin.apache.org/docs/latest/interpreter/jdbc.html#bind-to-notebook[here].+====++.Results of Solr query+image::images/jdbc-zeppelin/zeppelin_solrjdbc_6.png[image,width=481,height=400]++The below code block assumes that the Apache Solr driver is setup as the default JDBC interpreter driver.+If that is not the case, instructions for using a different prefix is available https://zeppelin.apache.org/docs/latest/interpreter/jdbc.html#how-to-use[here].++[source,text]+----+%jdbc+select fielda, fieldb, from test limit 10+----diff --git a/solr/solr-ref-guide/src/using-jmx-with-solr.adoc b/solr/solr-ref-guide/src/jmx-with-solr.adocsimilarity index 77%rename from solr/solr-ref-guide/src/using-jmx-with-solr.adocrename to solr/solr-ref-guide/src/jmx-with-solr.adocindex d65cbbf7b14..252659d7415 100644--- a/solr/solr-ref-guide/src/using-jmx-with-solr.adoc+++ b/solr/solr-ref-guide/src/jmx-with-solr.adoc@@ -1,4 +1,4 @@-= Using JMX with Solr+= JMX with Solr// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,9 +16,11 @@// specific language governing permissions and limitations// under the License.-http://www.oracle.com/technetwork/java/javase/tech/javamanagement-140525.html[Java Management Extensions (JMX)] is a technology that makes it possible for complex systems to be controlled by tools without the systems and tools having any previous knowledge of each other. In essence, it is a standard interface by which complex systems can be viewed and manipulated.+http://www.oracle.com/technetwork/java/javase/tech/javamanagement-140525.html[Java Management Extensions (JMX)] is a technology that makes it possible for complex systems to be controlled by tools without the systems and tools having any previous knowledge of each other.+In essence, it is a standard interface by which complex systems can be viewed and manipulated.-Solr, like any other good citizen of the Java universe, can be controlled via a JMX interface. Once enabled, you can use a JMX client, like jconsole, to connect with Solr.+Solr, like any other good citizen of the Java universe, can be controlled via a JMX interface.+Once enabled, you can use a JMX client, like jconsole, to connect with Solr.If you are unfamiliar with JMX, you may find the following overview useful: http://docs.oracle.com/javase/8/docs/technotes/guides/management/agent.html.@@ -30,13 +32,16 @@ If you have an existing MBean server running in Solr's JVM, or if you start Solr== Configuring MBean Servers-Versions of Solr prior to 7.0 defined JMX support in `solrconfig.xml`. This has been changed to the metrics reporter configuration defined above. Parameters for the reporter configuration allow defining the location or address of an existing MBean server.+Versions of Solr prior to 7.0 defined JMX support in `solrconfig.xml`. This has been changed to the metrics reporter configuration defined above.+Parameters for the reporter configuration allow defining the location or address of an existing MBean server.An MBean server can be started at the time of Solr's startup by passing the system parameter `-Dcom.sun.management.jmxremote`. See Oracle's documentation for additional settings available to start and control an MBean server at http://docs.oracle.com/javase/8/docs/technotes/guides/management/agent.html.=== Configuring a Remote Connection to Solr JMX-If you need to attach a JMX-enabled Java profiling tool, such as JConsole or VisualVM, to a remote Solr server, then you need to enable remote JMX access when starting the Solr server. Simply change the `ENABLE_REMOTE_JMX_OPTS` property in the `solr.in.sh` or `solr.in.cmd` (for Windows) file to true. You’ll also need to choose a port for the JMX RMI connector to bind to, such as 18983. For example, if your Solr include script sets:+If you need to attach a JMX-enabled Java profiling tool, such as JConsole or VisualVM, to a remote Solr server, then you need to enable remote JMX access when starting the Solr server.+Simply change the `ENABLE_REMOTE_JMX_OPTS` property in the `solr.in.sh` or `solr.in.cmd` (for Windows) file to `true`.+You’ll also need to choose a port for the JMX RMI connector to bind to, such as 18983. For example, if your Solr include script sets:[source,bash]----@@ -62,5 +67,7 @@ For more information about these settings, see: http://docs.oracle.com/javase/8/[IMPORTANT]====-Making JMX connections into machines running behind NATs (e.g., Amazon's EC2 service) is not a simple task. The `java.rmi.server.hostname` system property may help, but running `jconsole` on the server itself and using a remote desktop is often the simplest solution. See http://web.archive.org/web/20130525022506/http://jmsbrdy.com/monitoring-java-applications-running-on-ec2-i.+Making JMX connections into machines running behind NATs (e.g., Amazon's EC2 service) is not a simple task.+The `java.rmi.server.hostname` system property may help, but running `jconsole` on the server itself and using a remote desktop is often the simplest solution.+See http://web.archive.org/web/20130525022506/http://jmsbrdy.com/monitoring-java-applications-running-on-ec2-i.====diff --git a/solr/solr-ref-guide/src/join-query-parser.adoc b/solr/solr-ref-guide/src/join-query-parser.adocnew file mode 100644index 00000000000..b1536ca297b--- /dev/null+++ b/solr/solr-ref-guide/src/join-query-parser.adoc@@ -0,0 +1,237 @@+= Join Query Parser+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License++The Join query parser allows users to run queries that normalize relationships between documents.++Solr runs a subquery of the user's choosing (the `v` param), identifies all the values that matching documents have in a field of interest (the `from` param), and then returns documents where those values are contained in a second field of interest (the `to` param).++In practice, these semantics are much like "inner queries" in a SQL engine.+As an example, consider the Solr query below:++[source,text]+----+/solr/techproducts/select?q={!join from=manu_id_s to=id}title:ipod+----++This query, which returns a document for each manufacturer that makes a product with "ipod" in the title, is semantically identical to the SQL query below:++[source,text]+----+SELECT *+FROM techproducts+WHERE id IN (+ SELECT manu_id_s+ FROM techproducts+ WHERE title='ipod'+ )+----++The join operation is done on a term basis, so the `from` and `to` fields must use compatible field types.+For example: joining between a `StrField` and a `IntPointField` will not work.+Likewise joining between a `StrField` and a `TextField` that uses `LowerCaseFilterFactory` will only work for values that are already lower cased in the string field.++== Parameters++This query parser takes the following parameters:++`from`::+The name of a field which contains values to look for in the "to" field.+Can be single or multi-valued, but must have a field type compatible with the field represented in the "to" field.+This parameter is required.++`to`::+The name of a field whose value(s) will be checked against those found in the "from" field.+Can be single or multi-valued, but must have a field type compatible with the "from" field.+This parameter is required.++`fromIndex`::+The name of the index to run the "from" query (`v` parameter) on and where "from" values are gathered.+Must be located on the same node as the core processing the request.+This parameter is optional; it defaults to the value of the processing core if not specified.+See <<Joining Across Single Shard Collections,Joining Across Single Shard Collections>> or <<Cross Collection Join,Cross Collection Join>> below for more information.++`score`::+An optional parameter that instructs Solr to return information about the "from" query scores.+The value of this parameter controls what type of aggregation information is returned.+Options include `avg` (average), `max` (maximum), `min` (minimum), `total` (total), or `none`.+++If `method` is not specified but `score` is, then the `dvWithScore` method is used.+If `method` is specified and is not `dvWithScore`, then the `score` value is ignored.+See the `method` parameter documentation below for more details.+++`method`::+An optional parameter used to determine which of several query implementations should be used by Solr.+Options are restricted to: `index`, `dvWithScore`, and `topLevelDV`.+If unspecified the default value is `index`, unless the `score` parameter is present which overrides it to `dvWithScore`.+Each implementation has its own performance characteristics, and users are encouraged to experiment to determine which implementation is most performant for their use-case.+Details and performance heuristics are given below.+++`index` the default `method` unless the `score` parameter is specified.+Uses the terms index structures to process the request.+Performance scales with the cardinality and number of postings (term occurrences) in the "from" field.+Consider this method when the "from" field has low cardinality, when the "to" side returns a large number of documents, or when sporadic post-commit slowdowns cannot be tolerated (this is a disadvantage of other methods that `index` avoids).++++`dvWithScore` returns an optional "score" statistic alongside result documents.+Uses docValues structures if available, but falls back to the field cache when necessary.+The first access to the field cache slows down the initial requests following a commit and takes up additional space on the JVM heap, so docValues are recommended in most situations.+Performance scales linearly with the number of values matched in the "from" field.+This method must be used if score information is required, and should also be considered when the "from" query matches few documents, regardless of the number of "to" side documents returned.+++.dvWithScore and single value numerics+[WARNING]+====+The `dvWithScore` method doesn't support single value numeric fields. Users migrating from versions prior to 7.0 are encouraged to change field types to string and rebuild indexes during migration.+====+++`topLevelDV` can only be used when `to` and `from` fields have docValues data, and does not currently support numeric fields.+Uses top-level docValues data structures to find results.+These data structures outperform other methods as the number of values matched in the `from` field grows high.+But they are also expensive to build and need to be lazily populated after each commit, causing a sometimes-noticeable slowdown on the first query to use them after each commit.+If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.+Consider this method when the "from" query matches a large number of documents and the "to" result set is small to moderate in size, but only if sporadic post-commit slowness is tolerable.++== Joining Across Single Shard Collections++You can also specify a `fromIndex` parameter to join with a field from another core or a single shard collection. If running in SolrCloud mode, then the collection specified in the `fromIndex` parameter must have a single shard and a replica on all Solr nodes where the collection you're joining to has a replica.++Let's consider an example where you want to use a Solr join query to filter movies by directors that have won an Oscar. Specifically, imagine we have two collections with the following fields:++*movies*: id, title, director_id, ...++*movie_directors*: id, name, has_oscar, ...++To filter movies by directors that have won an Oscar using a Solr join on the *movie_directors* collection, you can send the following filter query to the *movies* collection:++[source,text]+----+fq={!join from=id fromIndex=movie_directors to=director_id}has_oscar:true+----++Notice that the query criteria of the filter (`has_oscar:true`) is based on a field in the collection specified using `fromIndex`. Keep in mind that you cannot return fields from the `fromIndex` collection using join queries, you can only use the fields for filtering results in the "to" collection (movies).++Next, let's understand how these collections need to be deployed in your cluster. Imagine the *movies* collection is deployed to a four node SolrCloud cluster and has two shards with a replication factor of two. Specifically, the *movies* collection has replicas on the following four nodes:++node 1: movies_shard1_replica1++node 2: movies_shard1_replica2++node 3: movies_shard2_replica1++node 4: movies_shard2_replica2++To use the *movie_directors* collection in Solr join queries with the *movies* collection, it needs to have a replica on each of the four nodes. In other words, *movie_directors* must have one shard and replication factor of four:++node 1: movie_directors_shard1_replica1++node 2: movie_directors_shard1_replica2++node 3: movie_directors_shard1_replica3++node 4: movie_directors_shard1_replica4++At query time, the `JoinQParser` will access the local replica of the *movie_directors* collection to perform the join. If a local replica is not available or active, then the query will fail. At this point, it should be clear that since you're limited to a single shard and the data must be replicated across all nodes where it is needed, this approach works better with smaller data sets where there is a one-to-many relationship between the from collection and the to collection. Moreover, if you add a replica to the to collection, then you also need to add a replica for the from collection.++For more information, Erick Erickson has written a blog post about join performance titled https://lucidworks.com/2012/06/20/solr-and-joins/[Solr and Joins].++== Cross Collection Join+The Cross Collection Join Filter is a method for the join parser that will execute a query against a remote Solr collection to get back a set of join keys that will be used to as a filter query against the local Solr collection.++The crossCollection join query will create an CrossCollectionQuery object.+The CrossCollectionQuery will first query a remote Solr collection and get back a streaming expression result of the join keys.+As the join keys are streamed to the node, a bitset of the matching documents in the local index is built up.+This avoids keeping the full set of join keys in memory at any given time.+This bitset is then inserted into the filter cache upon successful execution as with the normal behavior of the Solr filter cache.++If the local index is sharded according to the join key field, the cross collection join can leverage a secondary query parser called the "hash_range" query parser.+The hash_range query parser is responsible for returning only the documents that hash to a given range of values.+This allows the CrossCollectionQuery to query the remote Solr collection and return only the join keys that would match a specific shard in the local Solr collection.+This has the benefit of making sure that network traffic doesn't increase as the number of shards increases and allows for much greater scalability.++The CrossCollection join query works with both String and Point types of fields.+The fields that are being used for the join key must be single-valued and have docValues enabled.++It's advised to shard the local collection by the join key as this allows for the optimization mentioned above to be utilized.++The cross collection join queries should not be generally used as part of the `q` parameter, but rather it is designed to be used as a filter query (`fq` parameter) to ensure proper caching.++The remote Solr collection that is being queried should have a single-valued field for the join key with docValues enabled.++The remote Solr collection does not have any specific sharding requirements.++=== Join Query Parser Definition in solrconfig.xml++The cross collection join has some configuration options that can be specified in `solrconfig.xml`.++`routerField`::+If the documents are routed to shards using the CompositeID router by the join field, then that field name should be specified in the configuration here. This will allow the parser to optimize the resulting HashRange query.++`solrUrl`::+If specified, this array of strings specifies the white listed Solr URLs that you can pass to the solrUrl query parameter. Without this configuration the solrUrl parameter cannot be used. This restriction is necessary to prevent an attacker from using Solr to explore the network.++[source,xml]+----+ <queryParser name="join" class="org.apache.solr.search.JoinQParserPlugin">+ <str name="routerField">product_id_s</str>+ <arr name="allowSolrUrls">+ <str>http://othersolr.example.com:8983/solr</str>+ </arr>+ </queryParser>+----++=== Cross Collection Join Query Parameters++`fromIndex`::+The name of the external Solr collection to be queried to retrieve the set of join key values (required).++`zkHost`::+The connection string to be used to connect to ZooKeeper. `zkHost` and `solrUrl` are both optional parameters, and at most one of them should be specified. If neither `zkHost` nor `solrUrl` are specified, the local ZooKeeper cluster will be used. (optional).++`solrUrl`::+The URL of the external Solr node to be queried. Must be a character for character exact match of a whitelisted url. (optional, disabled by default for security).++`from`::+The join key field name in the external collection (required).++`to`::+The join key field name in the local collection.++`v`::+The query substituted in as a local param. This is the query string that will match documents in the remote collection.++`routed`::+If `true`, the cross collection join query will use each shard's hash range to determine the set of join keys to retrieve for that shard.+This parameter improves the performance of the cross-collection join, but it depends on the local collection being routed by the `to` field.+If this parameter is not specified, the cross collection join query will try to determine the correct value automatically.++`ttl`::+The length of time that a cross collection join query in the cache will be considered valid, in seconds.+Defaults to `3600` (one hour).+The cross collection join query will not be aware of changes to the remote collection, so if the remote collection is updated, cached cross collection queries may give inaccurate results.+After the `ttl` period has expired, the cross collection join query will re-execute the join against the remote collection.++Other Parameters::+Any normal Solr query parameter can also be specified/passed through as a local param.++=== Cross Collection Query Examples++[source,text]+----+http://localhost:8983/solr/localCollection/query?fl=id&q={!join method="crossCollection" fromIndex="otherCollection" from="fromField" to="toField" v="*:*"}+----diff --git a/solr/solr-ref-guide/src/js/customscripts.js b/solr/solr-ref-guide/src/js/customscripts.jsindex 32f7c70afd6..45e89d1e94c 100755--- a/solr/solr-ref-guide/src/js/customscripts.js+++ b/solr/solr-ref-guide/src/js/customscripts.js@@ -4,11 +4,6 @@$( document ).ready(function() {- /**- * AnchorJS- */- anchors.add('h2,h3,h4,h5');-// "Bootstrap Bootstrap"// NOTE: by default, we use "dynamic-tabs" in our wrapper instead of "tab-content"// so that if javascript is disabled, bootstrap's CSS doesn't hide all non-active "tab-pane" divs@@ -67,3 +62,18 @@ $(document).ready(function () {});});++// Forces all non-solr.apache.org links to open in new tab+$(document).ready(function () {+ $("a[href^=http]").each(function(){+ var all_links = document.querySelectorAll('a');+ for (var i = 0; i < all_links.length; i++){+ var a = all_links[i];+ if(a.hostname != location.hostname) {+ a.rel = 'noopener';+ a.target = '_blank';+ }+}+ }+)+})diff --git a/solr/solr-ref-guide/src/json-facet-api.adoc b/solr/solr-ref-guide/src/json-facet-api.adocindex 519dd214d13..1154502c9fe 100644--- a/solr/solr-ref-guide/src/json-facet-api.adoc+++ b/solr/solr-ref-guide/src/json-facet-api.adoc@@ -617,7 +617,7 @@ include::{example-source-dir}JsonRequestApiTest.java[tag=solrj-json-metrics-face====---An expanded form allows for <<local-parameters-in-queries.adoc#,Local Parameters>> to be specified. These may be used explicitly by some specialized aggregations such as `<<json-facet-api.adoc#relatedness-options,relatedness()>>`, but can also be used as parameter references to make aggregation expressions more readable, with out needing to use (global) request parameters:+An expanded form allows for <<local-params.adoc#,local params>> to be specified. These may be used explicitly by some specialized aggregations such as `<<json-facet-api.adoc#relatedness-options,relatedness()>>`, but can also be used as parameter references to make aggregation expressions more readable, with out needing to use (global) request parameters:[.dynamic-tabs]--@@ -993,7 +993,7 @@ curl -sS -X POST http://localhost:8983/solr/gettingstarted/query -d 'rows=0&q=*:<1> Use the entire collection as our "Background Set"<2> Use a query for "age >= 35" to define our (initial) "Foreground Set"<3> For both the top level `hobbies` facet & the sub-facet on `state` we will be sorting on the `relatedness(...)` values-<4> In both calls to the `relatedness(...)` function, we use <<local-parameters-in-queries.adoc#parameter-dereferencing,Parameter Variables>> to refer to the previously defined `fore` and `back` queries.+<4> In both calls to the `relatedness(...)` function, we use <<local-params.adoc#parameter-dereferencing,parameter variables>> to refer to the previously defined `fore` and `back` queries..The Facet Response[source,json,subs="verbatim,callouts"]diff --git a/solr/solr-ref-guide/src/json-faceting-domain-changes.adoc b/solr/solr-ref-guide/src/json-faceting-domain-changes.adocindex 95b2fa3d8af..8c9fd2763d6 100644--- a/solr/solr-ref-guide/src/json-faceting-domain-changes.adoc+++ b/solr/solr-ref-guide/src/json-faceting-domain-changes.adoc@@ -216,7 +216,7 @@ NOTE: While a `query` domain can be combined with an additional domain `filter`,When a collection contains <<indexing-nested-documents.adoc#, Nested Documents>>, the `blockChildren` or `blockParent` domain options can be used to transform an existing domain containing one type of document, into a domain containing the documents with the specified relationship (child or parent of) to the documents from the original domain.-Both of these options work similarly to the corresponding <<other-parsers.adoc#block-join-query-parsers,Block Join Query Parsers>> by taking in a single String query that exclusively matches all parent documents in the collection. If `blockParent` is used, then the resulting domain will contain all parent documents of the children from the original domain. If `blockChildren` is used, then the resulting domain will contain all child documents of the parents from the original domain. Quite often facets over child documents needs to be counted in parent documents, this can be done by `uniqueBlock(\_root_)` as described in <<json-facet-api#uniqueblock-and-block-join-counts, Block Join Facet Counts>>.+Both of these options work similarly to the corresponding <<block-join-query-parser.adoc#,Block Join Query Parsers>> by taking in a single String query that exclusively matches all parent documents in the collection. If `blockParent` is used, then the resulting domain will contain all parent documents of the children from the original domain. If `blockChildren` is used, then the resulting domain will contain all child documents of the parents from the original domain. Quite often facets over child documents needs to be counted in parent documents, this can be done by `uniqueBlock(\_root_)` as described in <<json-facet-api#uniqueblock-and-block-join-counts, Block Join Facet Counts>>.[source,json,subs="verbatim,callouts"]]----@@ -241,7 +241,7 @@ Both of these options work similarly to the corresponding <<other-parsers.adoc#bA `join` domain change option can be used to specify arbitrary `from` and `to` fields to use in transforming from the existing domain to a related set of documents.-This works similarly to the <<other-parsers.adoc#join-query-parser,Join Query Parser>>, and has the same limitations when dealing with multi-shard collections.+This works similarly to the <<join-query-parser.adoc#,Join Query Parser>>, and has the same limitations when dealing with multi-shard collections.Example:[source,json]@@ -268,7 +268,7 @@ Example:-----`join` domain changes support an optional `method` parameter, which allows users to specify which join implementation they would like to use in this domain transform. Solr offers several join implementations, each with different performance characteristics. For more information on these implementations and their tradeoffs, see the `method` parameter documentation <<other-parsers.adoc#parameters,here>>. Join domain changes support all `method` values except `crossCollection`.+`join` domain changes support an optional `method` parameter, which allows users to specify which join implementation they would like to use in this domain transform. Solr offers several join implementations, each with different performance characteristics. For more information on these implementations and their tradeoffs, see the `method` parameter documentation <<join-query-parser.adoc#parameters,here>>. Join domain changes support all `method` values except `crossCollection`.== Graph Traversal Domain Changesdiff --git a/solr/solr-ref-guide/src/json-query-dsl.adoc b/solr/solr-ref-guide/src/json-query-dsl.adocindex 28eb02d0655..81b90d89415 100644--- a/solr/solr-ref-guide/src/json-query-dsl.adoc+++ b/solr/solr-ref-guide/src/json-query-dsl.adoc@@ -23,9 +23,9 @@ Queries and filters provided in JSON requests can be specified using a rich, pow== Query DSL StructureThe JSON Request API accepts query values in three different formats:-* A valid <<the-standard-query-parser.adoc#,query string>> that uses the default `deftype` (`lucene`, in most cases). e.g., `title:solr`.+* A valid <<standard-query-parser.adoc#,query string>> that uses the default `deftype` (`lucene`, in most cases). e.g., `title:solr`.-* A valid <<local-parameters-in-queries.adoc#,local parameters query string>> that specifies its `deftype` explicitly. e.g., `{!dismax qf=title}solr`.+* A valid <<local-params.adoc#,local params query string>> that specifies its `deftype` explicitly. e.g., `{!dismax qf=title}solr`.* A valid JSON object with the name of the query parser and any relevant parameters. e.g., `{ "lucene": {"df":"title", "query":"solr"}}`.** The top level "query" JSON block generally only has a single property representing the name of the query parser to use. The value for the query parser property is a child block containing any relevant parameters as JSON properties. The whole structure is analogous to a "local-params" query string. The query itself (often represented in local params using the name `v`) is specified with the key `query` instead.diff --git a/solr/solr-ref-guide/src/jvm-settings.adoc b/solr/solr-ref-guide/src/jvm-settings.adocindex 70c87fd1a63..f0a7e6602bc 100644--- a/solr/solr-ref-guide/src/jvm-settings.adoc+++ b/solr/solr-ref-guide/src/jvm-settings.adoc@@ -47,4 +47,18 @@ If you are using Sun's JVM, add the `-server` command-line option when you start== Checking JVM Settings-A great way to see what JVM settings your server is using, along with other useful information, is to use the admin RequestHandler, `/solr/admin/info/system`. This request handler will display a wealth of server statistics and settings.+=== System Request Handler++A great way to see what JVM settings your server is using, along with other useful information, is to use the `admin` request handler, `/solr/admin/info/system`.+This request handler will display a wealth of server statistics and settings.++=== Java Properties Screen++Many of the system environment variables include Java settings, and these can be seen on the main Dashboard of the Admin UI.++The Java Properties screen, however, provides easy access to all the properties of the JVM running Solr, including the classpaths, file encodings, JVM memory settings, operating system, and more.++In the Admin UI, it is available in the left-hand menu *Java Properties*.++.Java Properties Screen+image::images/jvm-settings/javaproperties.png[image,width=593,height=250]diff --git a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc b/solr/solr-ref-guide/src/jwt-authentication-plugin.adocindex 9ea6f30091d..8b6bc8eded8 100644--- a/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/jwt-authentication-plugin.adoc@@ -16,7 +16,10 @@// specific language governing permissions and limitations// under the License.-Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token] (JWT) based Bearer authentication with the use of the JWTAuthPlugin. This allows Solr to assert that a user is already authenticated with an external https://en.wikipedia.org/wiki/Identity_provider[Identity Provider] by validating that the JWT formatted https://en.wikipedia.org/wiki/Access_token[access token] is digitally signed by the Identity Provider. The typical use case is to integrate Solr with an https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect] enabled IdP.+Solr can support https://en.wikipedia.org/wiki/JSON_Web_Token[JSON Web Token] (JWT) based Bearer authentication with the use of the JWTAuthPlugin.++This allows Solr to assert that a user is already authenticated with an external https://en.wikipedia.org/wiki/Identity_provider[Identity Provider] by validating that the JWT formatted https://en.wikipedia.org/wiki/Access_token[access token] is digitally signed by the Identity Provider.+The typical use case is to integrate Solr with an https://en.wikipedia.org/wiki/OpenID_Connect[OpenID Connect] enabled IdP.== Enable JWT Authentication@@ -34,11 +37,13 @@ The simplest possible `security.json` for registering the plugin without configu}-----The plugin will by default require a valid JWT token for all traffic. If the `blockUnknown` property is set to false as in the above example, it is possible to start configuring the plugin using unauthenticated REST API calls, which is further described in section <<editing-jwt-authentication-plugin-configuration,Editing JWT Authentication Plugin Configuration>>.+The plugin will by default require a valid JWT token for all traffic.++If the `blockUnknown` property is set to `false` as in the above example, it is possible to start configuring the plugin using unauthenticated REST API calls, which is further described in section <<editing-jwt-authentication-plugin-configuration,Editing JWT Authentication Plugin Configuration>>.== Configuration Parameters-//TODO: standard is not to put parameters in tables but use labeled lists instead+//*TODO*: standard is not to put parameters in tables but use labeled lists instead[%header,format=csv,separator=;,cols="25%,50%,25%"]|===Key ; Description ; Default@@ -80,7 +85,11 @@ TIP: For backwards compatibility, all the configuration keys for the primary iss== More Configuration Examples=== With JWKS URL-To start enforcing authentication for all users, requiring a valid JWT in the `Authorization` header, you need to configure the plugin with one or more https://tools.ietf.org/html/rfc7517[JSON Web Key]s (JWK). This is a JSON document containing the key used to sign/encrypt the JWT. It could be a symmetric or asymmetric key. The JWK can either be fetched (and cached) from an external HTTPS endpoint or specified directly in `security.json`. Below is an example of the former:+To start enforcing authentication for all users, requiring a valid JWT in the `Authorization` header, you need to configure the plugin with one or more https://tools.ietf.org/html/rfc7517[JSON Web Key]s (JWK).+This is a JSON document containing the key used to sign/encrypt the JWT.+It could be a symmetric or asymmetric key.+The JWK can either be fetched (and cached) from an external HTTPS endpoint or specified directly in `security.json`.+Below is an example of the former:[source,json]----@@ -93,7 +102,7 @@ To start enforcing authentication for all users, requiring a valid JWT in the `A----=== With Admin UI Support-The next example shows configuring using https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] with a well-known URI for automatic configuration of many common settings, including ability to use the Admin UI with an OpenID Connect enabled Identity Provider.+This example shows configuration using https://openid.net/specs/openid-connect-discovery-1_0.html[OpenID Connect Discovery] with a well-known URI for automatic configuration of many common settings, including ability to use the Admin UI with an OpenID Connect enabled Identity Provider.[source,json]----@@ -107,7 +116,7 @@ The next example shows configuring using https://openid.net/specs/openid-connect}-----In this case, `jwksUrl`, `iss` and `authorizationEndpoint` will be automatically configured from the fetched configuration.+In this case, `jwksUrl`, `iss`, and `authorizationEndpoint` will be automatically configured from the fetched configuration.=== Complex ExampleLet's look at a more complex configuration, this time with two issuers configured, where one uses a static embedded JWK:@@ -158,8 +167,10 @@ Let's comment on this config:<8> Each issuer object should have a unique name<9> Here we pass the JWK inline instead of referring to a URL with `jwksUrl`<10> Set the client id registered with Identity Provider-<11> Configure the issuer id. Will be used for validating tokens. A token's 'iss' claim must match one of the configured issuer IDs.-<12> Configure the audience claim. A token's 'aud' claim must match 'aud' for one of the configured issuers.+<11> Configure the issuer id. Will be used for validating tokens.+A token's 'iss' claim must match one of the configured issuer IDs.+<12> Configure the audience claim.+A token's 'aud' claim must match 'aud' for one of the configured issuers.<13> This issuer is auto-configured through discovery, so 'iss' and JWK settings are not required<14> Provides SSL certificate(s) to trust IdP https communication.@@ -175,11 +186,13 @@ Please configure either the `trustedCerts` or `trustedCertsFile` option. Configu== Editing JWT Authentication Plugin Configuration-All properties mentioned above can be set or changed using the <<basic-authentication-plugin.adoc#editing-basic-authentication-plugin-configuration,Authentication API>>. You can thus start with a simple configuration with only `class` and `blockUnknown=false` configured and then configure the rest using the API.+All properties mentioned above can be set or changed using the <<basic-authentication-plugin.adoc#editing-basic-authentication-plugin-configuration,Authentication API>>.+You can thus start with a simple configuration with only `class` and `blockUnknown=false` configured and then configure the rest using the API.=== Set a Configuration Property-Set properties for the authentication plugin. Each of the configuration keys in the table above can be used as parameter keys for the `set-property` command.+Set properties for the authentication plugin.+Each of the configuration keys in the table above can be used as parameter keys for the `set-property` command.Example:@@ -229,7 +242,14 @@ curl -H "Authorization: Bearer xxxxxx.xxxxxx.xxxxxx" http://localhost:8983/solr/=== Admin UI-When this plugin is enabled, users will be redirected to a login page in the Admin UI once they attempt to do a restricted action. The page has a button that users will click and be redirected to the Identity Provider's login page. If more than one issuer (IdP) is configured, the first in the list will be used for Admin UI. Once authenticated, the user will be redirected back to Solr Admin UI to the last known location. The session will last as long as the JWT token expiry time and is valid for one Solr server only. That means you have to login again when navigating to another Solr node. There is also a logout menu in the left column where user can explicitly log out.+When this plugin is enabled, users will be redirected to a login page in the Admin UI once they attempt to do a restricted action.+The page has a button that users will click and be redirected to the Identity Provider's login page.++If more than one issuer (IdP) is configured, the first in the list will be used for Admin UI.+Once authenticated, the user will be redirected back to Solr Admin UI to the last known location.+The session will last as long as the JWT token expiry time and is valid for one Solr server only.+That means you have to login again when navigating to another Solr node.+There is also a logout menu in the left column where user can explicitly log out.== Using the Solr Control Script with JWT Authdiff --git a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adocindex bbfde5ed5f1..a955368c52c 100644--- a/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc+++ b/solr/solr-ref-guide/src/kerberos-authentication-plugin.adoc@@ -18,50 +18,63 @@If you are using Kerberos to secure your network environment, the Kerberos authentication plugin can be used to secure a Solr cluster.-This allows Solr to use a Kerberos service principal and keytab file to authenticate with ZooKeeper and between nodes of the Solr cluster (if applicable). Users of the Admin UI and all clients (such as <<using-solrj.adoc#,SolrJ>>) would also need to have a valid ticket before being able to use the UI or send requests to Solr.+This allows Solr to use a Kerberos service principal and keytab file to authenticate with ZooKeeper and between nodes of the Solr cluster (if applicable).+Users of the Admin UI and all clients (such as <<solrj.adoc#,SolrJ>>) would also need to have a valid ticket before being able to use the UI or send requests to Solr.-Support for the Kerberos authentication plugin is available in SolrCloud mode or standalone mode.+Support for Kerberos authentication is available in SolrCloud, user-managed, or single-node installations.[TIP]====-If you are using Solr with a Hadoop cluster secured with Kerberos and intend to store your Solr indexes in HDFS, also see the section <<running-solr-on-hdfs.adoc#,Running Solr on HDFS>> for additional steps to configure Solr for that purpose. The instructions on this page apply only to scenarios where Solr will be secured with Kerberos. If you only need to store your indexes in a Kerberized HDFS system, please see the other section referenced above.+If you are using Solr with a Hadoop cluster secured with Kerberos and intend to store your Solr indexes in HDFS, also see the section <<solr-on-hdfs.adoc#,Solr on HDFS>> for additional steps to configure Solr for that purpose.+The instructions on this page apply only to scenarios where Solr will be secured with Kerberos.+If you only need to store your indexes in a Kerberized HDFS system, please see the Running Solr on HDFS section.====== How Solr Works With Kerberos-When setting up Solr to use Kerberos, configurations are put in place for Solr to use a _service principal_, or a Kerberos username, which is registered with the Key Distribution Center (KDC) to authenticate requests. The configurations define the service principal name and the location of the keytab file that contains the credentials.+When setting up Solr to use Kerberos, configurations are put in place for Solr to use a _service principal_, or a Kerberos username, which is registered with the Key Distribution Center (KDC) to authenticate requests.+The configurations define the service principal name and the location of the keytab file that contains the credentials.-=== security.json--The Solr authentication model uses a file called `security.json`. A description of this file and how it is created and maintained is covered in the section <<authentication-and-authorization-plugins.adoc#,Authentication and Authorization Plugins>>. If this file is created after an initial startup of Solr, a restart of each node of the system is required.+As with all authentication plugins, Kerberos authentication configuration is stored in `security.json`.+This file is discussed in the section <<authentication-and-authorization-plugins.adoc#configuring-security-json,Configuring security.json>>.=== Service Principals and Keytab Files-Each Solr node must have a service principal registered with the Key Distribution Center (KDC). The Kerberos plugin uses SPNego to negotiate authentication.+Each Solr node must have a service principal registered with the Key Distribution Center (KDC).+The Kerberos plugin uses SPNego to negotiate authentication.-Using `HTTP/[email protected]`, as an example of a service principal:+Using `HTTP/[email protected]` as an example of a service principal:-* `HTTP` indicates the type of requests which this service principal will be used to authenticate. The `HTTP/` in the service principal is a must for SPNego to work with requests to Solr over HTTP.+* `HTTP` indicates the type of requests which this service principal will be used to authenticate.+The `HTTP/` in the service principal is a must for SPNego to work with requests to Solr over HTTP.* `host1` is the host name of the machine hosting the Solr node.-* `YOUR-DOMAIN.ORG` is the organization wide Kerberos realm.+* `YOUR-DOMAIN.ORG` is the organization-wide Kerberos realm.Multiple Solr nodes on the same host may have the same service principal, since the host name is common to them all.-Along with the service principal, each Solr node needs a keytab file which should contain the credentials of the service principal used. A keytab file contains encrypted credentials to support passwordless logins while obtaining Kerberos tickets from the KDC. For each Solr node, the keytab file should be kept in a secure location and not shared with users of the cluster.+Along with the service principal, each Solr node needs a keytab file which should contain the credentials of the service principal used.+A keytab file contains encrypted credentials to support passwordless logins while obtaining Kerberos tickets from the KDC.+For each Solr node, the keytab file should be kept in a secure location and not shared with users of the cluster.-Since a Solr cluster requires internode communication, each node must also be able to make Kerberos enabled requests to other nodes. By default, Solr uses the same service principal and keytab as a 'client principal' for internode communication. You may configure a distinct client principal explicitly, but doing so is not recommended and is not covered in the examples below.+Since a Solr cluster requires internode communication, each node must also be able to make Kerberos enabled requests to other nodes.+By default, Solr uses the same service principal and keytab as a 'client principal' for internode communication.+You may configure a distinct client principal explicitly, but doing so is not recommended and is not covered in the examples below.=== Kerberized ZooKeeperWhen setting up a kerberized SolrCloud cluster, it is recommended to enable Kerberos security for ZooKeeper as well.-In such a setup, the client principal used to authenticate requests with ZooKeeper can be shared for internode communication as well. This has the benefit of not needing to renew the ticket granting tickets (TGTs) separately, since the ZooKeeper client used by Solr takes care of this. To achieve this, a single JAAS configuration (with the app name as Client) can be used for the Kerberos plugin as well as for the ZooKeeper client.+In such a setup, the client principal used to authenticate requests with ZooKeeper can be shared for internode communication as well.+This has the benefit of not needing to renew the ticket granting tickets (TGTs) separately, since the ZooKeeper client used by Solr takes care of this.+To achieve this, a single JAAS configuration (with the app name as Client) can be used for the Kerberos plugin as well as for the ZooKeeper client.See the <<ZooKeeper Configuration>> section below for an example of starting ZooKeeper in Kerberos mode.=== Browser Configuration-In order for your browser to access the Solr Admin UI after enabling Kerberos authentication, it must be able to negotiate with the Kerberos authenticator service to allow you access. Each browser supports this differently, and some (like Chrome) do not support it at all. If you see 401 errors when trying to access the Solr Admin UI after enabling Kerberos authentication, it's likely your browser has not been configured properly to know how or where to negotiate the authentication request.+In order for your browser to access the Solr Admin UI after enabling Kerberos authentication, it must be able to negotiate with the Kerberos authenticator service to allow you access.+Each browser supports this differently, and some (like Chrome) do not support it at all.+If you see 401 errors when trying to access the Solr Admin UI after enabling Kerberos authentication, it's likely your browser has not been configured properly to know how or where to negotiate the authentication request.Detailed information on how to set up your browser is beyond the scope of this documentation; please see your system administrators for Kerberos for details on how to configure your browser.@@ -70,7 +83,8 @@ Detailed information on how to set up your browser is beyond the scope of this d.Consult Your Kerberos Admins![WARNING]====-Before attempting to configure Solr to use Kerberos authentication, please review each step outlined below and consult with your local Kerberos administrators on each detail to be sure you know the correct values for each parameter. Small errors can cause Solr to not start or not function properly, and are notoriously difficult to diagnose.+Before attempting to configure Solr to use Kerberos authentication, please review each step outlined below and consult with your local Kerberos administrators on each detail to be sure you know the correct values for each parameter.+Small errors can cause Solr to not start or not function properly, and are notoriously difficult to diagnose.====Configuration of the Kerberos plugin has several parts:@@ -86,16 +100,18 @@ We'll walk through each of these steps below..Using Hostnames[IMPORTANT]====-To use host names instead of IP addresses, use the `SOLR_HOST` configuration in `bin/solr.in.sh` or pass a `-Dhost=<hostname>` system parameter during Solr startup. This guide uses IP addresses. If you specify a hostname, replace all the IP addresses in the guide with the Solr hostname as appropriate.+To use host names instead of IP addresses, use the `SOLR_HOST` configuration in `bin/solr.in.sh` or pass a `-Dhost=<hostname>` system parameter during Solr startup.+This guide uses IP addresses. If you specify a hostname, replace all the IP addresses in the guide with the Solr hostname as appropriate.======= Get Service Principals and KeytabsBefore configuring Solr, make sure you have a Kerberos service principal for each Solr host and ZooKeeper (if ZooKeeper has not already been configured) available in the KDC server, and generate a keytab file as shown below.-This example assumes the hostname is `192.168.0.107` and your home directory is `/home/foo/`. This example should be modified for your own environment.+This example assumes the hostname is `192.168.0.107` and your home directory is `/home/foo/`.+This example should be modified for your own environment.-[source,plain]+[source,bash]----root@kdc:/# kadmin.localAuthenticating as principal foo/[email protected] with password.@@ -115,26 +131,31 @@ Entry for principal HTTP/192.168.0.107 with kvno 2, encryption type des-cbc-crckadmin.local: quit-----Copy the keytab file from the KDC server’s `/tmp/107.keytab` location to the Solr host at `/keytabs/107.keytab`. Repeat this step for each Solr node.+Copy the keytab file from the KDC server’s `/tmp/107.keytab` location to the Solr host at `/keytabs/107.keytab`.+Repeat this step for each Solr node.-You might need to take similar steps to create a ZooKeeper service principal and keytab if it has not already been set up. In that case, the example below shows a different service principal for ZooKeeper, so the above might be repeated with `zookeeper/host1` as the service principal for one of the nodes+You might need to take similar steps to create a ZooKeeper service principal and keytab if it has not already been set up.+In that case, the example below shows a different service principal for ZooKeeper, so the above might be repeated with `zookeeper/host1` as the service principal for one of the nodes=== ZooKeeper ConfigurationIf you are using a ZooKeeper that has already been configured to use Kerberos, you can skip the ZooKeeper-related steps shown here.-Since ZooKeeper manages the communication between nodes in a SolrCloud cluster, it must also be able to authenticate with each node of the cluster. Configuration requires setting up a service principal for ZooKeeper, defining a JAAS configuration file and instructing ZooKeeper to use both of those items.+Since ZooKeeper manages the communication between nodes in a SolrCloud cluster, it must also be able to authenticate with each node of the cluster.+Configuration requires setting up a service principal for ZooKeeper, defining a JAAS configuration file and instructing ZooKeeper to use both of those items.The first step is to create a file `java.env` in ZooKeeper's `conf` directory and add the following to it, as in this example:-[source,bash]+[source,conf]----export JVMFLAGS="-Djava.security.auth.login.config=/etc/zookeeper/conf/jaas-client.conf"-----The JAAS configuration file should contain the following parameters. Be sure to change the `principal` and `keyTab` path as appropriate. The file must be located in the path defined in the step above, with the filename specified.+The JAAS configuration file should contain the following parameters.+Be sure to change the `principal` and `keyTab` path as appropriate.+The file must be located in the path defined in the step above, with the filename specified.-[source,plain]+[source,conf]----Server {com.sun.security.auth.module.Krb5LoginModule required@@ -150,7 +171,7 @@ Server {Finally, add the following lines to the ZooKeeper configuration file `zoo.cfg`:-[source,bash]+[source,properties]----authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProviderjaasLoginRenew=3600000@@ -160,7 +181,7 @@ Once all of the pieces are in place, start ZooKeeper with the following paramete[source,bash]-----bin/zkServer.sh start -Djava.security.auth.login.config=/etc/zookeeper/conf/jaas-client.conf+$ bin/zkServer.sh start -Djava.security.auth.login.config=/etc/zookeeper/conf/jaas-client.conf----=== Create security.json@@ -176,27 +197,35 @@ Then use the `bin/solr zk` command to upload the file:[source,bash]----->bin/solr zk cp ./security.json zk:security.json -z localhost:2181+$ bin/solr zk cp ./security.json zk:security.json -z localhost:2181-----If you are using Solr in standalone mode, you need to create the `security.json` file and put it in your `$SOLR_HOME` directory.--More details on how to use a `/security.json` file in Solr are available in the section <<authentication-and-authorization-plugins.adoc#,Authentication and Authorization Plugins>>.+If you are using Solr in a single-node installation, you need to create the `security.json` file and put it in your `$SOLR_HOME` directory.[IMPORTANT]====-If you already have a `/security.json` file in ZooKeeper, download the file, add or modify the authentication section and upload it back to ZooKeeper using the <<command-line-utilities.adoc#,Command Line Utilities>> available in Solr.+If you already have a `/security.json` file in ZooKeeper, download the file, add or modify the authentication section and upload it back to ZooKeeper using the <<zookeeper-utilities.adoc#,ZooKeeper Utilities>> available in Solr.======= Define a JAAS Configuration File-The JAAS configuration file defines the properties to use for authentication, such as the service principal and the location of the keytab file. Other properties can also be set to ensure ticket caching and other features.+The JAAS configuration file defines the properties to use for authentication, such as the service principal and the location of the keytab file.+Other properties can also be set to ensure ticket caching and other features.-The following example can be copied and modified slightly for your environment. The location of the file can be anywhere on the server, but it will be referenced when starting Solr so it must be readable on the filesystem. The JAAS file may contain multiple sections for different users, but each section must have a unique name so it can be uniquely referenced in each application.+The following example can be copied and modified slightly for your environment.+The location of the file can be anywhere on the server, but it will be referenced when starting Solr so it must be readable on the filesystem.+The JAAS file may contain multiple sections for different users, but each section must have a unique name so it can be uniquely referenced in each application.-In the below example, we have created a JAAS configuration file with the name and path of `/home/foo/jaas-client.conf`. We will use this name and path when we define the Solr start parameters in the next section. Note that the client `principal` here is the same as the service principal. This will be used to authenticate internode requests and requests to ZooKeeper. Make sure to use the correct `principal` hostname and the `keyTab` file path.+In the below example, we have created a JAAS configuration file with the name and path of `/home/foo/jaas-client.conf`.+We will use this name and path when we define the Solr start parameters in the next section.+Note that the client `principal` here is the same as the service principal.+This will be used to authenticate internode requests and requests to ZooKeeper.+Make sure to use the correct `principal` hostname and the `keyTab` file path.-[source,plain]+// *TODO* The docs link below goes to Java 8, but Java 11 has changed the classpath:+// https://docs.oracle.com/en/java/javase/11/docs/api/jdk.security.auth/com/sun/security/auth/module/Krb5LoginModule.html++[source,conf]----Client {com.sun.security.auth.module.Krb5LoginModule required@@ -211,47 +240,105 @@ Client {The first line of this file defines the section name, which will be used with the `solr.kerberos.jaas.appname` parameter, defined below.-The main properties we are concerned with are the `keyTab` and `principal` properties, but there are others which may be required for your environment. The https://docs.oracle.com/javase/8/docs/jre/api/security/jaas/spec/com/sun/security/auth/module/Krb5LoginModule.html[javadocs for the Krb5LoginModule] (the class that's being used and is called in the second line above) provide a good outline of the available properties, but for reference the ones in use in the above example are explained here:+The main properties we are concerned with are the `keyTab` and `principal` properties, but there are others which may be required for your environment.+The https://docs.oracle.com/javase/8/docs/jre/api/security/jaas/spec/com/sun/security/auth/module/Krb5LoginModule.html[javadocs for the Krb5LoginModule] (the class that's being used and is called in the second line above) provide a good outline of the available properties, but for reference the ones in use in the above example are explained here:++* `useKeyTab`: this boolean property defines if we should use a keytab file (`true`, in this case).++* `keyTab`: the location and name of the keytab file for the principal this section of the JAAS configuration file is for.+The path should be enclosed in double-quotes.-* `useKeyTab`: this boolean property defines if we should use a keytab file (true, in this case).-* `keyTab`: the location and name of the keytab file for the principal this section of the JAAS configuration file is for. The path should be enclosed in double-quotes.* `storeKey`: this boolean property allows the key to be stored in the private credentials of the user.+* `useTicketCache`: this boolean property allows the ticket to be obtained from the ticket cache.+* `debug`: this boolean property will output debug messages for help in troubleshooting.+* `principal`: the name of the service principal to be used.=== Solr Startup Parameters-While starting up Solr, the following host-specific parameters need to be passed. These parameters can be passed at the command line with the `bin/solr` start command (see <<solr-control-script-reference.adoc#,Solr Control Script Reference>> for details on how to pass system parameters) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system.+While starting up Solr, the following host-specific parameters need to be passed.+These parameters can be passed at the command line with the `bin/solr` start command (see <<solr-control-script-reference.adoc#,Solr Control Script Reference>> for details on how to pass system parameters) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system.`solr.kerberos.name.rules`::-Used to map Kerberos principals to short names. Default value is `DEFAULT`. Example of a name rule: `RULE:[1:$1@$0](.\*EXAMPLE.COM)s/@.*//`.+++[%autowidth,frame=none]+|===+|Optional |Default: `DEFAULT`+|===+++Used to map Kerberos principals to short names.+Example of a name rule: `RULE:[1:$1@$0](.\*EXAMPLE.COM)s/@.*//`.-`solr.kerberos.cookie.domain`:: Used to issue cookies and should have the hostname of the Solr node. This parameter is required.+`solr.kerberos.cookie.domain`::+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++Used to issue cookies and should have the hostname of the Solr node.`solr.kerberos.cookie.portaware`::-When set to `true`, cookies are differentiated based on host and port, as opposed to standard cookies which are not port aware. This should be set if more than one Solr node is hosted on the same host. The default is `false`.+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++When set to `true`, cookies are differentiated based on host and port, as opposed to standard cookies which are not port aware.+This should be set if more than one Solr node is hosted on the same host.`solr.kerberos.principal`::-The service principal. This parameter is required.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The service principal.`solr.kerberos.keytab`::-Keytab file path containing service principal credentials. This parameter is required.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++Keytab file path containing service principal credentials`solr.kerberos.jaas.appname`::-The app name (section name) within the JAAS configuration file which is required for internode communication. Default is `Client`, which is used for ZooKeeper authentication as well. If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file.+++[%autowidth,frame=none]+|===+|Optional |Default: `Client`+|===+++The app name (section name) within the JAAS configuration file which is required for internode communication.+The default is used for ZooKeeper authentication as well.+If different users are used for ZooKeeper and Solr, they will need to have separate sections in the JAAS configuration file.`java.security.auth.login.config`::-Path to the JAAS configuration file for configuring a Solr client for internode communication. This parameter is required.+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++Path to the JAAS configuration file for configuring a Solr client for internode communication.-Here is an example that could be added to `bin/solr.in.sh`. Make sure to change this example to use the right hostname and the keytab file path.+Here is an example that could be added to `bin/solr.in.sh`.+Make sure to change this example to use the right hostname and the keytab file path.-[source,bash]+[source,properties]----SOLR_AUTH_TYPE="kerberos"SOLR_AUTHENTICATION_OPTS="-Djava.security.auth.login.config=/home/foo/jaas-client.conf -Dsolr.kerberos.cookie.domain=192.168.0.107 -Dsolr.kerberos.cookie.portaware=true -Dsolr.kerberos.principal=HTTP/[email protected] -Dsolr.kerberos.keytab=/keytabs/107.keytab"----+// *TODO* Update this for Java 11+.KDC with AES-256 encryption[IMPORTANT]====@@ -271,39 +358,90 @@ The Kerberos plugin can be configured to use delegation tokens, which allow an aThere are a few use cases for Solr where this might be helpful:* Using distributed clients (such as MapReduce) where each client may not have access to the user's credentials.-* When load on the Kerberos server is high. Delegation tokens can reduce the load because they do not access the server after the first request.+* When load on the Kerberos server is high.+Delegation tokens can reduce the load because they do not access the server after the first request.* If requests or permissions need to be delegated to another user.-To enable delegation tokens, several parameters must be defined. These parameters can be passed at the command line with the `bin/solr` start command (see <<solr-control-script-reference.adoc#,Solr Control Script Reference>> for details on how to pass system parameters) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system.+To enable delegation tokens, several parameters must be defined.+These parameters can be passed at the command line with the `bin/solr` start command (<<solr-control-script-reference.adoc#,Solr Control Script Reference>>) or defined in `bin/solr.in.sh` or `bin/solr.in.cmd` as appropriate for your operating system.`solr.kerberos.delegation.token.enabled`::-This is `false` by default, set to `true` to enable delegation tokens. This parameter is required if you want to enable tokens.+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++Set to `true` to enable delegation tokens.+This parameter is required if you want to enable tokens.`solr.kerberos.delegation.token.kind`::-The type of delegation tokens. By default this is `solr-dt`. Likely this does not need to change. No other option is available at this time.+++[%autowidth,frame=none]+|===+|Optional |Default: `solr-dt`+|===+++The type of delegation tokens.+The only option available at this time is the default.`solr.kerberos.delegation.token.validity`::-Time, in seconds, for which delegation tokens are valid. The default is 36000 seconds.+++[%autowidth,frame=none]+|===+|Optional |Default: `36000`+|===+++Time, in seconds, for which delegation tokens are valid.`solr.kerberos.delegation.token.signer.secret.provider`::-Where delegation token information is stored internally. The default is `zookeeper` which must be the location for delegation tokens to work across Solr servers (when running in SolrCloud mode). No other option is available at this time.+++[%autowidth,frame=none]+|===+|Optional |Default: `zookeeper`+|===+++Where delegation token information is stored internally.+The default is `zookeeper` which must be the location for delegation tokens to work across Solr servers (when running in SolrCloud mode).+No other option is available at this time.`solr.kerberos.delegation.token.signer.secret.provider.zookeper.path`::-The ZooKeeper path where the secret provider information is stored. This is in the form of the path + /security/token. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/token`.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The ZooKeeper path where the secret provider information is stored.+This is in the form of the path + `/security/token`.+The path can include the chroot or the chroot can be omitted if you are not using it.+This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/token`.`solr.kerberos.delegation.token.secret.manager.znode.working.path`::-The ZooKeeper path where token information is stored. This is in the form of the path + /security/zkdtsm. The path can include the chroot or the chroot can be omitted if you are not using it. This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/zkdtsm`.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The ZooKeeper path where token information is stored.+This is in the form of the path + `/security/zkdtsm`.+The path can include the chroot or the chroot can be omitted if you are not using it.+This example includes the chroot: `server1:9983,server2:9983,server3:9983/solr/security/zkdtsm`.=== Start Solr-Once the configuration is complete, you can start Solr with the `bin/solr` script, as in the example below, which is for users in SolrCloud mode only. This example assumes you modified `bin/solr.in.sh` or `bin/solr.in.cmd`, with the proper values, but if you did not, you would pass the system parameters along with the start command. Note you also need to customize the `-z` property as appropriate for the location of your ZooKeeper nodes.+Once the configuration is complete, you can start Solr with the `bin/solr` script, as in the example below, which is for users in SolrCloud mode only.++This example assumes you modified `bin/solr.in.sh` or `bin/solr.in.cmd`, with the proper values, but if you did not, you would pass the system parameters along with the start command.+Note you also need to customize the `-z` property as appropriate for the location of your ZooKeeper nodes.[source,bash]-----bin/solr -c -z server1:2181,server2:2181,server3:2181/solr+$ bin/solr -c -z server1:2181,server2:2181,server3:2181/solr-----NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.+NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.=== Test the Configuration@@ -312,7 +450,7 @@ NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting+[source,bash]-----curl --negotiate -u : "http://192.168.0.107:8983/solr/"+$ curl --negotiate -u : "http://192.168.0.107:8983/solr/"----== Using SolrJ with a Kerberized Solr@@ -325,11 +463,12 @@ System.setProperty("java.security.auth.login.config", "/home/foo/jaas-client.conHttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());-----You need to specify a Kerberos service principal for the client and a corresponding keytab in the JAAS client configuration file above. This principal should be different from the service principal we created for Solr.+You need to specify a Kerberos service principal for the client and a corresponding keytab in the JAAS client configuration file above.+This principal should be different from the service principal we created for Solr.Here’s an example:-[source,plain]+[source,conf]----SolrJClient {com.sun.security.auth.module.Krb5LoginModule required@@ -390,5 +529,7 @@ CloudSolrClient client = new CloudSolrClient.Builder(Collections.singletonList("[TIP]====-Hadoop's delegation token responses are in JSON map format. A response parser for that is available in `DelegationTokenResponse`. Other response parsers may not work well with Hadoop responses.+Hadoop's delegation token responses are in JSON map format.+A response parser for that is available in `DelegationTokenResponse`.+Other response parsers may not work well with Hadoop responses.====diff --git a/solr/solr-ref-guide/src/language-analysis.adoc b/solr/solr-ref-guide/src/language-analysis.adocindex 3f3fbef9701..ab8f7c06c9d 100644--- a/solr/solr-ref-guide/src/language-analysis.adoc+++ b/solr/solr-ref-guide/src/language-analysis.adoc@@ -19,15 +19,19 @@This section contains information about tokenizers and filters related to character set conversion or for use with specific languages.-For the European languages, tokenization is fairly straightforward. Tokens are delimited by white space and/or a relatively small set of punctuation characters.+For the European languages, tokenization is fairly straightforward.+Tokens are delimited by white space and/or a relatively small set of punctuation characters.-In other languages the tokenization rules are often not so simple. Some European languages may also require special tokenization rules, such as rules for decompounding German words.+In other languages the tokenization rules are often not so simple.+Some European languages may also require special tokenization rules, such as rules for decompounding German words.-For information about language detection at index time, see <<detecting-languages-during-indexing.adoc#,Detecting Languages During Indexing>>.+For information about language detection at index time, see <<language-detection.adoc#,Language Detection>>.== KeywordMarkerFilterFactory-Protects words from being modified by stemmers. A customized protected word list may be specified with the "protected" attribute in the schema. Any words in the protected word list will not be modified by any stemmer in Solr.+Protects words from being modified by stemmers.+A customized protected word list may be specified with the "protected" attribute in the schema.+Any words in the protected word list will not be modified by any stemmer in Solr.A sample Solr `protwords.txt` with comments can be found in the `sample_techproducts_configs` <<config-sets.adoc#,configset>> directory:@@ -67,9 +71,12 @@ A sample Solr `protwords.txt` with comments can be found in the `sample_techprodEmits each token twice, one with the `KEYWORD` attribute and once without.-If placed before a stemmer, the result will be that you will get the unstemmed token preserved on the same position as the stemmed one. Queries matching the original exact term will get a better score while still maintaining the recall benefit of stemming. Another advantage of keeping the original token is that wildcard truncation will work as expected.+If placed before a stemmer, the result will be that you will get the unstemmed token preserved on the same position as the stemmed one.+Queries matching the original exact term will get a better score while still maintaining the recall benefit of stemming.+Another advantage of keeping the original token is that wildcard truncation will work as expected.-To configure, add the `KeywordRepeatFilterFactory` early in the analysis chain. It is recommended to also include `RemoveDuplicatesTokenFilterFactory` to avoid duplicates when tokens are not stemmed.+To configure, add the `KeywordRepeatFilterFactory` early in the analysis chain.+It is recommended to also include `RemoveDuplicatesTokenFilterFactory` to avoid duplicates when tokens are not stemmed.A sample fieldType configuration could look like this:@@ -113,7 +120,8 @@ IMPORTANT: When adding the same token twice, it will also score twice (double),Overrides stemming algorithms by applying a custom mapping, then protecting these terms from being modified by stemmers.-A customized mapping of words to stems, in a tab-separated file, can be specified to the `dictionary` attribute in the schema. Words in this mapping will be stemmed to the stems from the file, and will not be further changed by any stemmer.+A customized mapping of words to stems, in a tab-separated file, can be specified to the `dictionary` attribute in the schema.+Words in this mapping will be stemmed to the stems from the file, and will not be further changed by any stemmer.[.dynamic-tabs]--@@ -158,7 +166,9 @@ If you have a checkout of Solr's source code locally, you can also find this exa== Dictionary Compound Word Token Filter-This filter splits, or _decompounds_, compound words into individual words using a dictionary of the component words. Each input token is passed through unchanged. If it can also be decompounded into subwords, each subword is also added to the stream at the same logical position.+This filter splits, or _decompounds_, compound words into individual words using a dictionary of the component words.+Each input token is passed through unchanged.+If it can also be decompounded into subwords, each subword is also added to the stream at the same logical position.Compound words are most commonly found in Germanic languages.@@ -166,7 +176,10 @@ Compound words are most commonly found in Germanic languages.*Arguments:*-`dictionary`:: (required) The path of a file that contains a list of simple words, one per line. Blank lines and lines that begin with "`#`" are ignored. See <<resource-loading.adoc#,Resource Loading>> for more information.+`dictionary`:: (required) The path of a file that contains a list of simple words, one per line.+Blank lines and lines that begin with "`#`" are ignored.+++See <<resource-loading.adoc#,Resource Loading>> for more information.`minWordSize`:: (integer, default 5) Any token shorter than this is not decompounded.@@ -218,7 +231,8 @@ Unicode Collation is a language-sensitive method of sorting text that can also bUnicode Collation in Solr is fast, because all the work is done at index time.-Rather than specifying an analyzer within `<fieldtype ... class="solr.TextField">`, the `solr.CollationField` and `solr.ICUCollationField` field type classes provide this functionality. `solr.ICUCollationField`, which is backed by http://site.icu-project.org[the ICU4J library], provides more flexible configuration, has more locales, is significantly faster, and requires less memory and less index space, since its keys are smaller than those produced by the JDK implementation that backs `solr.CollationField`.+Rather than specifying an analyzer within `<fieldtype ... class="solr.TextField">`, the `solr.CollationField` and `solr.ICUCollationField` field type classes provide this functionality.+`solr.ICUCollationField`, which is backed by http://site.icu-project.org[the ICU4J library], provides more flexible configuration, has more locales, is significantly faster, and requires less memory and less index space, since its keys are smaller than those produced by the JDK implementation that backs `solr.CollationField`.To use `solr.ICUCollationField`, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.@@ -231,7 +245,8 @@ To use `solr.ICUCollationField`, you must add additional .jars to Solr's classpaUsing a System collator:-`locale`:: (required) http://www.rfc-editor.org/rfc/rfc3066.txt[RFC 3066] locale ID. See http://demo.icu-project.org/icu-bin/locexp[the ICU locale explorer] for a list of supported locales.+`locale`:: (required) http://www.rfc-editor.org/rfc/rfc3066.txt[RFC 3066] locale ID.+See http://demo.icu-project.org/icu-bin/locexp[the ICU locale explorer] for a list of supported locales.`strength`:: Valid values are `primary`, `secondary`, `tertiary`, `quaternary`, or `identical`. See http://userguide.icu-project.org/collation/concepts#TOC-Comparison-Levels[Comparison Levels in ICU Collation Concepts] for more information.@@ -249,19 +264,23 @@ Expert options:`alternate`:: Valid values are `shifted` or `non-ignorable`. Can be used to ignore punctuation/whitespace.-`caseLevel`:: (true/false) If true, in combination with `strength="primary"`, accents are ignored but case is taken into account. The default is false. See http://userguide.icu-project.org/collation/concepts#TOC-CaseLevel[CaseLevel in ICU Collation Concepts] for more information.+`caseLevel`:: (true/false) If true, in combination with `strength="primary"`, accents are ignored but case is taken into account.+The default is `false`. See http://userguide.icu-project.org/collation/concepts#TOC-CaseLevel[CaseLevel in ICU Collation Concepts] for more information.`caseFirst`:: Valid values are `lower` or `upper`. Useful to control which is sorted first when case is not ignored.`numeric`:: (true/false) If true, digits are sorted according to numeric value, e.g., foobar-9 sorts before foobar-10. The default is false.-`variableTop`:: Single character or contraction. Controls what is variable for `alternate`.+`variableTop`:: Single character or contraction.+Controls what is variable for `alternate`.=== Sorting Text for a Specific LanguageIn this example, text is sorted according to the default German rules provided by ICU4J.-Locales are typically defined as a combination of language and country, but you can specify just the language if you want. For example, if you specify "de" as the language, you will get sorting that works well for the German language. If you specify "de" as the language and "CH" as the country, you will get German sorting specifically tailored for Switzerland.+Locales are typically defined as a combination of language and country, but you can specify just the language if you want.+For example, if you specify "de" as the language, you will get sorting that works well for the German language.+If you specify "de" as the language and "CH" as the country, you will get German sorting specifically tailored for Switzerland.[source,xml]----@@ -278,7 +297,8 @@ Locales are typically defined as a combination of language and country, but you<copyField source="manu" dest="manuGERMAN"/>-----In the example above, we defined the strength as "primary". The strength of the collation determines how strict the sort order will be, but it also depends upon the language. For example, in English, "primary" strength ignores differences in case and accents.+In the example above, we defined the strength as "primary". The strength of the collation determines how strict the sort order will be, but it also depends upon the language.+For example, in English, "primary" strength ignores differences in case and accents.Another example:@@ -295,7 +315,8 @@ Another example:<copyField source="city" dest="city_sort"/>-----The type will be used for the fields where the data contains Polish text. The "secondary" strength will ignore case differences, but, unlike "primary" strength, a letter with diacritic(s) will be sorted differently from the same base letter without diacritics.+The type will be used for the fields where the data contains Polish text.+The "secondary" strength will ignore case differences, but, unlike "primary" strength, a letter with diacritic(s) will be sorted differently from the same base letter without diacritics.An example using the "city_sort" field to sort:@@ -306,9 +327,12 @@ q=*:*&fl=city&sort=city_sort+asc=== Sorting Text for Multiple Languages-There are two approaches to supporting multiple languages: if there is a small list of languages you wish to support, consider defining collated fields for each language and using `copyField`. However, adding a large number of sort fields can increase disk and indexing costs. An alternative approach is to use the Unicode `default` collator.+There are two approaches to supporting multiple languages: if there is a small list of languages you wish to support, consider defining collated fields for each language and using `copyField`. However, adding a large number of sort fields can increase disk and indexing costs.+An alternative approach is to use the Unicode `default` collator.-The Unicode `default` or `ROOT` locale has rules that are designed to work well for most languages. To use the `default` locale, simply define the locale as the empty string. This Unicode default sort is still significantly more advanced than the standard Solr sort.+The Unicode `default` or `ROOT` locale has rules that are designed to work well for most languages.+To use the `default` locale, simply define the locale as the empty string.+This Unicode default sort is still significantly more advanced than the standard Solr sort.[source,xml]----@@ -319,9 +343,11 @@ The Unicode `default` or `ROOT` locale has rules that are designed to work well=== Sorting Text with Custom Rules-You can define your own set of sorting rules. It's easiest to take existing rules that are close to what you want and customize them.+You can define your own set of sorting rules.+It's easiest to take existing rules that are close to what you want and customize them.-In the example below, we create a custom rule set for German called DIN 5007-2. This rule set treats umlauts in German differently: it treats ö as equivalent to oe, ä as equivalent to ae, and ü as equivalent to ue. For more information, see the http://icu-project.org/apiref/icu4j/com/ibm/icu/text/RuleBasedCollator.html[ICU RuleBasedCollator javadocs].+In the example below, we create a custom rule set for German called DIN 5007-2. This rule set treats umlauts in German differently: it treats ö as equivalent to oe, ä as equivalent to ae, and ü as equivalent to ue.+For more information, see the http://icu-project.org/apiref/icu4j/com/ibm/icu/text/RuleBasedCollator.html[ICU RuleBasedCollator javadocs].This example shows how to create a custom rule set for `solr.ICUCollationField` and dump it to a file:@@ -401,9 +427,11 @@ Using a Tailored ruleset:=== ASCII Folding-This filter converts alphabetic, numeric, and symbolic Unicode characters which are not in the first 127 ASCII characters (the "Basic Latin" Unicode block) into their ASCII equivalents, if one exists. Only those characters with reasonable ASCII alternatives are converted.+This filter converts alphabetic, numeric, and symbolic Unicode characters which are not in the first 127 ASCII characters (the "Basic Latin" Unicode block) into their ASCII equivalents, if one exists.+Only those characters with reasonable ASCII alternatives are converted.-This can increase recall by causing more matches. On the other hand, it can reduce precision because language-specific character differences may be lost.+This can increase recall by causing more matches.+On the other hand, it can reduce precision because language-specific character differences may be lost.*Factory class:* `solr.ASCIIFoldingFilterFactory`@@ -447,7 +475,8 @@ This can increase recall by causing more matches. On the other hand, it can reduThis filter converts any character in the Unicode "Decimal Number" general category (`Nd`) into their equivalent Basic Latin digits (0-9).-This can increase recall by causing more matches. On the other hand, it can reduce precision because language-specific character differences may be lost.+This can increase recall by causing more matches.+On the other hand, it can reduce precision because language-specific character differences may be lost.*Factory class:* `solr.DecimalDigitFilterFactory`@@ -483,7 +512,8 @@ This can increase recall by causing more matches. On the other hand, it can redu== OpenNLP Integration-The `lucene/analysis/opennlp` module provides OpenNLP integration via several analysis components: a tokenizer, a part-of-speech tagging filter, a phrase chunking filter, and a lemmatization filter. In addition to these analysis components, Solr also provides an update request processor to extract named entities - see <<update-request-processors.adoc#update-processor-factories-that-can-be-loaded-as-plugins,Update Processor Factories That Can Be Loaded as Plugins>>.+The `lucene/analysis/opennlp` module provides OpenNLP integration via several analysis components: a tokenizer, a part-of-speech tagging filter, a phrase chunking filter, and a lemmatization filter.+In addition to these analysis components, Solr also provides an update request processor to extract named entities - see <<update-request-processors.adoc#update-processor-factories-that-can-be-loaded-as-plugins,Update Processor Factories That Can Be Loaded as Plugins>>.NOTE: The <<OpenNLP Tokenizer>> must be used with all other OpenNLP analysis components, for two reasons: first, the OpenNLP Tokenizer detects and marks the sentence boundaries required by all the OpenNLP filters; and second, since the pre-trained OpenNLP models used by these filters were trained using the corresponding language-specific sentence-detection/tokenization models, the same tokenization, using the same models, must be used at runtime for optimal performance.@@ -491,15 +521,19 @@ To use the OpenNLP components, you must add additional .jars to Solr's classpath=== OpenNLP Tokenizer-The OpenNLP Tokenizer takes two language-specific binary model files as parameters: a sentence detector model and a tokenizer model. The last token in each sentence is flagged, so that following OpenNLP-based filters can use this information to apply operations to tokens one sentence at a time. See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.+The OpenNLP Tokenizer takes two language-specific binary model files as parameters: a sentence detector model and a tokenizer model.+The last token in each sentence is flagged, so that following OpenNLP-based filters can use this information to apply operations to tokens one sentence at a time.+See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.*Factory class:* `solr.OpenNLPTokenizerFactory`*Arguments:*-`sentenceModel`:: (required) The path of a language-specific OpenNLP sentence detection model file. See <<resource-loading.adoc#,Resource Loading>> for more information.+`sentenceModel`:: (required) The path of a language-specific OpenNLP sentence detection model file.+See <<resource-loading.adoc#,Resource Loading>> for more information.-`tokenizerModel`:: (required) The path of a language-specific OpenNLP tokenization model file. See <<resource-loading.adoc#,Resource Loading>> for more information.+`tokenizerModel`:: (required) The path of a language-specific OpenNLP tokenization model file.+See <<resource-loading.adoc#,Resource Loading>> for more information.*Example:*@@ -533,7 +567,8 @@ The OpenNLP Tokenizer takes two language-specific binary model files as paramete=== OpenNLP Part-Of-Speech Filter-This filter sets each token's type attribute to the part of speech (POS) assigned by the configured model. See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.+This filter sets each token's type attribute to the part of speech (POS) assigned by the configured model.+See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.NOTE: Lucene currently does not index token types, so if you want to keep this information, you have to preserve it either in a payload or as a synonym; see the examples below.@@ -541,11 +576,12 @@ NOTE: Lucene currently does not index token types, so if you want to keep this i*Arguments:*-`posTaggerModel`:: (required) The path of a language-specific OpenNLP POS tagger model file. See <<resource-loading.adoc#,Resource Loading>> for more information.+`posTaggerModel`:: (required) The path of a language-specific OpenNLP POS tagger model file.+See <<resource-loading.adoc#,Resource Loading>> for more information.*Examples:*-The OpenNLP tokenizer will tokenize punctuation, which is useful for following token filters, but ordinarily you don't want to include punctuation in your index, so the `TypeTokenFilter` (<<filter-descriptions.adoc#type-token-filter,described here>>) is included in the examples below, with `stop.pos.txt` containing the following:+The OpenNLP tokenizer will tokenize punctuation, which is useful for following token filters, but ordinarily you don't want to include punctuation in your index, so the `TypeTokenFilter` (<<filters.adoc#type-token-filter,described here>>) is included in the examples below, with `stop.pos.txt` containing the following:.stop.pos.txt[source,text]@@ -597,7 +633,7 @@ Index the POS for each token as a payload:====---Index the POS for each token as a synonym, after prefixing the POS with "@" (see the <<filter-descriptions.adoc#type-as-synonym-filter,TypeAsSynonymFilter description>>):+Index the POS for each token as a synonym, after prefixing the POS with "@" (see the <<filters.adoc#type-as-synonym-filter,TypeAsSynonymFilter description>>):[source,xml]----@@ -626,7 +662,9 @@ Only index nouns - the `keep.pos.txt` file contains lines `NN`, `NNS`, `NNP` and=== OpenNLP Phrase Chunking Filter-This filter sets each token's type attribute based on the output of an OpenNLP phrase chunking model. The chunk labels replace the POS tags that previously were in each token's type attribute. See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.+This filter sets each token's type attribute based on the output of an OpenNLP phrase chunking model.+The chunk labels replace the POS tags that previously were in each token's type attribute.+See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.Prerequisite: the <<OpenNLP Tokenizer>> and the <<OpenNLP Part-Of-Speech Filter>> must precede this filter.@@ -636,7 +674,8 @@ NOTE: Lucene currently does not index token types, so if you want to keep this i*Arguments:*-`chunkerModel`:: (required) The path of a language-specific OpenNLP phrase chunker model file. See <<resource-loading.adoc#,Resource Loading>> for more information.+`chunkerModel`:: (required) The path of a language-specific OpenNLP phrase chunker model file.+See <<resource-loading.adoc#,Resource Loading>> for more information.*Examples*:@@ -676,7 +715,7 @@ Index the phrase chunk label for each token as a payload:====---Index the phrase chunk label for each token as a synonym, after prefixing it with "#" (see the <<filter-descriptions.adoc#type-as-synonym-filter,TypeAsSynonymFilter description>>):+Index the phrase chunk label for each token as a synonym, after prefixing it with "#" (see the <<filters.adoc#type-as-synonym-filter,TypeAsSynonymFilter description>>):[source,xml]----@@ -692,7 +731,10 @@ Index the phrase chunk label for each token as a synonym, after prefixing it wit=== OpenNLP Lemmatizer Filter-This filter replaces the text of each token with its lemma. Both a dictionary-based lemmatizer and a model-based lemmatizer are supported. If both are configured, the dictionary-based lemmatizer is tried first, and then the model-based lemmatizer is consulted for out-of-vocabulary tokens. See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.+This filter replaces the text of each token with its lemma.+Both a dictionary-based lemmatizer and a model-based lemmatizer are supported.+If both are configured, the dictionary-based lemmatizer is tried first, and then the model-based lemmatizer is consulted for out-of-vocabulary tokens.+See the http://opennlp.apache.org/models.html[OpenNLP website] for information on downloading pre-trained models.*Factory class:* `solr.OpenNLPLemmatizerFilter`@@ -700,9 +742,12 @@ This filter replaces the text of each token with its lemma. Both a dictionary-baEither `dictionary` or `lemmatizerModel` must be provided, and both may be provided - see the examples below:-`dictionary`:: (optional) The path of a lemmatization dictionary file. See <<resource-loading.adoc#,Resource Loading>> for more information. The dictionary file must be encoded as UTF-8, with one entry per line, in the form `word[tab]lemma[tab]part-of-speech`, e.g., `wrote[tab]write[tab]VBD`.+`dictionary`:: (optional) The path of a lemmatization dictionary file.+See <<resource-loading.adoc#,Resource Loading>> for more information.+The dictionary file must be encoded as UTF-8, with one entry per line, in the form `word[tab]lemma[tab]part-of-speech`, e.g., `wrote[tab]write[tab]VBD`.-`lemmatizerModel`:: (optional) The path of a language-specific OpenNLP lemmatizer model file. See <<resource-loading.adoc#,Resource Loading>> for more information.+`lemmatizerModel`:: (optional) The path of a language-specific OpenNLP lemmatizer model file.+See <<resource-loading.adoc#,Resource Loading>> for more information.*Examples:*@@ -778,7 +823,8 @@ Perform model-based lemmatization only, preserving the original token and emitti== Language-Specific Factories-These factories are each designed to work with specific languages. The languages covered here are:+These factories are each designed to work with specific languages.+The languages covered here are:* <<Arabic>>* <<Bengali>>@@ -860,7 +906,8 @@ This algorithm defines both character normalization and stemming, so these are s=== Bengali-There are two filters written specifically for dealing with Bengali language. They use the Lucene classes `org.apache.lucene.analysis.bn.BengaliNormalizationFilter` and `org.apache.lucene.analysis.bn.BengaliStemFilter`.+There are two filters written specifically for dealing with Bengali language.+They use the Lucene classes `org.apache.lucene.analysis.bn.BengaliNormalizationFilter` and `org.apache.lucene.analysis.bn.BengaliStemFilter`.*Factory classes:* `solr.BengaliStemFilterFactory`, `solr.BengaliNormalizationFilterFactory`@@ -903,7 +950,8 @@ There are two filters written specifically for dealing with Bengali language. Th=== Brazilian Portuguese-This is a Java filter written specifically for stemming the Brazilian dialect of the Portuguese language. It uses the Lucene class `org.apache.lucene.analysis.br.BrazilianStemmer`. Although that stemmer can be configured to use a list of protected words (which should not be stemmed), this factory does not accept any arguments to specify such a list.+This is a Java filter written specifically for stemming the Brazilian dialect of the Portuguese language.+It uses the Lucene class `org.apache.lucene.analysis.br.BrazilianStemmer`. Although that stemmer can be configured to use a list of protected words (which should not be stemmed), this factory does not accept any arguments to specify such a list.*Factory class:* `solr.BrazilianStemFilterFactory`@@ -1033,9 +1081,13 @@ Solr can stem Catalan using the Snowball Porter Stemmer with an argument of `lan=== Traditional Chinese-The default configuration of the <<tokenizers.adoc#icu-tokenizer,ICU Tokenizer>> is suitable for Traditional Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.+The default configuration of the <<tokenizers.adoc#icu-tokenizer,ICU Tokenizer>> is suitable for Traditional Chinese text.+It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words.+To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.-<<tokenizers.adoc#standard-tokenizer,Standard Tokenizer>> can also be used to tokenize Traditional Chinese text. Following the Word Break rules from the Unicode Text Segmentation algorithm, it produces one token per Chinese character. When combined with <<CJK Bigram Filter>>, overlapping bigrams of Chinese characters are formed.+<<tokenizers.adoc#standard-tokenizer,Standard Tokenizer>> can also be used to tokenize Traditional Chinese text.+Following the Word Break rules from the Unicode Text Segmentation algorithm, it produces one token per Chinese character.+When combined with <<CJK Bigram Filter>>, overlapping bigrams of Chinese characters are formed.<<CJK Width Filter>> folds fullwidth ASCII variants into the equivalent Basic Latin forms.@@ -1085,29 +1137,38 @@ Forms bigrams (overlapping 2-character sequences) of CJK characters that are genBy default, all CJK characters produce bigrams, but finer grained control is available by specifying orthographic type arguments `han`, `hiragana`, `katakana`, and `hangul`. When set to `false`, characters of the corresponding type will be passed through as unigrams, and will not be included in any bigrams.-When a CJK character has no adjacent characters to form a bigram, it is output in unigram form. If you want to always output both unigrams and bigrams, set the `outputUnigrams` argument to `true`.+When a CJK character has no adjacent characters to form a bigram, it is output in unigram form.+If you want to always output both unigrams and bigrams, set the `outputUnigrams` argument to `true`.In all cases, all non-CJK input is passed through unmodified.*Arguments:*-`han`:: (true/false) If false, Han (Chinese) characters will not form bigrams. Default is true.+`han`:: (true/false) If false, Han (Chinese) characters will not form bigrams.+Default is true.-`hiragana`:: (true/false) If false, Hiragana (Japanese) characters will not form bigrams. Default is true.+`hiragana`:: (true/false) If false, Hiragana (Japanese) characters will not form bigrams.+Default is true.-`katakana`:: (true/false) If false, Katakana (Japanese) characters will not form bigrams. Default is true.+`katakana`:: (true/false) If false, Katakana (Japanese) characters will not form bigrams.+Default is true.-`hangul`:: (true/false) If false, Hangul (Korean) characters will not form bigrams. Default is true.+`hangul`:: (true/false) If false, Hangul (Korean) characters will not form bigrams.+Default is true.-`outputUnigrams`:: (true/false) If true, in addition to forming bigrams, all characters are also passed through as unigrams. Default is false.+`outputUnigrams`:: (true/false) If true, in addition to forming bigrams, all characters are also passed through as unigrams.+Default is false.See the example under <<Traditional Chinese>>.=== Simplified Chinese-For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <<HMM Chinese Tokenizer>>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.+For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the <<HMM Chinese Tokenizer>>. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model.+To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.-The default configuration of the <<tokenizers.adoc#icu-tokenizer,ICU Tokenizer>> is also suitable for Simplified Chinese text. It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.+The default configuration of the <<tokenizers.adoc#icu-tokenizer,ICU Tokenizer>> is also suitable for Simplified Chinese text.+It follows the Word Break rules from the Unicode Text Segmentation algorithm for non-Chinese text, and uses a dictionary to segment Chinese words.+To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See the `solr/contrib/analysis-extras/README.md` for information on which jars you need to add.Also useful for Chinese analysis:@@ -1162,7 +1223,9 @@ Also useful for Chinese analysis:=== HMM Chinese Tokenizer-For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the `solr.HMMChineseTokenizerFactory` in the `analysis-extras` contrib module. This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.+For Simplified Chinese, Solr provides support for Chinese sentence and word segmentation with the `solr.HMMChineseTokenizerFactory` in the `analysis-extras` contrib module.+This component includes a large dictionary and segments Chinese text into words with the Hidden Markov Model.+To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.*Factory class:* `solr.HMMChineseTokenizerFactory`@@ -1174,7 +1237,8 @@ To use the default setup with fallback to English Porter stemmer for English wor`<analyzer class="org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer"/>`-Or to configure your own analysis setup, use the `solr.HMMChineseTokenizerFactory` along with your custom filter setup. See an example of this in the <<Simplified Chinese>> section.+Or to configure your own analysis setup, use the `solr.HMMChineseTokenizerFactory` along with your custom filter setup.+See an example of this in the <<Simplified Chinese>> section.=== Czech@@ -1407,15 +1471,19 @@ Solr includes support for stemming Finnish, and Lucene includes an example stopw==== Elision Filter-Removes article elisions from a token stream. This filter can be useful for languages such as French, Catalan, Italian, and Irish.+Removes article elisions from a token stream.+This filter can be useful for languages such as French, Catalan, Italian, and Irish.*Factory class:* `solr.ElisionFilterFactory`*Arguments:*-`articles`:: The pathname of a file that contains a list of articles, one per line, to be stripped. Articles are words such as "le", which are commonly abbreviated, such as in _l'avion_ (the plane). This file should include the abbreviated form, which precedes the apostrophe. In this case, simply "_l_". If no `articles` attribute is specified, a default set of French articles is used.+`articles`:: The pathname of a file that contains a list of articles, one per line, to be stripped.+Articles are words such as "le", which are commonly abbreviated, such as in _l'avion_ (the plane). This file should include the abbreviated form, which precedes the apostrophe.+In this case, simply "_l_". If no `articles` attribute is specified, a default set of French articles is used.-`ignoreCase`:: (boolean) If true, the filter ignores the case of words when comparing them to the common word file. Defaults to `false`+`ignoreCase`:: (boolean) If true, the filter ignores the case of words when comparing them to the common word file.+Defaults to `false`*Example:*@@ -1771,7 +1839,8 @@ Solr includes two stemmers for Italian: one in the `solr.SnowballPorterFilterFac=== Irish-Solr can stem Irish using the Snowball Porter Stemmer with an argument of `language="Irish"`. Solr includes `solr.IrishLowerCaseFilterFactory`, which can handle Irish-specific constructs. Solr also includes a set of contractions for Irish which can be stripped using `solr.ElisionFilterFactory`.+Solr can stem Irish using the Snowball Porter Stemmer with an argument of `language="Irish"`. Solr includes `solr.IrishLowerCaseFilterFactory`, which can handle Irish-specific constructs.+Solr also includes a set of contractions for Irish which can be stripped using `solr.ElisionFilterFactory`.*Factory class:* `solr.SnowballPorterFilterFactory`@@ -1835,7 +1904,8 @@ Also useful for Japanese analysis, from lucene-analyzers-common:==== Japanese Iteration Mark CharFilter-Normalizes horizontal Japanese iteration marks (odoriji) to their expanded form. Vertical iteration marks are not supported.+Normalizes horizontal Japanese iteration marks (odoriji) to their expanded form.+Vertical iteration marks are not supported.*Factory class:* `JapaneseIterationMarkCharFilterFactory`@@ -1855,7 +1925,9 @@ Tokenizer for Japanese that uses morphological analysis, and annotates each term*Arguments:*-`mode`:: Use `search` mode to get a noun-decompounding effect useful for search. `search` mode improves segmentation for search at the expense of part-of-speech accuracy. Valid values for `mode` are:+`mode`:: Use `search` mode to get a noun-decompounding effect useful for search.+`search` mode improves segmentation for search at the expense of part-of-speech accuracy.+Valid values for `mode` are:+* `normal`: default segmentation* `search`: segmentation useful for search (extra compound splitting)@@ -1863,7 +1935,8 @@ Tokenizer for Japanese that uses morphological analysis, and annotates each term+For some applications it might be good to use `search` mode for indexing and `normal` mode for queries to increase precision and prevent parts of compounds from being matched and highlighted.-`userDictionary`:: filename for a user dictionary, which allows overriding the statistical model with your own entries for segmentation, part-of-speech tags and readings without a need to specify weights. See `lang/userdict_ja.txt` for a sample user dictionary file.+`userDictionary`:: filename for a user dictionary, which allows overriding the statistical model with your own entries for segmentation, part-of-speech tags and readings without a need to specify weights.+See `lang/userdict_ja.txt` for a sample user dictionary file.`userDictionaryEncoding`:: user dictionary encoding (default is UTF-8)@@ -1881,7 +1954,8 @@ Replaces original terms' text with the corresponding base form (lemma). (`Japane==== Japanese Part Of Speech Stop Filter-Removes terms with one of the configured parts-of-speech. `JapaneseTokenizer` annotates terms with parts-of-speech.+Removes terms with one of the configured parts-of-speech.+`JapaneseTokenizer` annotates terms with parts-of-speech.*Factory class* *:* `JapanesePartOfSpeechStopFilterFactory`@@ -1889,7 +1963,8 @@ Removes terms with one of the configured parts-of-speech. `JapaneseTokenizer` an`tags`:: filename for a list of parts-of-speech for which to remove terms; see `conf/lang/stoptags_ja.txt` in the `sample_techproducts_config` <<config-sets.adoc#,configset>> for an example.-`enablePositionIncrements`:: if `luceneMatchVersion` is `4.3` or earlier and `enablePositionIncrements="false"`, no position holes will be left by this filter when it removes tokens. *This argument is invalid if `luceneMatchVersion` is `5.0` or later.*+`enablePositionIncrements`:: if `luceneMatchVersion` is `4.3` or earlier and `enablePositionIncrements="false"`, no position holes will be left by this filter when it removes tokens.+*This argument is invalid if `luceneMatchVersion` is `5.0` or later.*==== Japanese Katakana Stem Filter@@ -1901,7 +1976,8 @@ Normalizes common katakana spelling variations ending in a long sound character*Arguments:*-`minimumLength`:: terms below this length will not be stemmed. Default is 4, value must be 2 or more.+`minimumLength`:: terms below this length will not be stemmed.+Default is 4, value must be 2 or more.==== CJK Width Filter@@ -1960,7 +2036,8 @@ Example:[[hebrew-lao-myanmar-khmer]]=== Hebrew, Lao, Myanmar, Khmer-Lucene provides support, in addition to UAX#29 word break rules, for Hebrew's use of the double and single quote characters, and for segmenting Lao, Myanmar, and Khmer into syllables with the `solr.ICUTokenizerFactory` in the `analysis-extras` contrib module. To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.+Lucene provides support, in addition to UAX#29 word break rules, for Hebrew's use of the double and single quote characters, and for segmenting Lao, Myanmar, and Khmer into syllables with the `solr.ICUTokenizerFactory` in the `analysis-extras` contrib module.+To use this tokenizer, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.See <<tokenizers.adoc#icu-tokenizer,the ICUTokenizer>> for more information.@@ -2022,9 +2099,11 @@ For normalization, there is a `NorwegianNormalizationFilterFactory` which is a v==== Norwegian Light Stemmer-The `NorwegianLightStemFilterFactory` requires a "two-pass" sort for the -dom and -het endings. This means that in the first pass the word "kristendom" is stemmed to "kristen", and then all the general rules apply so it will be further stemmed to "krist". The effect of this is that "kristen," "kristendom," "kristendommen," and "kristendommens" will all be stemmed to "krist."+The `NorwegianLightStemFilterFactory` requires a "two-pass" sort for the -dom and -het endings.+This means that in the first pass the word "kristendom" is stemmed to "kristen", and then all the general rules apply so it will be further stemmed to "krist". The effect of this is that "kristen," "kristendom," "kristendommen," and "kristendommens" will all be stemmed to "krist."-The second pass is to pick up -dom and -het endings. Consider this example:+The second pass is to pick up -dom and -het endings.+Consider this example:[width="100%",options="header",]|===@@ -2043,7 +2122,8 @@ The second pass is to pick up -dom and -het endings. Consider this example:*Arguments:*-`variant`:: Choose the Norwegian language variant to use. Valid values are:+`variant`:: Choose the Norwegian language variant to use.+Valid values are:+* `nb:` Bokmål (default)* `nn:` Nynorsk@@ -2099,7 +2179,8 @@ The `NorwegianMinimalStemFilterFactory` stems plural forms of Norwegian nouns on*Arguments:*-`variant`:: Choose the Norwegian language variant to use. Valid values are:+`variant`:: Choose the Norwegian language variant to use.+Valid values are:+* `nb:` Bokmål (default)* `nn:` Nynorsk@@ -2211,7 +2292,9 @@ Solr includes support for normalizing Persian, and Lucene includes an example st=== Polish-Solr provides support for Polish stemming with the `solr.StempelPolishStemFilterFactory`, and `solr.MorphologikFilterFactory` for lemmatization, in the `contrib/analysis-extras` module. The `solr.StempelPolishStemFilterFactory` component includes an algorithmic stemmer with tables for Polish. To use either of these filters, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.+Solr provides support for Polish stemming with the `solr.StempelPolishStemFilterFactory`, and `solr.MorphologikFilterFactory` for lemmatization, in the `contrib/analysis-extras` module.+The `solr.StempelPolishStemFilterFactory` component includes an algorithmic stemmer with tables for Polish.+To use either of these filters, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.*Factory class:* `solr.StempelPolishStemFilterFactory` and `solr.MorfologikFilterFactory`@@ -2266,7 +2349,10 @@ More information about the Stempel stemmer is available in {lucene-javadocs}/anaNote the lower case filter is applied _after_ the Morfologik stemmer; this is because the Polish dictionary contains proper names and then proper term case may be important to resolve disambiguities (or even lookup the correct lemma at all).-The Morfologik dictionary parameter value is a constant specifying which dictionary to choose. The dictionary resource must be named `path/to/_language_.dict` and have an associated `.info` metadata file. See http://morfologik.blogspot.com/[the Morfologik project] for details. If the dictionary attribute is not provided, the Polish dictionary is loaded and used by default.+The Morfologik dictionary parameter value is a constant specifying which dictionary to choose.+The dictionary resource must be named `path/to/_language_.dict` and have an associated `.info` metadata file.+See http://morfologik.blogspot.com/[the Morfologik project] for details.+If the dictionary attribute is not provided, the Polish dictionary is loaded and used by default.=== Portuguese@@ -2414,7 +2500,8 @@ Solr includes two stemmers for Russian: one in the `solr.SnowballPorterFilterFacScandinavian is a language group spanning three languages <<Norwegian>>, <<Swedish>> and <<Danish>> which are very similar.-Swedish å, ä, ö are in fact the same letters as Norwegian and Danish å, æ, ø and thus interchangeable when used between these languages. They are however folded differently when people type them on a keyboard lacking these characters.+Swedish å, ä, ö are in fact the same letters as Norwegian and Danish å, æ, ø and thus interchangeable when used between these languages.+They are however folded differently when people type them on a keyboard lacking these characters.In that situation almost all Swedish people use a, a, o instead of å, ä, ö. Norwegians and Danes on the other hand usually type aa, ae and oe instead of å, æ and ø. Some do however use a, a, o, oo, ao and sometimes permutations of everything above.@@ -2426,7 +2513,8 @@ See also each language section for other relevant filters.This filter normalize use of the interchangeable Scandinavian characters æÆäÄöÖøØ and folded variants (aa, ao, ae, oe and oo) by transforming them to åÅæÆøØ.-It's a semantically less destructive solution than `ScandinavianFoldingFilter`, most useful when a person with a Norwegian or Danish keyboard queries a Swedish index and vice versa. This filter does *not* perform the common Swedish folds of å and ä to a nor ö to o.+It's a semantically less destructive solution than `ScandinavianFoldingFilter`, most useful when a person with a Norwegian or Danish keyboard queries a Swedish index and vice versa.+This filter does *not* perform the common Swedish folds of å and ä to a nor ö to o.*Factory class:* `solr.ScandinavianNormalizationFilterFactory`@@ -2470,7 +2558,8 @@ It's a semantically less destructive solution than `ScandinavianFoldingFilter`,==== Scandinavian Folding Filter-This filter folds Scandinavian characters åÅäæÄÆ -> a and öÖøØ -> o. It also discriminate against use of double vowels aa, ae, ao, oe and oo, leaving just the first one.+This filter folds Scandinavian characters åÅäæÄÆ -> a and öÖøØ -> o.+It also discriminate against use of double vowels aa, ae, ao, oe and oo, leaving just the first one.It's a semantically more destructive solution than `ScandinavianNormalizationFilter`, but can in addition help with matching raksmorgas as räksmörgås.@@ -2499,7 +2588,8 @@ It's a semantically more destructive solution than `ScandinavianNormalizationFil==== Serbian Normalization Filter-Solr includes a filter that normalizes Serbian Cyrillic and Latin characters. Note that this filter only works with lowercased input.+Solr includes a filter that normalizes Serbian Cyrillic and Latin characters.+Note that this filter only works with lowercased input.For user tips & advice on using this filter, see https://cwiki.apache.org/confluence/display/solr/SerbianLanguageSupport[Serbian Language Support] in the Solr Wiki.@@ -2507,7 +2597,8 @@ For user tips & advice on using this filter, see https://cwiki.apache.org/conflu*Arguments:*-`haircut`:: Select the extend of normalization. Valid values are:+`haircut`:: Select the extend of normalization.+Valid values are:+* `bald`: (Default behavior) Cyrillic characters are first converted to Latin; then, Latin characters have their diacritics removed, with the exception of https://en.wikipedia.org/wiki/D_with_stroke[LATIN SMALL LETTER D WITH STROKE] (U+0111) which is converted to "```dj```"* `regular`: Only Cyrillic to Latin normalization will be applied, preserving the Latin diatrics@@ -2637,7 +2728,8 @@ Also relevant are the <<Scandinavian,Scandinavian normalization filters>>.=== Thai-This filter converts sequences of Thai characters into individual Thai words. Unlike European languages, Thai does not use whitespace to delimit words.+This filter converts sequences of Thai characters into individual Thai words.+Unlike European languages, Thai does not use whitespace to delimit words.*Factory class:* `solr.ThaiTokenizerFactory`@@ -2728,7 +2820,8 @@ Solr includes support for stemming Turkish with the `solr.SnowballPorterFilterFa=== Ukrainian-Solr provides support for Ukrainian lemmatization with the `solr.MorphologikFilterFactory`, in the `contrib/analysis-extras` module. To use this filter, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.+Solr provides support for Ukrainian lemmatization with the `solr.MorphologikFilterFactory`, in the `contrib/analysis-extras` module.+To use this filter, you must add additional .jars to Solr's classpath (as described in the section <<solr-plugins.adoc#installing-plugins,Solr Plugins>>). See `solr/contrib/analysis-extras/README.md` for instructions on which jars you need to add.Lucene also includes an example Ukrainian stopword list, in the `lucene-analyzers-morfologik` jar.@@ -2770,4 +2863,7 @@ Lucene also includes an example Ukrainian stopword list, in the `lucene-analyzer====---The Morfologik `dictionary` parameter value is a constant specifying which dictionary to choose. The dictionary resource must be named `path/to/_language_.dict` and have an associated `.info` metadata file. See http://morfologik.blogspot.com/[the Morfologik project] for details. If the dictionary attribute is not provided, the Polish dictionary is loaded and used by default.+The Morfologik `dictionary` parameter value is a constant specifying which dictionary to choose.+The dictionary resource must be named `path/to/_language_.dict` and have an associated `.info` metadata file.+See http://morfologik.blogspot.com/[the Morfologik project] for details.+If the dictionary attribute is not provided, the Polish dictionary is loaded and used by default.diff --git a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc b/solr/solr-ref-guide/src/language-detection.adocsimilarity index 77%rename from solr/solr-ref-guide/src/detecting-languages-during-indexing.adocrename to solr/solr-ref-guide/src/language-detection.adocindex d3184588991..3383ebde479 100644--- a/solr/solr-ref-guide/src/detecting-languages-during-indexing.adoc+++ b/solr/solr-ref-guide/src/language-detection.adoc@@ -1,4 +1,4 @@-= Detecting Languages During Indexing+= Language Detection// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -24,7 +24,8 @@ Solr supports three implementations of this feature:* LangDetect language detection: https://github.com/shuyo/language-detection* OpenNLP language detection: http://opennlp.apache.org/docs/{ivy-opennlp-version}/manual/opennlp.html#tools.langdetect-You can see a comparison between the Tika and LangDetect implementations here: http://blog.mikemccandless.com/2011/10/accuracy-and-performance-of-googles.html. In general, the LangDetect implementation supports more languages with higher performance.+You can see a comparison between the Tika and LangDetect implementations here: http://blog.mikemccandless.com/2011/10/accuracy-and-performance-of-googles.html.+In general, the LangDetect implementation supports more languages with higher performance.For specific information on each of these language identification implementations, including a list of supported languages for each, see the relevant project websites.@@ -32,7 +33,8 @@ For more information about language analysis in Solr, see <<language-analysis.ad== Configuring Language Detection-You can configure the `langid` UpdateRequestProcessor in `solrconfig.xml`. Both implementations take the same parameters, which are described in the following section. At a minimum, you must specify the fields for language identification and a field for the resulting language code.+You can configure the `langid` UpdateRequestProcessor in `solrconfig.xml`. Both implementations take the same parameters, which are described in the following section.+At a minimum, you must specify the fields for language identification and a field for the resulting language code.=== Configuring Tika Language Detection@@ -80,7 +82,11 @@ Here is an example of a minimal OpenNLP `langid` configuration in `solrconfig.xm==== OpenNLP-specific Parameters`langid.model`::-An OpenNLP language detection model. The OpenNLP project provides a pre-trained 103 language model on the http://opennlp.apache.org/models.html[OpenNLP site's model dowload page]. Model training instructions are provided on the http://opennlp.apache.org/docs/{ivy-opennlp-version}/manual/opennlp.html#tools.langdetect[OpenNLP website]. This parameter is required. See <<resource-loading.adoc#,Resource Loading>> for information on where to put the model.+An OpenNLP language detection model.+++The OpenNLP project provides a pre-trained 103 language model on the http://opennlp.apache.org/models.html[OpenNLP site's model dowload page]. Model training instructions are provided on the http://opennlp.apache.org/docs/{ivy-opennlp-version}/manual/opennlp.html#tools.langdetect[OpenNLP website]. This parameter is required.+++See <<resource-loading.adoc#,Resource Loading>> for information on where to put the model.==== OpenNLP Language Codes@@ -97,13 +103,16 @@ When `true`, the default, enables language detection.A comma- or space-delimited list of fields to be processed by `langid`. This parameter is required.`langid.langField`::-Specifies the field for the returned language code. This parameter is required.+Specifies the field for the returned language code.+This parameter is required.`langid.langsField`::-Specifies the field for a list of returned language codes. If you use `langid.map.individual`, each detected language will be added to this field.+Specifies the field for a list of returned language codes.+If you use `langid.map.individual`, each detected language will be added to this field.`langid.overwrite`::-Specifies whether the content of the `langField` and `langsField` fields will be overwritten if they already contain values. The default is `false`.+Specifies whether the content of the `langField` and `langsField` fields will be overwritten if they already contain values.+The default is `false`.`langid.lcmap`::A space-separated list specifying colon delimited language code mappings to apply to the detected languages.@@ -115,24 +124,30 @@ This affects both the values put into the `langField` and `langsField` fields, a`langid.threshold`::Specifies a threshold value between 0 and 1 that the language identification score must reach before `langid` accepts it.+-With longer text fields, a high threshold such as `0.8` will give good results. For shorter text fields, you may need to lower the threshold for language identification, though you will be risking somewhat lower quality results. We recommend experimenting with your data to tune your results.+With longer text fields, a high threshold such as `0.8` will give good results.+For shorter text fields, you may need to lower the threshold for language identification, though you will be risking somewhat lower quality results.+We recommend experimenting with your data to tune your results.+The default is `0.5`.`langid.allowlist`::-Specifies a list of allowed language identification codes. Use this in combination with `langid.map` to ensure that you only index documents into fields that are in your schema.+Specifies a list of allowed language identification codes.+Use this in combination with `langid.map` to ensure that you only index documents into fields that are in your schema.`langid.map`::-Enables field name mapping. If `true`, Solr will map field names for all fields listed in `langid.fl`. The default is `false`.+Enables field name mapping.+If `true`, Solr will map field names for all fields listed in `langid.fl`. The default is `false`.`langid.map.fl`::A comma-separated list of fields for `langid.map` that is different than the fields specified in `langid.fl`.`langid.map.keepOrig`::-If `true`, Solr will copy the field during the field name mapping process, leaving the original field in place. The default is `false`.+If `true`, Solr will copy the field during the field name mapping process, leaving the original field in place.+The default is `false`.`langid.map.individual`::-If `true`, Solr will detect and map languages for each field individually. The default is `false`.+If `true`, Solr will detect and map languages for each field individually.+The default is `false`.`langid.map.individual.fl`::A comma-separated list of fields for use with `langid.map.individual` that is different than the fields specified in `langid.fl`.@@ -159,6 +174,7 @@ By default, fields are mapped as <field>_<language>. To change this pattern, youBy default, fields are mapped as `<field>_<language>`. To change this pattern, you can specify a Java replace in this parameter.`langid.enforceSchema`::-If `false`, the `langid` processor does not validate field names against your schema. This may be useful if you plan to rename or delete fields later in the UpdateChain.+If `false`, the `langid` processor does not validate field names against your schema.+This may be useful if you plan to rename or delete fields later in the UpdateChain.+The default is `true`.diff --git a/solr/solr-ref-guide/src/learning-to-rank.adoc b/solr/solr-ref-guide/src/learning-to-rank.adocindex 27819164b7b..378698595ef 100644--- a/solr/solr-ref-guide/src/learning-to-rank.adoc+++ b/solr/solr-ref-guide/src/learning-to-rank.adoc@@ -83,7 +83,7 @@ The LTR contrib module includes several feature classes as well as support for c==== Feature Extraction-The ltr contrib module includes a <<transforming-result-documents.adoc#,[features>> transformer] to support the calculation and return of feature values for https://en.wikipedia.org/wiki/Feature_extraction[feature extraction] purposes including and especially when you do not yet have an actual reranking model.+The ltr contrib module includes a <<document-transformers.adoc#,[features>> transformer] to support the calculation and return of feature values for https://en.wikipedia.org/wiki/Feature_extraction[feature extraction] purposes including and especially when you do not yet have an actual reranking model.==== Feature Selection and Model Training@@ -658,7 +658,7 @@ NOTE: No `"features"` are configured in `myWrapperModel` because the features ofCAUTION: `<lib dir="/path/to/models" regex=".*\.json" />` doesn't work as expected in this case, because `SolrResourceLoader` considers given resources as JAR if `<lib />` indicates files.-As an alternative to the above-described `DefaultWrapperModel`, it is possible to <<setting-up-an-external-zookeeper-ensemble#increasing-the-file-size-limit,increase ZooKeeper's file size limit>>.+As an alternative to the above-described `DefaultWrapperModel`, it is possible to <<zookeeper-ensemble.adoc#increasing-the-file-size-limit,increase ZooKeeper's file size limit>>.=== Applying Changesdiff --git a/solr/solr-ref-guide/src/legacy-scaling-and-distribution.adoc b/solr/solr-ref-guide/src/legacy-scaling-and-distribution.adocdeleted file mode 100644index 4596935c2d7..00000000000--- a/solr/solr-ref-guide/src/legacy-scaling-and-distribution.adoc+++ /dev/null@@ -1,32 +0,0 @@-= Legacy Scaling and Distribution-:page-children: introduction-to-scaling-and-distribution, distributed-search-with-index-sharding, index-replication, combining-distribution-and-replication, merging-indexes-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section describes how to set up distribution and replication in Solr. It is considered "legacy" behavior, since while it is still supported in Solr, the SolrCloud functionality described in the previous chapter is where the current development is headed. However, if you don't need all that SolrCloud delivers, search distribution and index replication may be sufficient.--This section covers the following topics:--<<introduction-to-scaling-and-distribution.adoc#,Introduction to Scaling and Distribution>>: Conceptual information about distribution and replication in Solr.--<<distributed-search-with-index-sharding.adoc#,Distributed Search with Index Sharding>>: Detailed information about implementing distributed searching in Solr.--<<index-replication.adoc#,Index Replication>>: Detailed information about replicating your Solr indexes.--<<combining-distribution-and-replication.adoc#,Combining Distribution and Replication>>: Detailed information about replicating shards in a distributed index.--<<merging-indexes.adoc#,Merging Indexes>>: Information about combining separate indexes in Solr.diff --git a/solr/solr-ref-guide/src/libs.adoc b/solr/solr-ref-guide/src/libs.adocindex ebc40218944..f4c1e9f0d3e 100644--- a/solr/solr-ref-guide/src/libs.adoc+++ b/solr/solr-ref-guide/src/libs.adoc@@ -32,7 +32,7 @@ There are several special places you can place Solr plugin `.jar` files:This directory is not present by default so create it.See <<taking-solr-to-production.adoc#solr-home-directory,Solr home directory>>.-* `<core_instance>/lib/`: In standalone Solr, you may want to add plugins just for a specific Solr core.+* `<core_instance>/lib/`: In a user-managed cluster or a single-node installation, you may want to add plugins just for a specific Solr core.Create this adjacent to the `conf/` directory; it's not present by default.* `<solr_install>/server/solr-webapp/webapp/WEB-INF/lib/`: The `.jar` files for Solr itself and it's dependencies live here.diff --git a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc b/solr/solr-ref-guide/src/local-parameters-in-queries.adocdeleted file mode 100644index d7d29b7c078..00000000000--- a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc+++ /dev/null@@ -1,81 +0,0 @@-= Local Parameters in Queries-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Local parameters are arguments in a Solr request that are specific to a query parameter.--Local parameters provide a way to add meta-data to certain argument types such as query strings. (In Solr documentation, local parameters are sometimes referred to as LocalParams.)--Local parameters are specified as prefixes to arguments. Take the following query argument, for example:--`q=solr rocks`--We can prefix this query string with local parameters to provide more information to the Standard Query Parser. For example, we can change the default operator type to "AND" and the default field to "title":--`q={!q.op=AND df=title}solr rocks`--These local parameters would change the query to require a match on both "solr" and "rocks" while searching the "title" field by default.--== Basic Syntax of Local Parameters--To specify a local parameter, insert the following before the argument to be modified:--* Begin with `{!`--* Insert any number of key=value pairs separated by white space--* End with `}` and immediately follow with the query argument--You may specify only one local parameters prefix per argument. Values in the key-value pairs may be quoted via single or double quotes, and backslash escaping works within quoted strings.--== Query Type Short Form--If a local parameter value appears without a name, it is given the implicit name of "type". This allows short-form representation for the type of query parser to use when parsing a query string. Thus--`q={!dismax qf=myfield}solr rocks`--is equivalent to:--`q={!type=dismax qf=myfield}solr rocks`--If no "type" is specified (either explicitly or implicitly) then the <<the-standard-query-parser.adoc#,lucene parser>> is used by default. Thus--`fq={!df=summary}solr rocks`--is equivalent to:--`fq={!type=lucene df=summary}solr rocks`--== Specifying the Parameter Value with the v Key--A special key of `v` within local parameters is an alternate way to specify the value of that parameter.--`q={!dismax qf=myfield}solr rocks`--is equivalent to--`q={!type=dismax qf=myfield v='solr rocks'`}--== Parameter Dereferencing--Parameter dereferencing, or indirection, lets you use the value of another argument rather than specifying it directly. This can be used to simplify queries, decouple user input from query parameters, or decouple front-end GUI parameters from defaults set in `solrconfig.xml`.--`q={!dismax qf=myfield}solr rocks`--`is equivalent to:`--`q={!type=dismax qf=myfield v=$qq}&qq=solr rocks`diff --git a/solr/solr-ref-guide/src/local-params.adoc b/solr/solr-ref-guide/src/local-params.adocnew file mode 100644index 00000000000..7bf706977fa--- /dev/null+++ b/solr/solr-ref-guide/src/local-params.adoc@@ -0,0 +1,89 @@+= Local Params+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++_Local params_ are arguments in a Solr request that are specific to a query parameter.++Local params provide a way to add meta-data to certain argument types such as query strings.+Local params are also sometimes referred to as LocalParams.++Local params are specified as prefixes to arguments.+Take the following query argument, for example:++`q=solr rocks`++We can prefix this query string with local params to provide more information to the Standard Query Parser.+For example, we can change the default operator type to "AND" and the default field to "title":++`q={!q.op=AND df=title}solr rocks`++These local params would change the query to require a match on both "solr" and "rocks" while searching the "title" field by default.++== Basic Syntax of Local Params++To specify a local param, insert the following before the argument to be modified:++* Begin with `{!`++* Insert any number of key=value pairs separated by white space++* End with `}` and immediately follow with the query argument++You may specify only one local params prefix per argument.+Values in the key-value pairs may be quoted via single or double quotes, and backslash escaping works within quoted strings.++== Query Type Short Form++If a local param value appears without a name, it is given the implicit name of "type".+This allows short-form representation for the type of query parser to use when parsing a query string.+Thus:++`q={!dismax qf=myfield}solr rocks`++is equivalent to:++`q={!type=dismax qf=myfield}solr rocks`++If no "type" is specified (either explicitly or implicitly) then the <<standard-query-parser.adoc#,lucene parser>> is used by default.+Thus:++`fq={!df=summary}solr rocks`++is equivalent to:++`fq={!type=lucene df=summary}solr rocks`++== Specifying the Parameter Value with the v Key++A special key of `v` within local params is an alternate way to specify the value of that parameter.++`q={!dismax qf=myfield}solr rocks`++is equivalent to++`q={!type=dismax qf=myfield v='solr rocks'`}++== Parameter Dereferencing++Parameter dereferencing, or indirection, lets you use the value of another argument rather than specifying it directly.+This can be used to simplify queries, decouple user input from query parameters, or decouple front-end GUI parameters from defaults set in `solrconfig.xml`.++`q={!dismax qf=myfield}solr rocks`++`is equivalent to:`++`q={!type=dismax qf=myfield v=$qq}&qq=solr rocks`diff --git a/solr/solr-ref-guide/src/logging.adoc b/solr/solr-ref-guide/src/logging.adocdeleted file mode 100644index 5d2f023f8de..00000000000--- a/solr/solr-ref-guide/src/logging.adoc+++ /dev/null@@ -1,35 +0,0 @@-= Logging-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Logging page shows recent messages logged by this Solr node.--When you click the link for "Logging", a page similar to the one below will be displayed:--.The Main Logging Screen, including an example of an error due to a bad document sent by a client-image::images/logging/logging.png[image,width=621,height=250]--While this example shows logged messages for only one core, if you have multiple cores in a single instance, they will each be listed, with the level for each.--== Selecting a Logging Level--When you select the *Level* link on the left, you see the hierarchy of classpaths and classnames for your instance. A row highlighted in yellow indicates that the class has logging capabilities. Click on a highlighted row, and a menu will appear to allow you to change the log level for that class. Characters in boldface indicate that the class will not be affected by level changes to root.--.Log level selection-image::images/logging/level_menu.png[image,width=589,height=250]--For an explanation of the various logging levels, see <<configuring-logging.adoc#,Configuring Logging>>.diff --git a/solr/solr-ref-guide/src/logs.adoc b/solr/solr-ref-guide/src/logs.adocindex e8dfc34647e..807cc058609 100644--- a/solr/solr-ref-guide/src/logs.adoc+++ b/solr/solr-ref-guide/src/logs.adoc@@ -18,7 +18,8 @@This section of the user guide provides an introduction to Solr log analytics.-NOTE: This is an appendix of the <<math-expressions.adoc#,Visual Guide to Streaming Expressions and Math Expressions>>. All the functions described below are covered in detail in the guide.+NOTE: This is an appendix of the <<math-expressions.adoc#,Visual Guide to Streaming Expressions and Math Expressions>>.+All the functions described below are covered in detail in the guide.See the <<math-start.adoc#,Getting Started>> chapter to learn how to get started with visualizations and Apache Zeppelin.== Loadingdiff --git a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-6.adocsimilarity index 86%rename from solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adocrename to solr/solr-ref-guide/src/major-changes-in-solr-6.adocindex 61d5f404fa9..e083069d52d 100644--- a/solr/solr-ref-guide/src/major-changes-from-solr-5-to-solr-6.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-6.adoc@@ -1,4 +1,4 @@-= Major Changes from Solr 5 to Solr 6+= Major Changes in Solr 6// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -57,7 +57,7 @@ Most non-text field types in the Solr sample configsets now default to using <<d== Java 8 Required-The minimum supported version of Java for Solr 6 (and the <<using-solrj.adoc#,SolrJ client libraries>>) is now Java 8.+The minimum supported version of Java for Solr 6 (and the <<solrj.adoc#,SolrJ client libraries>>) is now Java 8.== Index Format Changes@@ -65,18 +65,18 @@ Solr 6 has no support for reading Lucene/Solr 4.x and earlier indexes. Be sure t== Managed Schema is now the Default-Solr's default behavior when a `solrconfig.xml` does not explicitly define a `<schemaFactory/>` is now dependent on the `luceneMatchVersion` specified in that `solrconfig.xml`. When `luceneMatchVersion < 6.0`, `ClassicIndexSchemaFactory` will continue to be used for back compatibility, otherwise an instance of <<schema-factory-definition-in-solrconfig.adoc#,`ManagedIndexSchemaFactory`>> will be used.+Solr's default behavior when a `solrconfig.xml` does not explicitly define a `<schemaFactory/>` is now dependent on the `luceneMatchVersion` specified in that `solrconfig.xml`. When `luceneMatchVersion < 6.0`, `ClassicIndexSchemaFactory` will continue to be used for back compatibility, otherwise an instance of <<schema-factory.adoc#,`ManagedIndexSchemaFactory`>> will be used.The most notable impacts of this change are:* Existing `solrconfig.xml` files that are modified to use `luceneMatchVersion >= 6.0`, but do _not_ have an explicitly configured `ClassicIndexSchemaFactory`, will have their `schema.xml` file automatically upgraded to a `managed-schema` file.* Schema modifications via the <<schema-api.adoc#,Schema API>> will now be enabled by default.-Please review the <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> section for more details.+Please review the <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> section for more details.== Default Similarity Changes-Solr's default behavior when a Schema does not explicitly define a global <<other-schema-elements.adoc#,`<similarity/>`>> is now dependent on the `luceneMatchVersion` specified in the `solrconfig.xml`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarityFactory` will be used, otherwise an instance of `SchemaSimilarityFactory` will be used. Most notably this change means that users can take advantage of per Field Type similarity declarations, without needing to also explicitly declare a global usage of `SchemaSimilarityFactory`.+Solr's default behavior when a Schema does not explicitly define a global <<schema-elements.adoc#similarity,`<similarity/>`>> is now dependent on the `luceneMatchVersion` specified in the `solrconfig.xml`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarityFactory` will be used, otherwise an instance of `SchemaSimilarityFactory` will be used. Most notably this change means that users can take advantage of per Field Type similarity declarations, without needing to also explicitly declare a global usage of `SchemaSimilarityFactory`.Regardless of whether it is explicitly declared, or used as an implicit global default, `SchemaSimilarityFactory` 's implicit behavior when a Field Types do not declare an explicit `<similarity />` has also been changed to depend on the the `luceneMatchVersion`. When `luceneMatchVersion < 6.0`, an instance of `ClassicSimilarity` will be used, otherwise an instance of `BM25Similarity` will be used. A `defaultSimFromFieldType` init option may be specified on the `SchemaSimilarityFactory` declaration to change this behavior. Please review the `SchemaSimilarityFactory` javadocs for more details.diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-7.adocindex ff6968d2756..181eb47f873 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-7.adoc@@ -40,7 +40,7 @@ To provide additional flexibility, two new types of replicas have been added, naAs part of this change, the traditional type of replica is now named NRT. If you do not explicitly define a number of TLOG or PULL replicas, Solr defaults to creating NRT replicas. If this model is working for you, you will not have to change anything.-See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more details on the new replica modes, and how define the replica type in your cluster.+See the section <<solrcloud-shards-indexing.adoc#types-of-replicas,Types of Replicas>> for more details on the new replica modes, and how define the replica type in your cluster.=== AutoscalingSolr autoscaling is a new suite of features in Solr to make managing a SolrCloud cluster easier and more automated.@@ -49,9 +49,7 @@ At its core, Solr autoscaling provides users with a rule syntax to define prefer=== Other Features & Enhancements-// TODO 7.1 - update link to docs when complete-* The Analytics Component has been refactored.-** The documentation for this component is in progress; until it is available, please refer to https://issues.apache.org/jira/browse/SOLR-11144[SOLR-11144] for more details.+* The <<analytics.adoc#,Analytics Component>> has been refactored.* There were several other new features released in earlier 6.x releases, which you may have missed:** <<learning-to-rank.adoc#,Learning to Rank>>@@ -70,7 +68,7 @@ Several changes have been made to configsets that ship with Solr; not only their* The `data_driven_configset` and `basic_configset` have been removed, and replaced by the `_default` configset. The `sample_techproducts_configset` also remains, and is designed for use with the example documents shipped with Solr in the `example/exampledocs` directory.* When creating a new collection, if you do not specify a configset, the `_default` will be used.** If you use SolrCloud, the `_default` configset will be automatically uploaded to ZooKeeper.-** If you use standalone mode, the instanceDir will be created automatically, using the `_default` configset as its basis.+** If you run a user-managed cluster or a single-node installation, the instanceDir will be created automatically, using the `_default` configset as its basis.=== Schemaless Improvements@@ -108,13 +106,13 @@ curl http://host:8983/api/collections/mycollection/config -d '{"set-user-propert=== Changes to Default Behaviors* JSON is now the default response format. If you rely on XML responses, you must now define `wt=xml` in your request. In addition, line indentation is enabled by default (`indent=on`).-* The `sow` parameter (short for "Split on Whitespace") now defaults to `false`, which allows support for multi-word synonyms out of the box. This parameter is used with the eDismax and standard/"lucene" query parsers. If this parameter is not explicitly specified as `true`, query text will not be split on whitespace before analysis.+* The `sow` parameter (short for "Split on Whitespace") now defaults to `false`, which allows support for multi-word synonyms out of the box. This parameter is used with the eDisMax and standard/"lucene" query parsers. If this parameter is not explicitly specified as `true`, query text will not be split on whitespace before analysis.* The `legacyCloud` parameter now defaults to `false`. If an entry for a replica does not exist in `state.json`, that replica will not get registered.+This may affect users who bring up replicas and they are automatically registered as a part of a shard. It is possible to fall back to the old behavior by setting the property `legacyCloud=true`, in the cluster properties using the following command:+`./server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:2181 -cmd clusterprop -name legacyCloud -val true`-* The eDismax query parser parameter `lowercaseOperators` now defaults to `false` if the `luceneMatchVersion` in `solrconfig.xml` is 7.0.0 or above. Behavior for `luceneMatchVersion` lower than 7.0.0 is unchanged (so, `true`). This means that clients must sent boolean operators (such as AND, OR and NOT) in upper case in order to be recognized, or you must explicitly set this parameter to `true`.+* The eDisMax query parser parameter `lowercaseOperators` now defaults to `false` if the `luceneMatchVersion` in `solrconfig.xml` is 7.0.0 or above. Behavior for `luceneMatchVersion` lower than 7.0.0 is unchanged (so, `true`). This means that clients must sent boolean operators (such as AND, OR and NOT) in upper case in order to be recognized, or you must explicitly set this parameter to `true`.* The `handleSelect` parameter in `solrconfig.xml` now defaults to `false` if the `luceneMatchVersion` is 7.0.0 or above. This causes Solr to ignore the `qt` parameter if it is present in a request. If you have request handlers without a leading '/', you can set `handleSelect="true"` or consider migrating your configuration.+The `qt` parameter is still used as a SolrJ special parameter that specifies the request handler (tail URL path) to use.@@ -168,9 +166,9 @@ The following changes were made in SolrJ.* `HttpSolrClient#setDefaultMaxConnectionsPerHost` and `HttpSolrClient#setMaxTotalConnections` have been removed. These now default very high and can only be changed via parameter when creating an HttpClient instance.=== Other Deprecations and Removals-* The `defaultOperator` parameter in the schema is no longer supported. Use the `q.op` parameter instead. This option had been deprecated for several releases. See the section <<the-standard-query-parser.adoc#standard-query-parser-parameters,Standard Query Parser Parameters>> for more information.-* The `defaultSearchField` parameter in the schema is no longer supported. Use the `df` parameter instead. This option had been deprecated for several releases. See the section <<the-standard-query-parser.adoc#standard-query-parser-parameters,Standard Query Parser Parameters>> for more information.-* The `mergePolicy`, `mergeFactor` and `maxMergeDocs` parameters have been removed and are no longer supported. You should define a `mergePolicyFactory` instead. See the section <<indexconfig-in-solrconfig.adoc#mergepolicyfactory,the mergePolicyFactory>> for more information.+* The `defaultOperator` parameter in the schema is no longer supported. Use the `q.op` parameter instead. This option had been deprecated for several releases. See the section <<standard-query-parser.adoc#standard-query-parser-parameters,Standard Query Parser Parameters>> for more information.+* The `defaultSearchField` parameter in the schema is no longer supported. Use the `df` parameter instead. This option had been deprecated for several releases. See the section <<standard-query-parser.adoc#standard-query-parser-parameters,Standard Query Parser Parameters>> for more information.+* The `mergePolicy`, `mergeFactor` and `maxMergeDocs` parameters have been removed and are no longer supported. You should define a `mergePolicyFactory` instead. See the section <<index-segments-merging.adoc#mergepolicyfactory,the mergePolicyFactory>> for more information.* The PostingsSolrHighlighter has been deprecated. It's recommended that you move to using the UnifiedHighlighter instead. See the section <<highlighting.adoc#the-unified-highlighter,Unified Highlighter>> for more information about this highlighter.* Index-time boosts have been removed from Lucene, and are no longer available from Solr. If any boosts are provided, they will be ignored by the indexing chain. As a replacement, index-time scoring factors should be indexed in a separate field and combined with the query score using a function query. See the section <<function-queries.adoc#,Function Queries>> for more information.* The `StandardRequestHandler` is deprecated. Use `SearchHandler` instead.@@ -198,7 +196,7 @@ Note again that this is not a complete list of all changes that may impact your** The metrics "75thPctlRequestTime", "95thPctlRequestTime", "99thPctlRequestTime" and "999thPctlRequestTime" in Overseer Status API have been renamed to "75thPcRequestTime", "95thPcRequestTime" and so on for consistency with stats output in other parts of Solr.** The metrics "avgRequestsPerMinute", "5minRateRequestsPerMinute" and "15minRateRequestsPerMinute" have been replaced by corresponding per-second rates viz. "avgRequestsPerSecond", "5minRateRequestsPerSecond" and "15minRateRequestsPerSecond" for consistency with stats output in other parts of Solr.* A new highlighter named UnifiedHighlighter has been added. You are encouraged to try out the UnifiedHighlighter by setting `hl.method=unified` and report feedback. It's more efficient/faster than the other highlighters, especially compared to the original Highlighter. See `HighlightParams.java` for a listing of highlight parameters annotated with which highlighters use them. `hl.useFastVectorHighlighter` is now considered deprecated in lieu of `hl.method=fastVector`.-* The <<query-settings-in-solrconfig.adoc#,`maxWarmingSearchers` parameter>> now defaults to 1, and more importantly commits will now block if this limit is exceeded instead of throwing an exception (a good thing). Consequently there is no longer a risk in overlapping commits. Nonetheless users should continue to avoid excessive committing. Users are advised to remove any pre-existing `maxWarmingSearchers` entries from their `solrconfig.xml` files.+* The <<caches-warming.adoc#,`maxWarmingSearchers` parameter>> now defaults to 1, and more importantly commits will now block if this limit is exceeded instead of throwing an exception (a good thing). Consequently there is no longer a risk in overlapping commits. Nonetheless users should continue to avoid excessive committing. Users are advised to remove any pre-existing `maxWarmingSearchers` entries from their `solrconfig.xml` files.* The <<other-parsers.adoc#complex-phrase-query-parser,Complex Phrase query parser>> now supports leading wildcards. Beware of its possible heaviness, users are encouraged to use ReversedWildcardFilter in index time analysis.* The JMX metric "avgTimePerRequest" (and the corresponding metric in the metrics API for each handler) used to be a simple non-decaying average based on total cumulative time and the number of requests. The Codahale Metrics implementation applies exponential decay to this value, which heavily biases the average towards the last 5 minutes.* Parallel SQL now uses Apache Calcite as its SQL framework. As part of this change the default aggregation mode has been changed to `facet` rather than `map_reduce`. There have also been changes to the SQL aggregate response and some SQL syntax changes. Consult the <<parallel-sql-interface.adoc#,Parallel SQL Interface>> documentation for full details.diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-8.adocindex a1b1aac89cc..53647f22154 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-8.adoc@@ -33,7 +33,7 @@ A thorough review of the list in <<Major Changes in Earlier 7.x Versions>>, belo*If using SolrCloud, you must be on Solr 7.3.0 or higher*. Solr's LeaderInRecovery (LIR) functionality <<Solr 7.3,changed significantly>> in Solr 7.3. While these changes were back-compatible for all subsequent 7.x releases, that compatibility has been removed in 8.0.In order to upgrade to Solr 8.x, all nodes of your cluster must be running Solr 7.3 or higher. If an upgrade is attempted with nodes running versions earlier than 7.3, documents could be lost.-If you are not using Solr in SolrCloud mode (you use Standalone Mode instead), we expect you can upgrade to Solr 8 from any 7.x version without major issues.+If you are not using Solr in SolrCloud mode (you run a user-managed cluster or a single-node installation), we expect you can upgrade to Solr 8 from any 7.x version without major issues.=== Rolling Upgrades with Solr 8@@ -152,7 +152,7 @@ The following changes impact how fields behave.Note that if you have not specified any similarityFactory in the schema, or use the default`SchemaSimilarityFactory`, then `LegacyBM25Similarity` is automatically selected when the value for `luceneMatchVersion` is lower than `8.0.0`.+-See also the section <<other-schema-elements.adoc#similarity,Similarity>> for more information.+See also the section <<schema-elements.adoc#similarity,Similarity>> for more information.*Memory Codecs Removed*@@ -197,7 +197,7 @@ Also note that the default set of date patterns (formats) have expanded from pre*Solr Cell*-* The extraction contrib (<<uploading-data-with-solr-cell-using-apache-tika.adoc#,Solr Cell>>) no longer does any date parsing, and thus no longer supports the `date.formats` parameter. To ensure date strings are properly parsed, use the `ParseDateFieldUpdateProcessorFactory` in your update chain. This update request processor is found by default with the "parse-date" update processor when running Solr in "<<schemaless-mode.adoc#set-the-default-updaterequestprocessorchain,schemaless mode>>".+* The extraction contrib (<<indexing-with-tika.adoc#,Solr Cell>>) no longer does any date parsing, and thus no longer supports the `date.formats` parameter. To ensure date strings are properly parsed, use the `ParseDateFieldUpdateProcessorFactory` in your update chain. This update request processor is found by default with the "parse-date" update processor when running Solr in "<<schemaless-mode.adoc#set-the-default-updaterequestprocessorchain,schemaless mode>>".*Langid Contrib*@@ -285,9 +285,12 @@ The login screen's purpose is cosmetic only - Admin UI-triggered Solr requests w* The `shards` parameter, used to manually select the shards and replicas that receive distributed requests, now checks nodes against an allow-list of acceptable values for security reasons.+-In SolrCloud mode this allow-list is automatically configured to contain all live nodes. In standalone mode the allow-list is empty by default. Upgrading users who use the `shards` parameter in standalone mode can correct this value by setting the `shardsWhitelist` property in any `shardHandler` configurations in their `solrconfig.xml` file.+In SolrCloud mode this allow-list is automatically configured to contain all live nodes.++In a user-managed cluster or a single-node installation the allow-list is empty by default.+Upgrading users who use the `shards` parameter in these installations can set this value by setting the `shardsWhitelist` property in any `shardHandler` configurations in their `solrconfig.xml` file.+-For more information, see the <<distributed-requests.adoc#configuring-the-shardhandlerfactory,Distributed Request>> documentation.+For more information, see the <<solrcloud-distributed-requests.adoc#configuring-the-shardhandlerfactory,Distributed Request>> documentation.=== Solr 7.6@@ -344,7 +347,7 @@ While most users are still encouraged to use the `NRTCachingDirectoryFactory`, w+For more information about the new directory factory, see the Jira issue https://issues.apache.org/jira/browse/LUCENE-8438[LUCENE-8438].+-For more information about the directory factory configuration in Solr, see the section <<datadir-and-directoryfactory-in-solrconfig.adoc#,DataDir and DirectoryFactory in SolrConfig>>.+For more information about the directory factory configuration in Solr, see the section <<index-location-format.adoc#,DataDir and DirectoryFactory in SolrConfig>>.=== Solr 7.5@@ -359,7 +362,7 @@ When upgrading to Solr 7.5, users should be aware of the following major changes*Index Merge Policy*-* When using the <<indexconfig-in-solrconfig.adoc#mergepolicyfactory,`TieredMergePolicy`>>, the default merge policy for Solr, `optimize` and `expungeDeletes` now respect the `maxMergedSegmentMB` configuration parameter, which defaults to `5000` (5GB).+* When using the <<index-segments-merging.adoc#mergepolicyfactory,`TieredMergePolicy`>>, the default merge policy for Solr, `optimize` and `expungeDeletes` now respect the `maxMergedSegmentMB` configuration parameter, which defaults to `5000` (5GB).+If it is absolutely necessary to control the number of segments present after optimize, specify `maxSegments` as a positive integer. Setting `maxSegments` higher than `1` are honored on a "best effort" basis.+@@ -388,9 +391,9 @@ When upgrading to Solr 7.4, users should be aware of the following major changes* Configuring `slowQueryThresholdMillis` now logs slow requests to a separate file named `solr_slow_requests.log`. Previously they would get logged in the `solr.log` file.-*Legacy Scaling (non-SolrCloud)*+*User-Managed Clusters*-* In the <<index-replication.adoc#,leader-follower model>> of scaling Solr, a follower no longer commits an empty index when a completely new index is detected on leader during replication. To return to the previous behavior pass `false` to `skipCommitOnLeaderVersionZero` in the follower section of replication handler configuration, or pass it to the `fetchindex` command.+* In the <<user-managed-index-replication.adoc#,leader-follower model>> of scaling Solr, a follower no longer commits an empty index when a completely new index is detected on leader during replication. To return to the previous behavior pass `false` to `skipCommitOnLeaderVersionZero` in the follower section of replication handler configuration, or pass it to the `fetchindex` command.If you are upgrading from a version earlier than Solr 7.3, please see previous version notes below.@@ -444,9 +447,9 @@ See the https://cwiki.apache.org/confluence/display/solr/ReleaseNote72[7.2 ReleaWhen upgrading to Solr 7.2, users should be aware of the following major changes from v7.1:-*Local Parameters*+*Local Params*-* Starting a query string with <<local-parameters-in-queries.adoc#,local parameters>> `{!myparser ...}` is used to switch from one query parser to another, and is intended for use by Solr system developers, not end users doing searches. To reduce negative side-effects of unintended hack-ability, Solr now limits the cases when local parameters will be parsed to only contexts in which the default parser is "<<other-parsers.adoc#lucene-query-parser,lucene>>" or "<<other-parsers.adoc#function-query-parser,func>>".+* Starting a query string with <<local-params.adoc#,local params>> `{!myparser ...}` is used to switch from one query parser to another, and is intended for use by Solr system developers, not end users doing searches. To reduce negative side-effects of unintended hack-ability, Solr now limits the cases when local params will be parsed to only contexts in which the default parser is "<<standard-query-parser.adoc#,lucene>>" or "<<other-parsers.adoc#function-query-parser,func>>".+So, if `defType=edismax` then `q={!myparser ...}` won't work. In that example, put the desired query parser into the `defType` parameter.+@@ -456,7 +459,7 @@ If you must have full backwards compatibility, use `luceneMatchVersion=7.1.0` or*eDisMax Query Parser*-* The eDisMax parser by default no longer allows subqueries that specify a Solr parser using either local parameters, or the older `\_query_` magic field trick.+* The eDisMax parser by default no longer allows subqueries that specify a Solr parser using either local params, or the older `\_query_` magic field trick.+For example, `{!prefix f=myfield v=enterp}` or `\_query_:"{!prefix f=myfield v=enterp}"` are not supported by default any longer. If you want to allow power-users to do this, set `uf=* _query_` or some other value that includes `\_query_`.+diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex 53632f7ba86..ae0caeba754 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -170,7 +170,7 @@ Upgrade is to be done using Collection API MIGRATESTATEFORMAT action using a preSee for example https://solr.apache.org/guide/8_8/cluster-node-management.html#migratestateforma[Solr 8.8 Ref Guide].// Can't link directly to .adoc file, need to link to 8.something ref guide as MIGRATESTATEFORMAT no longer exists in 9.0.-* If you're using Solr in standalone mode with the <<the-query-elevation-component.adoc#,Query Elevation Component>> with it's elevation file in the data directory, you'll have to move it to the <<config-sets.adoc#,Configset>> instead.+* If you're using Solr in standalone mode with the <<query-elevation-component.adoc#,Query Elevation Component>> with it's elevation file in the data directory, you'll have to move it to the <<config-sets.adoc#,Configset>> instead.The only reason QEC supported the data directory was to support loading its changes on a commit instead of a more expensive core reload.That feature now works from the configset dir too.SolrCloud doesn't support that but may sometime.@@ -189,13 +189,13 @@ If re-indexing is too onerous, use this mechanism in Solr 8.=== solr.xml maxBooleanClauses now enforced recursively-Lucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global `<<format-of-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is enforced.+Lucene 9.0 has additional safety checks over previous versions that impact how the `solr.xml` global `<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` option is enforced.In previous versios of Solr, this option was a hard limit on the number of clauses in any `BooleanQuery` object - but it was only enforced for the _direct_ clauses. Starting with Solr 9, this global limit is now also enforced against the total number of clauses in a _nested_ query structure.Users who upgrade from prior versions of Solr may find that some requests involving complex internal query structures (Example: long query strings using `edismax` with many `qf` and `pf` fields that include query time synonym expansion) which worked in the past now hit this limit and fail.-User's in this situation are advised to consider the complexity f their queries/configuration, and increase the value of `<<format-of-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` if warranted.+User's in this situation are advised to consider the complexity f their queries/configuration, and increase the value of `<<configuring-solr-xml#global-maxbooleanclauses,maxBooleanClauses>>` if warranted.=== base_url removed from stored state@@ -220,4 +220,4 @@ What percentage of them get reported to a tracing server is up to you.* BasicAuthPlugin property 'blockUnknown' now defaults to 'true'. This change is backward incompatible. If you need the pre-9.0 default behavior, you need to explicitly set 'blockUnknown':'false' in security.json.-* The allow-list defining allowed URLs for the `shards` parameter is not in the `shardHandler` configuration anymore. It is defined by the `allowUrls` top-level property of the `solr.xml` file. For more information, see <<format-of-solr-xml.adoc#_allow_urls, Format of solr.allowUrls>> documentation.+* The allow-list defining allowed URLs for the `shards` parameter is not in the `shardHandler` configuration anymore. It is defined by the `allowUrls` top-level property of the `solr.xml` file. For more information, see <<configuring-solr-xml.adoc#allow-urls, Format of solr.allowUrls>> documentation.diff --git a/solr/solr-ref-guide/src/managed-resources.adoc b/solr/solr-ref-guide/src/managed-resources.adocindex 0bc186a6d46..f35f1608a25 100644--- a/solr/solr-ref-guide/src/managed-resources.adoc+++ b/solr/solr-ref-guide/src/managed-resources.adoc@@ -18,9 +18,11 @@Managed resources expose a REST API endpoint for performing Create-Read-Update-Delete (CRUD) operations on a Solr object.-Any long-lived Solr object that has configuration settings and/or data is a good candidate to be a managed resource. Managed resources complement other programmatically manageable components in Solr, such as the RESTful schema API to add fields to a managed schema.+Any long-lived Solr object that has configuration settings and/or data is a good candidate to be a managed resource.+Managed resources complement other programmatically manageable components in Solr, such as the RESTful schema API to add fields to a managed schema.-Consider a Web-based UI that offers Solr-as-a-Service where users need to configure a set of stop words and synonym mappings as part of an initial setup process for their search application. This type of use case can easily be supported using the Managed Stop Filter & Managed Synonym Graph Filter Factories provided by Solr, via the Managed resources REST API.+Consider a Web-based UI that offers Solr-as-a-Service where users need to configure a set of stop words and synonym mappings as part of an initial setup process for their search application.+This type of use case can easily be supported using the Managed Stop Filter & Managed Synonym Graph Filter Factories provided by Solr, via the Managed resources REST API.Users can also write their own custom plugins, that leverage the same internal hooks to make additional resources REST managed.@@ -33,11 +35,12 @@ bin/solr -e techproducts== Managed Resources Overview-Let's begin learning about managed resources by looking at a couple of examples provided by Solr for managing stop words and synonyms using a REST API. After reading this section, you'll be ready to dig into the details of how managed resources are implemented in Solr so you can start building your own implementation.+Let's begin learning about managed resources by looking at a couple of examples provided by Solr for managing stop words and synonyms using a REST API.+After reading this section, you'll be ready to dig into the details of how managed resources are implemented in Solr so you can start building your own implementation.=== Managing Stop Words-To begin, you need to define a field type that uses the <<filter-descriptions.adoc#managed-stop-filter,ManagedStopFilterFactory>>, such as:+To begin, you need to define a field type that uses the <<filters.adoc#managed-stop-filter,ManagedStopFilterFactory>>, such as:[source,xml,subs="verbatim,callouts"]----@@ -52,13 +55,14 @@ To begin, you need to define a field type that uses the <<filter-descriptions.adThere are two important things to notice about this field type definition:-<1> The filter implementation class is `solr.ManagedStopFilterFactory`. This is a special implementation of the <<filter-descriptions.adoc#stop-filter,StopFilterFactory>> that uses a set of stop words that are managed from a REST API.+<1> The filter implementation class is `solr.ManagedStopFilterFactory`. This is a special implementation of the <<filters.adoc#stop-filter,StopFilterFactory>> that uses a set of stop words that are managed from a REST API.<2> The `managed=”english”` attribute gives a name to the set of managed stop words, in this case indicating the stop words are for English text.The REST endpoint for managing the English stop words in the techproducts collection is: `/solr/techproducts/schema/analysis/stopwords/english`.-The example resource path should be mostly self-explanatory. It should be noted that the ManagedStopFilterFactory implementation determines the `/schema/analysis/stopwords` part of the path, which makes sense because this is an analysis component defined by the schema.+The example resource path should be mostly self-explanatory.+It should be noted that the ManagedStopFilterFactory implementation determines the `/schema/analysis/stopwords` part of the path, which makes sense because this is an analysis component defined by the schema.It follows that a field type that uses the following filter:@@ -101,7 +105,8 @@ Assuming you sent this request to Solr, the response body is a JSON document:The `sample_techproducts_configs` <<config-sets.adoc#,configset>> ships with a pre-built set of managed stop words, however you should only interact with this file using the API and not edit it directly.-One thing that should stand out to you in this response is that it contains a `managedList` of words as well as `initArgs`. This is an important concept in this framework -- managed resources typically have configuration and data. For stop words, the only configuration parameter is a boolean that determines whether to ignore the case of tokens during stop word filtering (ignoreCase=true|false). The data is a list of words, which is represented as a JSON array named `managedList` in the response.+One thing that should stand out to you in this response is that it contains a `managedList` of words as well as `initArgs`. This is an important concept in this framework -- managed resources typically have configuration and data.+For stop words, the only configuration parameter is a boolean that determines whether to ignore the case of tokens during stop word filtering (ignoreCase=true|false). The data is a list of words, which is represented as a JSON array named `managedList` in the response.Now, let’s add a new word to the English stop word list using an HTTP PUT:@@ -110,7 +115,9 @@ Now, let’s add a new word to the English stop word list using an HTTP PUT:curl -X PUT -H 'Content-type:application/json' --data-binary '["foo"]' "http://localhost:8983/solr/techproducts/schema/analysis/stopwords/english"-----Here we’re using curl to PUT a JSON list containing a single word “foo” to the managed English stop words set. Solr will return 200 if the request was successful. You can also put multiple words in a single PUT request.+Here we’re using curl to PUT a JSON list containing a single word “foo” to the managed English stop words set.+Solr will return 200 if the request was successful.+You can also put multiple words in a single PUT request.You can test to see if a specific word exists by sending a GET request for that word as a child resource of the set, such as:@@ -128,11 +135,13 @@ To delete a stop word, you would do:curl -X DELETE "http://localhost:8983/solr/techproducts/schema/analysis/stopwords/english/foo"-----NOTE: PUT/POST is used to add terms to an existing list instead of replacing the list entirely. This is because it is more common to add a term to an existing list than it is to replace a list altogether, so the API favors the more common approach of incrementally adding terms especially since deleting individual terms is also supported.+NOTE: PUT/POST is used to add terms to an existing list instead of replacing the list entirely.+This is because it is more common to add a term to an existing list than it is to replace a list altogether, so the API favors the more common approach of incrementally adding terms especially since deleting individual terms is also supported.=== Managing Synonyms-For the most part, the API for managing synonyms behaves similar to the API for stop words, except instead of working with a list of words, it uses a map, where the value for each entry in the map is a set of synonyms for a term. As with stop words, the `sample_techproducts_configs` <<config-sets.adoc#,configset>> includes a pre-built set of synonym mappings suitable for the sample data that is activated by the following field type definition in `schema.xml`:+For the most part, the API for managing synonyms behaves similar to the API for stop words, except instead of working with a list of words, it uses a map, where the value for each entry in the map is a set of synonyms for a term.+As with stop words, the `sample_techproducts_configs` <<config-sets.adoc#,configset>> includes a pre-built set of synonym mappings suitable for the sample data that is activated by the following field type definition in `schema.xml`:[source,xml]----@@ -191,16 +200,19 @@ To add a new synonym mapping, you can PUT/POST a single mapping such as:curl -X PUT -H 'Content-type:application/json' --data-binary '{"mad":["angry","upset"]}' "http://localhost:8983/solr/techproducts/schema/analysis/synonyms/english"-----The API will return status code 200 if the PUT request was successful. To determine the synonyms for a specific term, you send a GET request for the child resource, such as `/schema/analysis/synonyms/english/mad` would return `["angry","upset"]`.+The API will return status code 200 if the PUT request was successful.+To determine the synonyms for a specific term, you send a GET request for the child resource, such as `/schema/analysis/synonyms/english/mad` would return `["angry","upset"]`.-You can also PUT a list of symmetric synonyms, which will be expanded into a mapping for each term in the list. For example, you could PUT the following list of symmetric synonyms using the JSON list syntax instead of a map:+You can also PUT a list of symmetric synonyms, which will be expanded into a mapping for each term in the list.+For example, you could PUT the following list of symmetric synonyms using the JSON list syntax instead of a map:[source,bash]----curl -X PUT -H 'Content-type:application/json' --data-binary '["funny", "entertaining", "whimiscal", "jocular"]' "http://localhost:8983/solr/techproducts/schema/analysis/synonyms/english"-----Note that the expansion is performed when processing the PUT request so the underlying persistent state is still a managed map. Consequently, if after sending the previous PUT request, you did a GET for `/schema/analysis/synonyms/english/jocular`, then you would receive a list containing `["funny", "entertaining", "whimiscal"]`. Once you've created synonym mappings using a list, each term must be managed separately.+Note that the expansion is performed when processing the PUT request so the underlying persistent state is still a managed map.+Consequently, if after sending the previous PUT request, you did a GET for `/schema/analysis/synonyms/english/jocular`, then you would receive a list containing `["funny", "entertaining", "whimiscal"]`. Once you've created synonym mappings using a list, each term must be managed separately.Lastly, you can delete a mapping by sending a DELETE request to the managed endpoint.@@ -210,15 +222,19 @@ Changes made to managed resources via this REST API are not applied to the activFor example: after adding or deleting a stop word, you must reload the core/collection before changes become active; related APIs: <<coreadmin-api.adoc#,CoreAdmin API>> and <<collections-api.adoc#,Collections API>>.-This approach is required when running in distributed mode so that we are assured changes are applied to all cores in a collection at the same time so that behavior is consistent and predictable. It goes without saying that you don’t want one of your replicas working with a different set of stop words or synonyms than the others.+This approach is required when running in distributed mode so that we are assured changes are applied to all cores in a collection at the same time so that behavior is consistent and predictable.+It goes without saying that you don’t want one of your replicas working with a different set of stop words or synonyms than the others.-One subtle outcome of this _apply-changes-at-reload_ approach is that the once you make changes with the API, there is no way to read the active data. In other words, the API returns the most up-to-date data from an API perspective, which could be different than what is currently being used by Solr components.+One subtle outcome of this _apply-changes-at-reload_ approach is that the once you make changes with the API, there is no way to read the active data.+In other words, the API returns the most up-to-date data from an API perspective, which could be different than what is currently being used by Solr components.However, the intent of this API implementation is that changes will be applied using a reload within a short time frame after making them so the time in which the data returned by the API differs from what is active in the server is intended to be negligible.[IMPORTANT]====-Changing things like stop words and synonym mappings typically require reindexing existing documents if being used by index-time analyzers. The RestManager framework does not guard you from this, it simply makes it possible to programmatically build up a set of stop words, synonyms, etc. See the section <<reindexing.adoc#,Reindexing>> for more information about reindexing your documents.+Changing things like stop words and synonym mappings typically require reindexing existing documents if being used by index-time analyzers.+The RestManager framework does not guard you from this, it simply makes it possible to programmatically build up a set of stop words, synonyms, etc.+See the section <<reindexing.adoc#,Reindexing>> for more information about reindexing your documents.====== RestManager Endpoint@@ -258,7 +274,8 @@ The response body is a JSON document containing metadata about managed resourcesYou can also create new managed resource using PUT/POST to the appropriate URL – before ever configuring anything that uses these resources.-For example, imagine we want to build up a set of German stop words. Before we can start adding stop words, we need to create the endpoint:+For example, imagine we want to build up a set of German stop words.+Before we can start adding stop words, we need to create the endpoint:`/solr/techproducts/schema/analysis/stopwords/german`@@ -271,7 +288,9 @@ curl -X PUT -H 'Content-type:application/json' --data-binary \"http://localhost:8983/solr/techproducts/schema/analysis/stopwords/german"-----Solr will respond with status code 200 if the request is successful. Effectively, this action registers a new endpoint for a managed resource in the RestManager. From here you can start adding German stop words as we saw above:+Solr will respond with status code 200 if the request is successful.+Effectively, this action registers a new endpoint for a managed resource in the RestManager.+From here you can start adding German stop words as we saw above:[source,bash]----diff --git a/solr/solr-ref-guide/src/math-expressions.adoc b/solr/solr-ref-guide/src/math-expressions.adocindex 9e2b0c180ef..f78f15fe931 100644--- a/solr/solr-ref-guide/src/math-expressions.adoc+++ b/solr/solr-ref-guide/src/math-expressions.adoc@@ -1,7 +1,27 @@= Streaming Expressions and Math Expressions-:page-children: visualization, math-start, loading, search-sample, transform, scalar-math, vector-math, variables, matrix-math, term-vectors, probability-distributions, statistics, regression, curve-fitting, time-series, numerical-analysis, dsp, simulations, machine-learning, graph, computational-geometry, logs+:page-children: visualization, \+ math-start, \+ loading, \+ search-sample, \+ transform, \+ scalar-math, \+ vector-math, \+ variables, \+ matrix-math, \+ term-vectors, \+ probability-distributions, \+ statistics, \+ regression, \+ curve-fitting, \+ time-series, \+ numerical-analysis, \+ dsp, \+ simulations, \+ machine-learning, \+ graph, \+ computational-geometry, \+ logs:page-show-toc: false-// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -59,7 +79,7 @@ image::images/math-expressions/searchiris.png[]*<<dsp.adoc#,Signal Processing>>*: Convolution, cross-correlation, autocorrelation and fast Fourier transforms.-*<<simulations.adoc#,Simulations>>*: Monte Carlo simulations and random walks+*<<simulations.adoc#,Simulations>>*: Monte Carlo simulations and random walks.*<<machine-learning.adoc#,Machine Learning>>*: Distance, KNN, DBSCAN, K-means, fuzzy K-means and other ML functions.diff --git a/solr/solr-ref-guide/src/math-start.adoc b/solr/solr-ref-guide/src/math-start.adocindex 63879babf3c..35517764073 100644--- a/solr/solr-ref-guide/src/math-start.adoc+++ b/solr/solr-ref-guide/src/math-start.adoc@@ -18,17 +18,16 @@== Language-*Streaming expressions* and *math expressions* are function languages that run-inside SolrCloud. The languages consist of functions-that are designed to be *composed* to form programming logic.+*Streaming expressions* and *math expressions* are function languages that run inside SolrCloud.+The languages consist of functions that are designed to be *composed* to form programming logic.-*Streaming expressions* are functions that return streams of tuples. Streaming expression functions can be composed to form a transformation pipeline.+*Streaming expressions* are functions that return streams of tuples.+Streaming expression functions can be composed to form a transformation pipeline.The pipeline starts with a *stream source*, such as `search`, which initiates a stream of tuples.One or more *stream decorators*, such as `select`, wraps the stream source and transforms the stream of tuples.-*Math expressions* are functions that operate over and return primitives and in-memory-arrays and matrices. The core use case for math expressions is performing mathematical operations and-visualization.+*Math expressions* are functions that operate over and return primitives and in-memory arrays and matrices.+The core use case for math expressions is performing mathematical operations and visualization.Streaming expressions and math expressions can be combined to *search,sample, aggregate, transform, analyze* and *visualize* data in SolrCloud collections.@@ -37,13 +36,11 @@ sample, aggregate, transform, analyze* and *visualize* data in SolrCloud collect== ExecutionSolr's `/stream` request handler executes streaming expressions and math expressions.-This handler compiles the expression, runs the expression logic-and returns a JSON result.+This handler compiles the expression, runs the expression logic and returns a JSON result.=== Admin UI Stream Panel-The easiest way to run streaming expressions and math expressions is through-the *stream* panel on the Solr Admin UI.+The easiest way to run streaming expressions and math expressions is through the <<stream-screen.adoc#,Stream Screen>> in the Solr Admin UI.A sample `search` streaming expression is shown in the screenshot below:@@ -55,12 +52,10 @@ image::images/math-expressions/add.png[]=== Curl Example-The HTTP interface to the `/stream` handler can be used to-send a streaming expression request and retrieve the response.+The HTTP interface to the `/stream` handler can be used to send a streaming expression request and retrieve the response.-Curl is a useful tool for running streaming expressions when the result-needs to be spooled to disk or is too large for the Solr admin stream panel. Below-is an example of a curl command to the `/stream` handler.+Curl is a useful tool for running streaming expressions when the result needs to be spooled to disk or is too large for the Solr admin stream panel.+Below is an example of a curl command to the `/stream` handler.[source,bash]----@@ -96,8 +91,7 @@ The JSON response from the stream handler for this request is shown below:== Visualization-The visualizations in this guide were performed with Apache Zeppelin using the-Zeppelin-Solr interpreter.+The visualizations in this guide were performed with Apache Zeppelin using the Zeppelin-Solr interpreter.=== Zeppelin-Solr Interpreter@@ -110,19 +104,15 @@ The screenshot below shows the panel for configuring Zeppelin-Solr.image::images/math-expressions/zepconf.png[]-Configure the `solr.baseUrl` and `solr.collection` to point to the location where the streaming-expressions and math expressions will be sent for execution. The `solr.collection` is-just the execution collection and does not need to hold data, although it can hold data.-streaming expressions can choose to query any of the collections that are attached-to the same SolrCloud as the execution collection.+Configure the `solr.baseUrl` and `solr.collection` to point to the location where the streaming expressions and math expressions will be sent for execution.+The `solr.collection` is just the execution collection and does not need to hold data, although it can hold data.+streaming expressions can choose to query any of the collections that are attached to the same SolrCloud as the execution collection.=== zplotStreaming expression result sets can be visualized automatically by Zeppelin-Solr.Math expression results need to be formatted for visualization using the `zplot` function.-This function has support for plotting *vectors*, *matrices*, *probability distributions* and-*2D clustering results*.+This function has support for plotting *vectors*, *matrices*, *probability distributions* and *2D clustering results*.-There are many examples in the guide which show how to visualize both streaming expressions-and math expressions.+There are many examples in the guide which show how to visualize both streaming expressions and math expressions.diff --git a/solr/solr-ref-guide/src/matrix-math.adoc b/solr/solr-ref-guide/src/matrix-math.adocindex fd9ec3f5130..dd60b54d214 100644--- a/solr/solr-ref-guide/src/matrix-math.adoc+++ b/solr/solr-ref-guide/src/matrix-math.adoc@@ -64,12 +64,14 @@ responds with:== Row and Column Labels-A matrix can have column and rows and labels. The functions+A matrix can have column and rows and labels.+The functions`setRowLabels`, `setColumnLabels`, `getRowLabels`, and `getColumnLabels`can be used to set and get the labels.The label values are set using string arrays.-The example below sets the row and column labels. In other sections of the+The example below sets the row and column labels.+In other sections of theuser guide examples are shown where functions return matriceswith the labels already set.@@ -172,7 +174,8 @@ responds with:== Matrix AttributesA matrix can also have an arbitrary set of named attributes associated-with it. Certain functions, such as the `termVectors` function,+with it.+Certain functions, such as the `termVectors` function,return matrices that contain attributes that describe data in the matrix.Attributes can be retrieved by name using the `getAttribute` function anddiff --git a/solr/solr-ref-guide/src/mbean-request-handler.adoc b/solr/solr-ref-guide/src/mbean-request-handler.adocindex 019f7587b33..16cc14af771 100644--- a/solr/solr-ref-guide/src/mbean-request-handler.adoc+++ b/solr/solr-ref-guide/src/mbean-request-handler.adoc@@ -27,20 +27,17 @@ Restricts results by object key.Restricts results by category name.`stats`::-Specifies whether statistics are returned with results. You can override the `stats` parameter on a per-field basis. The default is `false`.+Specifies whether statistics are returned with results.+You can override the `stats` parameter on a per-field basis.+The default is `false`.`wt`::-The output format. This operates the same as the <<response-writers.adoc#,`wt` parameter in a query>>. The default is `json`.+The output format.+This operates the same as the <<response-writers.adoc#,`wt` parameter in a query>>. The default is `json`.== MBeanRequestHandler Examples-// TODO 7.1 - replace with link to tutorial-All of the examples in this section assume you are running the "techproducts" Solr example:--[source,bash]------bin/solr -e techproducts-----+All of the examples in this section assume you are running the <<tutorial-techproducts.adoc#,"techproducts" example>>.To return information about the CACHE category only:diff --git a/solr/solr-ref-guide/src/merging-indexes.adoc b/solr/solr-ref-guide/src/merging-indexes.adocdeleted file mode 100644index 740ee333fc8..00000000000--- a/solr/solr-ref-guide/src/merging-indexes.adoc+++ /dev/null@@ -1,47 +0,0 @@-= Merging Indexes-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--If you need to combine indexes from two different projects or from multiple servers previously used in a distributed configuration, you can use either the IndexMergeTool included in `lucene-misc` or the `CoreAdminHandler`.--To merge indexes, they must meet these requirements:--* The two indexes must be compatible: their schemas should include the same fields and they should analyze fields the same way.-* The indexes must not include duplicate data.--Optimally, the two indexes should be built using the same schema.--== Using IndexMergeTool--To merge the indexes, do the following:--. Make sure that both indexes you want to merge are closed.-. Issue this command:-+-[source,bash]------java -cp $SOLR/server/solr-webapp/webapp/WEB-INF/lib/lucene-core-VERSION.jar:$SOLR/server/solr-webapp/webapp/WEB-INF/lib/lucene-misc-VERSION.jar org/apache/lucene/misc/IndexMergeTool /path/to/newindex /path/to/old/index1 /path/to/old/index2------+-This will create a new index at `/path/to/newindex` that contains both index1 and index2.-. Copy this new directory to the location of your application's Solr index (move the old one aside first, of course) and start Solr.--== Using CoreAdmin--The `MERGEINDEXES` command of the <<coreadmin-api.adoc#coreadmin-mergeindexes,CoreAdminHandler>> can be used to merge indexes into a new core – either from one or more arbitrary `indexDir` directories or by merging from one or more existing `srcCore` core names.--See the <<coreadmin-api.adoc#coreadmin-mergeindexes,CoreAdminHandler>> section for details.diff --git a/solr/solr-ref-guide/src/metrics-reporting.adoc b/solr/solr-ref-guide/src/metrics-reporting.adocindex 6fd1fe4c194..a8c6dc6881d 100644--- a/solr/solr-ref-guide/src/metrics-reporting.adoc+++ b/solr/solr-ref-guide/src/metrics-reporting.adoc@@ -20,10 +20,14 @@ Solr includes a developer API and instrumentation for the collection of detailedInternally this feature uses the http://metrics.dropwizard.io[Dropwizard Metrics API], which uses the following classes of meters to measure events:-* *counters* - simply count events. They provide a single long value, e.g., the number of requests.-* *meters* - additionally compute rates of events. Provide a count (as above) and 1-, 5-, and 15-minute exponentially decaying rates, similar to the Unix system load average.-* *histograms* - calculate approximate distribution of events according to their values. Provide the following approximate statistics, with a similar exponential decay as above: mean (arithmetic average), median, maximum, minimum, standard deviation, and 75^th^, 95^th^, 98^th^, 99^th^ and 999^th^ percentiles.-* *timers* - measure the number and duration of events. They provide a count and histogram of timings.+* *counters* - simply count events.+They provide a single long value, e.g., the number of requests.+* *meters* - additionally compute rates of events.+Provide a count (as above) and 1-, 5-, and 15-minute exponentially decaying rates, similar to the Unix system load average.+* *histograms* - calculate approximate distribution of events according to their values.+Provide the following approximate statistics, with a similar exponential decay as above: mean (arithmetic average), median, maximum, minimum, standard deviation, and 75^th^, 95^th^, 98^th^, 99^th^ and 999^th^ percentiles.+* *timers* - measure the number and duration of events.+They provide a count and histogram of timings.* *gauges* - offer instantaneous reading of a current value, e.g., current queue depth, current number of active connections, free heap size.Some of these meters may be missing or empty for any number of valid reasons.@@ -32,7 +36,8 @@ This is configurable for several types of missing values; see the <<The <metricsEach group of related metrics with unique names is managed in a *metric registry*. Solr maintains several such registries, each corresponding to a high-level group such as: `jvm`, `jetty`, `node`, and `core` (see <<Metric Registries>> below).-For each group (and/or for each registry) there can be several *reporters*, which are components responsible for communication of metrics from selected registries to external systems. Currently implemented reporters support emitting metrics via JMX, Ganglia, Graphite and SLF4J.+For each group (and/or for each registry) there can be several *reporters*, which are components responsible for communication of metrics from selected registries to external systems.+Currently implemented reporters support emitting metrics via JMX, Ganglia, Graphite and SLF4J.There is also a dedicated `/admin/metrics` handler that can be queried to report all or a subset of the current metrics from multiple registries.@@ -40,13 +45,15 @@ There is also a dedicated `/admin/metrics` handler that can be queried to reportSolr includes multiple metric registries, which group related metrics.-Metrics are maintained and accumulated through all lifecycles of components from the start of the process until its shutdown - e.g., metrics for a particular SolrCore are tracked through possibly several load, unload and/or rename operations, and are deleted only when a core is explicitly deleted. However, metrics are not persisted across process restarts; restarting Solr will discard all collected metrics.+Metrics are maintained and accumulated through all lifecycles of components from the start of the process until its shutdown - e.g., metrics for a particular SolrCore are tracked through possibly several load, unload and/or rename operations, and are deleted only when a core is explicitly deleted.+However, metrics are not persisted across process restarts; restarting Solr will discard all collected metrics.These are the major groups of metrics that are collected:=== JVM Registry-This registry is returned at `solr.jvm` and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=jvm` to limit to only these metrics.+This registry is returned at `solr.jvm` and includes the following information.+When making requests with the <<Metrics API>>, you can specify `&group=jvm` to limit to only these metrics.* direct and mapped buffer pools* class loading / unloading@@ -54,36 +61,39 @@ This registry is returned at `solr.jvm` and includes the following information.* GC count and time* heap, non-heap memory and GC pools* number of threads, their states and deadlocks-* System properties such as Java information, various installation directory paths, ports, and similar information. You can control what appears here by modifying `solr.xml`.-// TODO for 7.0 fix this+* System properties such as Java information, various installation directory paths, ports, and similar information.+You can control what appears here by modifying `solr.xml`.=== Overseer Registry-This registry is returned at `solr.overseer` when run in SolrCloud mode and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=overseer` to limit to only these metrics.+This registry is returned at `solr.overseer` when run in SolrCloud mode and includes the following information.+When making requests with the <<Metrics API>>, you can specify `&group=overseer` to limit to only these metrics.* size of the Overseer queues (collection work queue and cluster state update queue)=== Node / CoreContainer Registry-This registry is returned at `solr.node` and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=node` to limit to only these metrics.+This registry is returned at `solr.node` and includes the following information.+When making requests with the <<Metrics API>>, you can specify `&group=node` to limit to only these metrics.* handler requests (count, timing): collections, info, admin, configsets, etc.* number of cores (loaded, lazy, unloaded)=== Core (SolrCore) Registry-The <<Core Level Metrics,Core (SolrCore) Registry>> includes `solr.core.<collection>`, one for each core. When making requests with the <<Metrics API>>, you can specify `&group=core` to limit to only these metrics.+The <<Core Level Metrics,Core (SolrCore) Registry>> includes `solr.core.<collection>`, one for each core.+When making requests with the <<Metrics API>>, you can specify `&group=core` to limit to only these metrics.-* all common RequestHandlers report: request timers / counters, timeouts, errors. Handlers that support- process distributed shard requests also report `shardRequests` sub-counters for each type of distributed- request.+* all common RequestHandlers report: request timers / counters, timeouts, errors.+Handlers that support process distributed shard requests also report `shardRequests` sub-counters for each type of distributed request.* <<Index Merge Metrics,index-level events>>: meters for minor / major merges, number of merged docs, number of deleted docs, gauges for currently running merges and their size.* shard replication and transaction log replay on replicas,* open / available / pending connections for shard handler and update handler.=== Jetty Registry-This registry is returned at `solr.jetty` and includes the following information. When making requests with the <<Metrics API>>, you can specify `&group=jetty` to limit to only these metrics.+This registry is returned at `solr.jetty` and includes the following information.+When making requests with the <<Metrics API>>, you can specify `&group=jetty` to limit to only these metrics.* threads and pools,* connection and request timers,@@ -93,14 +103,15 @@ This registry is returned at `solr.jetty` and includes the following informationThe metrics available in your system can be customized by modifying the `<metrics>` element in `solr.xml`.-TIP: See also the section <<format-of-solr-xml.adoc#,Format of Solr.xml>> for more information about the `solr.xml` file, where to find it, and how to edit it.+TIP: See also the section <<configuring-solr-xml.adoc#,Format of Solr.xml>> for more information about the `solr.xml` file, where to find it, and how to edit it.=== Disabling the Metrics CollectionThe `<metrics>` element in `solr.xml` supports one attribute `enabled`, which takes a boolean value,for example `<metrics enabled="true">`.The default value of this attribute is `true`, meaning that metrics are being collected, processed and-reported by Solr according to the configured metric reporters. They are also available from the+reported by Solr according to the configured metric reporters.+They are also available from themetrics APIs.The `false` value of this attribute (`<metrics enabled="false">`) turns off metrics collection and processing.@@ -130,28 +141,40 @@ If this section is not defined, the following default configuration is used whic=== The <metrics> <reporters> Element-Reporters consume the metrics data generated by Solr. See the section <<Reporters>> below for more details on how to configure custom reporters.+Reporters consume the metrics data generated by Solr.+See the section <<Reporters>> below for more details on how to configure custom reporters.=== The <metrics> <suppliers> Element-Suppliers help Solr generate metrics data. The `<metrics><suppliers>` section of `solr.xml` allows you to define your own implementations of metrics and configure parameters for them.+Suppliers help Solr generate metrics data.+The `<metrics><suppliers>` section of `solr.xml` allows you to define your own implementations of metrics and configure parameters for them.Implementation of a custom metrics supplier is beyond the scope of this guide, but there are other customizations possible with the default implementation, via the elements described below.-<counter>:: This element defines the implementation and configuration of a `Counter` supplier. The default implementation does not support any configuration.+<counter>:: This element defines the implementation and configuration of a `Counter` supplier.+The default implementation does not support any configuration.-<meter>:: This element defines the implementation of a `Meter` supplier. The default implementation supports an additional parameter:-`<str name="clock">`::: The type of clock to use for calculating EWMA rates. The supported values are:+<meter>:: This element defines the implementation of a `Meter` supplier.+The default implementation supports an additional parameter:+`<str name="clock">`::: The type of clock to use for calculating EWMA rates.+The supported values are:* `user`, the default, which uses `System.nanoTime()`* `cpu`, which uses the current thread's CPU time-<histogram>:: This element defines the implementation of a `Histogram` supplier. This element also supports the `clock` parameter shown above with the `meter` element, and also:-`<str name="reservoir">`::: The fully-qualified class name of the `Reservoir` implementation to use. The default is `com.codahale.metrics.ExponentiallyDecayingReservoir` but there are other options available with the http://metrics.dropwizard.io/{ivy-dropwizard-version}/manual/core.html#histograms[Codahale Metrics library] that Solr uses. The following parameters are supported, within the mentioned limitations:-* `size`, the reservoir size. The default is 1028.-* `alpha`, the decay parameter. The default is 0.015. This is only valid for the `ExponentiallyDecayingReservoir`.+<histogram>:: This element defines the implementation of a `Histogram` supplier.+This element also supports the `clock` parameter shown above with the `meter` element, and also:+`<str name="reservoir">`::: The fully-qualified class name of the `Reservoir` implementation to use.+The default is `com.codahale.metrics.ExponentiallyDecayingReservoir` but there are other options available with the http://metrics.dropwizard.io/{ivy-dropwizard-version}/manual/core.html#histograms[Codahale Metrics library] that Solr uses.+The following parameters are supported, within the mentioned limitations:+* `size`, the reservoir size.+The default is `1028`.+* `alpha`, the decay parameter.+The default is `0.015`.+This is only valid for the `ExponentiallyDecayingReservoir`.* `window`, the window size, in seconds, and only valid for the `SlidingTimeWindowReservoir`. The default is 300 (5 minutes).-<timer>:: This element defines an implementation of a `Timer` supplier. The default implementation supports the `clock` and `reservoir` parameters described above.+<timer>:: This element defines an implementation of a `Timer` supplier.+The default implementation supports the `clock` and `reservoir` parameters described above.As an example of a section of `solr.xml` that defines some of these custom parameters, the following defines the default `Meter` supplier with a non-default `clock` and the default `Timer` is used with a non-default reservoir:@@ -256,18 +279,24 @@ One or more of the predefined groups (see above).`registry`::One or more of valid fully-qualified registry names.-If both `group` and `registry` attributes are specified only the `group` attribute is considered. If neither attribute is specified then the plugin will be used for all groups and registries. Multiple group or registry names can be specified, separated by comma and/or space.+If both `group` and `registry` attributes are specified only the `group` attribute is considered.+If neither attribute is specified then the plugin will be used for all groups and registries.+Multiple group or registry names can be specified, separated by comma and/or space.-Additionally, several implementation-specific initialization arguments can be specified in nested elements. There are some arguments that are common to SLF4J, Ganglia and Graphite reporters:+Additionally, several implementation-specific initialization arguments can be specified in nested elements.+There are some arguments that are common to SLF4J, Ganglia and Graphite reporters:`period`::-The period in seconds between reports. Default value is `60`.+The period in seconds between reports.+Default value is `60`.`prefix`::-A prefix to be added to metric names, which may be helpful in logical grouping of related Solr instances, e.g., machine name or cluster name. Default is empty string, i.e., just the registry name and metric name will be used to form a fully-qualified metric name.+A prefix to be added to metric names, which may be helpful in logical grouping of related Solr instances, e.g., machine name or cluster name.+Default is empty string, i.e., just the registry name and metric name will be used to form a fully-qualified metric name.`filter`::-If not empty then only metric names that start with this value will be reported. Default is no filtering, i.e., all metrics from the selected registry will be reported.+If not empty then only metric names that start with this value will be reported.+Default is no filtering, i.e., all metrics from the selected registry will be reported.Reporters are instantiated for every group and registry that they were configured for, at the time when the respective components are initialized (e.g., on JVM startup or SolrCore load).@@ -275,7 +304,8 @@ When reporters are created their configuration is validated (and e.g., necessaryReporters are closed when the corresponding component is being closed (e.g., on SolrCore close, or JVM shutdown) but metrics that they reported are still maintained in respective registries, as explained in the previous section.-The following sections provide information on implementation-specific arguments. All implementation classes provided with Solr can be found under `org.apache.solr.metrics.reporters`.+The following sections provide information on implementation-specific arguments.+All implementation classes provided with Solr can be found under `org.apache.solr.metrics.reporters`.=== JMX Reporter@@ -284,24 +314,31 @@ The JMX Reporter uses the `org.apache.solr.metrics.reporters.SolrJmxReporter` clIt takes the following arguments:`domain`::-The JMX domain name. If not specified then the registry name will be used.+The JMX domain name.+If not specified then the registry name will be used.`serviceUrl`::-The service URL for a JMX server. If not specified, Solr will attempt to discover if the JVM has an MBean server and will use that address. See below for additional information on this.+The service URL for a JMX server.+If not specified, Solr will attempt to discover if the JVM has an MBean server and will use that address.+See below for additional information on this.`agentId`::-The agent ID for a JMX server. Note either `serviceUrl` or `agentId` can be specified but not both - if both are specified then the default MBean server will be used.+The agent ID for a JMX server.+Note either `serviceUrl` or `agentId` can be specified but not both - if both are specified then the default MBean server will be used.-Object names created by this reporter are hierarchical, dot-separated but also properly structured to form corresponding hierarchies in e.g., JConsole. This hierarchy consists of the following elements in the top-down order:+Object names created by this reporter are hierarchical, dot-separated but also properly structured to form corresponding hierarchies in e.g., JConsole.+This hierarchy consists of the following elements in the top-down order:* registry name (e.g., `solr.core.collection1.shard1.replica1`). Dot-separated registry names are also split into ObjectName hierarchy levels, so that metrics for this registry will be shown under `/solr/core/collection1/shard1/replica1` in JConsole, with each domain part being assigned to `dom1, dom2, ... domN` property.* reporter name (the value of reporter's `name` attribute)* category, scope and name for request handlers* or additional `name1, name2, ... nameN` elements for metrics from other components.-The JMX Reporter replaces the JMX functionality available in Solr versions before 7.0. If you have upgraded from an earlier version and have an MBean Server running when Solr starts, Solr will automatically discover the location of the local MBean server and use a default configuration for the SolrJmxReporter.+The JMX Reporter replaces the JMX functionality available in Solr versions before 7.0.+If you have upgraded from an earlier version and have an MBean Server running when Solr starts, Solr will automatically discover the location of the local MBean server and use a default configuration for the SolrJmxReporter.-You can start a local MBean server with a system property at startup by adding `-Dcom.sun.management.jmxremote` to your start command. This will not add the reporter configuration to `solr.xml`, so if you enable it with a system property, you must always start Solr with the system property or JMX will not be enabled in subsequent starts.+You can start a local MBean server with a system property at startup by adding `-Dcom.sun.management.jmxremote` to your start command.+This will not add the reporter configuration to `solr.xml`, so if you enable it with a system property, you must always start Solr with the system property or JMX will not be enabled in subsequent starts.=== SLF4J Reporter@@ -310,10 +347,12 @@ The SLF4J Reporter uses the `org.apache.solr.metrics.reporters.SolrSlf4jReporterIt takes the following arguments, in addition to common arguments described <<Reporter Arguments,above>>.`logger`::-The name of the logger to use. Default is empty, in which case the group (or the initial part of the registry name that identifies a metrics group) will be used if specified in the plugin configuration.+The name of the logger to use.+Default is empty, in which case the group (or the initial part of the registry name that identifies a metrics group) will be used if specified in the plugin configuration.Users can specify logger name (and the corresponding logger configuration in e.g., Log4j configuration) to output metrics-related logging to separate file(s), which can then be processed by external applications.-Here is an example for configuring the default `log4j2.xml` which ships in Solr. This can be used in conjunction with the `solr.xml` example provided earlier in this page to configure the SolrSlf4jReporter:+Here is an example for configuring the default `log4j2.xml` which ships in Solr.+This can be used in conjunction with the `solr.xml` example provided earlier in this page to configure the SolrSlf4jReporter:[source,xml]----@@ -402,7 +441,8 @@ The host name where Graphite server is running (required).`port`::The port number for the server (required).`pickled`::-If `true`, use "pickled" Graphite protocol which may be more efficient. Default is `false` (use plain-text protocol).+If `true`, use "pickled" Graphite protocol which may be more efficient.+Default is `false` (use plain-text protocol).When plain-text protocol is used (`pickled==false`) it's possible to use this reporter to integrate with systems other than Graphite, if they can accept space-separated and line-oriented input over network in the following format:@@ -426,7 +466,8 @@ example.solr.jetty.org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses== Core Level Metrics-These metrics are available only on a per-core basis. Metrics can be aggregated across cores using Shard and Cluster reporters.+These metrics are available only on a per-core basis.+Metrics can be aggregated across cores using Shard and Cluster reporters.=== Index Merge Metrics@@ -495,28 +536,44 @@ The `admin/metrics` endpoint provides access to all the metrics for all metric gA few query parameters are available to limit your request to only certain metrics:-`group`:: The metric group to retrieve. The default is `all` to retrieve all metrics for all groups. Other possible values are: `jvm`, `jetty`, `node`, and `core`. More than one group can be specified in a request; multiple group names should be separated by a comma.+`group`:: The metric group to retrieve.+The default is `all` to retrieve all metrics for all groups.+Other possible values are: `jvm`, `jetty`, `node`, and `core`. More than one group can be specified in a request; multiple group names should be separated by a comma.-`type`:: The type of metric to retrieve. The default is `all` to retrieve all metric types. Other possible values are `counter`, `gauge`, `histogram`, `meter`, and `timer`. More than one type can be specified in a request; multiple types should be separated by a comma.+`type`:: The type of metric to retrieve.+The default is `all` to retrieve all metric types.+Other possible values are `counter`, `gauge`, `histogram`, `meter`, and `timer`. More than one type can be specified in a request; multiple types should be separated by a comma.-`prefix`:: The first characters of metric name that will filter the metrics returned to those starting with the provided string. It can be combined with `group` and/or `type` parameters. More than one prefix can be specified in a request; multiple prefixes should be separated by a comma. Prefix matching is also case-sensitive.+`prefix`:: The first characters of metric name that will filter the metrics returned to those starting with the provided string.+It can be combined with `group` and/or `type` parameters.+More than one prefix can be specified in a request; multiple prefixes should be separated by a comma.+Prefix matching is also case-sensitive.-`regex`:: A regular expression matching metric names. Note: dot separators in metric names must be escaped, e.g.,+`regex`:: A regular expression matching metric names.+Note: dot separators in metric names must be escaped, e.g.,`QUERY\./select\..*` is a valid regex that matches all metrics with the `QUERY./select.` prefix.-`property`:: Allows requesting only this metric from any compound metric. Multiple `property` parameters can be combined to act as an OR request. For example, to only get the 99th and 999th percentile values from all metric types and groups, you can add `&property=p99_ms&property=p999_ms` to your request. This can be combined with `group`, `type`, and `prefix` as necessary.+`property`:: Allows requesting only this metric from any compound metric.+Multiple `property` parameters can be combined to act as an OR request.+For example, to only get the 99th and 999th percentile values from all metric types and groups, you can add `&property=p99_ms&property=p999_ms` to your request.+This can be combined with `group`, `type`, and `prefix` as necessary.`key`:: fully-qualified metric name, which specifies one concrete metric instance (parameter can be-specified multiple times to retrieve multiple concrete metrics). *NOTE: when this parameter is used, other-selection methods listed above are ignored.* Fully-qualified name consists of registry name, colon and-metric name, with optional colon and metric property. Colons in names can be escaped using back-slash `\`-character. Examples:+specified multiple times to retrieve multiple concrete metrics).+++Fully-qualified name consists of registry name, colon and metric name, with optional colon and metric property.+Colons in names can be escaped using backslash (`\`) character.+Examples:* `key=solr.node:CONTAINER.fs.totalSpace`* `key=solr.core.collection1:QUERY./select.requestTimes:max_ms`* `key=solr.jvm:system.properties:user.name`-`compact`:: When false, a more verbose format of the response will be returned. Instead of a response like this:+++*NOTE: when this parameter is used, other selection methods listed above are ignored.*++`compact`:: When false, a more verbose format of the response will be returned.+Instead of a response like this:+[source,json]----diff --git a/solr/solr-ref-guide/src/monitoring-solr.adoc b/solr/solr-ref-guide/src/monitoring-solr.adocindex 957abf4a154..f3af4f9700d 100644--- a/solr/solr-ref-guide/src/monitoring-solr.adoc+++ b/solr/solr-ref-guide/src/monitoring-solr.adoc@@ -1,5 +1,17 @@= Monitoring Solr-:page-children: metrics-reporting, mbean-request-handler, configuring-logging, using-jmx-with-solr, monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference, solr-tracing+:page-children: configuring-logging, \+ ping, \+ metrics-reporting, \+ performance-statistics-reference, \+ plugins-stats-screen, \+ mbean-request-handler, \+ monitoring-with-prometheus-and-grafana, \+ jmx-with-solr, \+ thread-dump, \+ distributed-tracing, \+ circuit-breakers, \+ rate-limiters, \+ task-management// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -17,23 +29,28 @@// specific language governing permissions and limitations// under the License.-Administration and monitoring can be performed using the web-based administration console, through the command line interface, or using REST APIs.---Common administrative tasks include:--<<metrics-reporting.adoc#,Metrics Reporting>>: Details of Solr's metrics registries and Metrics API.--<<mbean-request-handler.adoc#,MBean Request Handler>>: How to use Solr's MBeans for programmatic access to the system plugins and stats.--<<configuring-logging.adoc#,Configuring Logging>>: Describes how to configure logging for Solr.--<<using-jmx-with-solr.adoc#,Using JMX with Solr>>: Describes how to use Java Management Extensions with Solr.--<<monitoring-solr-with-prometheus-and-grafana.adoc#,Monitoring Solr with Prometheus and Grafana>>: Describes how to monitor Solr with Prometheus and Grafana.--<<performance-statistics-reference.adoc#,Performance Statistics Reference>>: Additional information on statistics returned from JMX.--<<solr-tracing.adoc#,Distributed Solr Tracing>>: Describes how to do distributed tracing for Solr requests.--+[.lead]+This section covers tools and approaches for monitoring Solr via the web-based administration UI, the command line interface, or REST APIs.++****+// This tags the below list so it can be used in the parent page section list+// tag::monitoring-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<configuring-logging.adoc#,Configuring Logging>>: Log levels and logging slow queries.+| <<ping.adoc#,Ping>>: Ping a named core to determine whether it is active.+| <<metrics-reporting.adoc#,Metrics Reporting>>: Metrics registries and Metrics API.+| <<performance-statistics-reference.adoc#,Performance Statistics Reference>>: Statistics returned from JMX.+| <<plugins-stats-screen.adoc#,Plugins/Stats Screen>>: Admin UI for handler and component statistics.+| <<mbean-request-handler.adoc#,MBean Request Handler>>: MBeans for programmatic access to system plugins and stats.+| <<monitoring-with-prometheus-and-grafana.adoc#,Monitoring with Prometheus and Grafana>>: Monitor Solr with Prometheus and Grafana.+| <<jmx-with-solr.adoc#,JMX with Solr>>: Java Management Extensions and Solr.+| <<thread-dump.adoc#,Thread Dump Screen>>: Admin UI for information about each thread.+| <<distributed-tracing.adoc#,Distributed Solr Tracing>>: Distributed tracing for Solr requests.+| <<circuit-breakers.adoc#,Circuit Breakers>>: Limit loads based on Java heap usage or CPU utilization.+| <<rate-limiters.adoc#,Request Rate Limiters>> Limit concurrent requests by type.+| <<task-management.adoc#,Task Management>> Control running tasks.+|+|===+// end::monitoring-sections[]+****diff --git a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc b/solr/solr-ref-guide/src/monitoring-with-prometheus-and-grafana.adocsimilarity index 75%rename from solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adocrename to solr/solr-ref-guide/src/monitoring-with-prometheus-and-grafana.adocindex e5bf09ec510..cfd8527b85b 100644--- a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc+++ b/solr/solr-ref-guide/src/monitoring-with-prometheus-and-grafana.adoc@@ -1,4 +1,4 @@-= Monitoring Solr with Prometheus and Grafana+= Monitoring with Prometheus and Grafana// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,20 +18,23 @@If you use https://prometheus.io[Prometheus] and https://grafana.com[Grafana] for metrics storage and data visualization, Solr includes a Prometheus exporter to collect metrics and other data.-A Prometheus exporter (`solr-exporter`) allows users to monitor not only Solr metrics which come from <<metrics-reporting.adoc#metrics-api,Metrics API>>, but also facet counts which come from <<searching.adoc#,Searching>> and responses to <<collections-api.adoc#,Collections API>> commands and <<ping.adoc#,PingRequestHandler>> requests.+A Prometheus exporter (`solr-exporter`) allows users to monitor not only Solr metrics which come from <<metrics-reporting.adoc#,Metrics API>>, but also facet counts which come from <<query-guide.adoc#,Query Guide>> and responses to <<collections-api.adoc#,Collections API>> commands and <<ping.adoc#,PingRequestHandler>> requests.This graphic provides a more detailed view:.solr-exporter Diagram-image::images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png[image,width=600]+image::images/monitoring-with-prometheus-and-grafana/solr-exporter-diagram.png[image,width=600]The Prometheus exporter is included in Solr as a contrib, and is located in `contrib/prometheus-exporter` in your Solr instance.There are three aspects to running `solr-exporter`:-* Modify the `solr-exporter-config.xml` to define the data to collect. Solr has a default configuration you can use, but if you would like to modify it before running the exporter the first time, see the section below <<Exporter Configuration>>.-* Start the exporter from within Solr. See the section below <<Starting the Exporter>>.-* Modify your Prometheus configuration to listen on the correct port. See the section below <<Prometheus Configuration>>+* Modify the `solr-exporter-config.xml` to define the data to collect.+Solr has a default configuration you can use, but if you would like to modify it before running the exporter the first time, see the section <<Exporter Configuration>> below.+* Start the exporter from within Solr.+See the section below <<Starting the Exporter>>.+* Modify your Prometheus configuration to listen on the correct port.+See the section below <<Prometheus Configuration>>== Starting the ExporterYou can start `solr-exporter` by running `./bin/solr-exporter` (Linux) or `.\bin\solr-exporter.cmd` (Windows) from the `contrib/prometheus-exporter` directory.@@ -44,14 +47,14 @@ See the commands below depending on your operating system and Solr operating mod====[.tab-label]*Linux*-.Standalone mode+.User-managed / Single-node[source,bash]----$ cd contrib/prometheus-exporter$ ./bin/solr-exporter -p 9854 -b http://localhost:8983/solr -f ./conf/solr-exporter-config.xml -n 8-----.SolrCloud mode+.SolrCloud[source,bash]----$ cd contrib/prometheus-exporter@@ -63,14 +66,14 @@ $ ./bin/solr-exporter -p 9854 -z localhost:2181/solr -f ./conf/solr-exporter-con====[.tab-label]*Windows*-.Standalone mode+.User-managed / Single-node[source,text]----> cd contrib\prometheus-exporter> .\bin\solr-exporter.cmd -p 9854 -b http://localhost:8983/solr -f .\conf\solr-exporter-config.xml -n 8-----.SolrCloud mode+.SolrCloud[source,text]----> cd contrib\prometheus-exporter@@ -88,24 +91,30 @@ All parameters can be provided via an environment variable, instead of through tDisplays command line help and usage.`-p`, `--port`, `$PORT`::-The port where Prometheus will listen for new data. This port will be used to configure Prometheus.-It can be any port not already in use on your server. The default is `8989`.+The port where Prometheus will listen for new data.+This port will be used to configure Prometheus.+It can be any port not already in use on your server.+The default is `8989`.`-b`, `--baseurl`, `$SOLR_URL`::-The Solr base URL (such as `\http://localhost:8983/solr`) when Solr is running in Standalone mode.-If you are running Solr in SolrCloud mode, do not specify this parameter.+The Solr base URL (such as `\http://localhost:8983/solr`) when Solr is running in a user-managed cluster or a single-node installation.+If you are running SolrCloud, do not specify this parameter.If neither the `-b` parameter nor the `-z` parameter are defined, the default is `-b \http://localhost:8983/solr`.`-z`, `--zkhost`, `$ZK_HOST`::-The ZooKeeper connect string (such as `localhost:9983`, or `localhost:2181/solr`) when Solr is running in SolrCloud mode.-If you are running Solr in Standalone mode, do not specify this parameter.+The ZooKeeper connect string (such as `localhost:9983`, or `localhost:2181/solr`) when Solr is running SolrCloud.+If you are running a user-managed cluster or single-node installation, do not specify this parameter.If neither the `-b` parameter nor the `-z` parameter are defined, the `-b` parameter default is used.`-f`, `--config-file`, `$CONFIG_FILE`::-The path to the configuration file that defines the Solr metrics to read. The default is `contrib/prometheus-exporter/conf/solr-exporter-config.xml`.+The path to the configuration file that defines the Solr metrics to read.+The default is `contrib/prometheus-exporter/conf/solr-exporter-config.xml`.`-n`, `--num-threads`, `$NUM_THREADS`::-The number of threads. The `solr-exporter` creates thread pools for requests to Solr. Request latency can be improved by increasing the number of threads. The default is `1`.+The number of threads.+The `solr-exporter` creates thread pools for requests to Solr.+Request latency can be improved by increasing the number of threads.+The default is `1`.`-s`, `--scrape-interval`, `$SCRAPE_INTERVAL`::The number of seconds between collecting metrics from Solr.@@ -121,13 +130,16 @@ The Solr's metrics exposed by `solr-exporter` can be seen at: `\http://localhostThe `./bin` scripts provided with the Prometheus Exporter support the use of custom java options through the following environment variables:`JAVA_HEAP`::-Sets the initial (`Xms`) and max (`Xmx`) Java heap size. The default is `512m`.+Sets the initial (`Xms`) and max (`Xmx`) Java heap size.+The default is `512m`.`JAVA_MEM`::-Custom java memory settings (e.g., `-Xms1g -Xmx2g`). This is ignored if `JAVA_HEAP` is provided.+Custom java memory settings (e.g., `-Xms1g -Xmx2g`).+This is ignored if `JAVA_HEAP` is provided.`GC_TUNE`::-Custom Java garbage collection settings. The default is `-XX:+UseG1GC`.+Custom Java garbage collection settings.+The default is `-XX:+UseG1GC`.`JAVA_OPTS`::Extra JVM options.@@ -144,7 +156,7 @@ All <<#command-line-parameters,command line parameters>> are able to be provided=== Getting Metrics from a Secured SolrCloudYour SolrCloud might be secured by measures described in <<securing-solr.adoc#,Securing Solr>>.-The security configuration can be injected into `solr-exporter` using environment variables in a fashion similar to other clients using <<using-solrj.adoc#,SolrJ>>.+The security configuration can be injected into `solr-exporter` using environment variables in a fashion similar to other clients using <<solrj.adoc#,SolrJ>>.This is possible because the main script picks up <<Environment Variable Options>> and passes them on to the Java process.Example for a SolrCloud instance secured by <<basic-authentication-plugin.adoc#,Basic Authentication>>, <<enabling-ssl.adoc#,SSL>> and <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>:@@ -167,12 +179,16 @@ $ export CLASSPATH_PREFIX="../../server/solr-webapp/webapp/WEB-INF/lib/commons-c$ ./bin/solr-exporter -p 9854 -z zk1:2181,zk2:2181,zk3:2181 -f ./conf/solr-exporter-config.xml -n 16-----Note: The Exporter needs the `commons-codec` library for SSL/BasicAuth, but does not bring it. Therefore the example reuses it from the Solr web app. Of course, you can use a different source.+NOTE:: The Exporter needs the `commons-codec` library for SSL/BasicAuth, but does not bring it.+Therefore the example reuses it from the Solr web app.+Of course, you can use a different source.== Exporter Configuration-The configuration for the `solr-exporter` defines the data to get from Solr. This includes the metrics, but can also include queries to the PingRequestHandler, the Collections API, and a query to any query request handler.+The configuration for the `solr-exporter` defines the data to get from Solr.+This includes the metrics, but can also include queries to the PingRequestHandler, the Collections API, and a query to any query request handler.-A default example configuration is in `contrib/prometheus-exporter/conf/solr-exporter-config.xml`. Below is a slightly shortened version of it:+A default example configuration is in `contrib/prometheus-exporter/conf/solr-exporter-config.xml`.+Below is a slightly shortened version of it:[source,xml]----@@ -303,7 +319,8 @@ A default example configuration is in `contrib/prometheus-exporter/conf/solr-exp----=== Configuration Tags and Elements-The `solr-exporter` works by making a request to Solr according to the definitions in the configuration file, scraping the response, and converting it to a JSON structure Prometheus can understand. The configuration file defines the elements to request, how to scrape them, and where to place the extracted data in the JSON template.+The `solr-exporter` works by making a request to Solr according to the definitions in the configuration file, scraping the response, and converting it to a JSON structure Prometheus can understand.+The configuration file defines the elements to request, how to scrape them, and where to place the extracted data in the JSON template.The `solr-exporter` configuration file always starts and closes with two simple elements:@@ -316,26 +333,36 @@ The `solr-exporter` configuration file always starts and closes with two simple</config>-----Between these elements, the data the `solr-exporter` should request is defined. There are several possible types of requests to make:+Between these elements, the data the `solr-exporter` should request is defined.+There are several possible types of requests to make:[horizontal]`<ping>`:: Scrape the response to a <<ping.adoc#,PingRequestHandler>> request.`<metrics>`:: Scrape the response to a <<metrics-reporting.adoc#metrics-api,Metrics API>> request.`<collections>`:: Scrape the response to a <<collections-api.adoc#,Collections API>> request.-`<search>`:: Scrape the response to a <<searching.adoc#,search>> request.+`<search>`:: Scrape the response to a <<query-guide.adoc#,search>> request.-Within each of these types, we need to define the query and how to work with the response. To do this, we define two additional elements:+Within each of these types, we need to define the query and how to work with the response.+To do this, we define two additional elements:`<query>`::-Defines the query parameter(s) used for the request. This section uses several additional properties to define your query:--`collection`::: The collection to issue the query against. Only used in SolrCloud mode.-`core`::: The core to issue the query against. Only used in Standalone mode.-`path`::: The path to the query endpoint where the request will be sent. Examples include `admin/metrics` or `/select` or `admin/collections`.-`params`::: Additional query parameters. These will vary depending on the request type and the endpoint. For example, if using the Metrics endpoint, you can add parameters to limit the query to a certain group and/or prefix. If you're using the Collections API, the command you want to use would be a parameter.+Defines the query parameter(s) used for the request.+This section uses several additional properties to define your query:++`collection`::: The collection to issue the query against.+Only used with SolrCloud clusters.+`core`::: The core to issue the query against.+Only used with user-managed clusters or single-node installations.+`path`::: The path to the query endpoint where the request will be sent.+Examples include `admin/metrics` or `/select` or `admin/collections`.+`params`::: Additional query parameters.+These will vary depending on the request type and the endpoint.+For example, if using the Metrics endpoint, you can add parameters to limit the query to a certain group and/or prefix.+If you're using the Collections API, the command you want to use would be a parameter.`<jsonQueries>`::-This is an array that defines one or more JSON Queries in jq syntax. For more details about how to structure these queries, see https://stedolan.github.io/jq/manual/[the jq user manual].+This is an array that defines one or more JSON Queries in jq syntax.+For more details about how to structure these queries, see https://stedolan.github.io/jq/manual/[the jq user manual].+A jq query has to output JSON in the following format:+@@ -367,22 +394,27 @@ The `solr-exporter` converts the JSON to the following exposition format:The following parameters should be set:`name`::-The metric name to set. For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].+The metric name to set.+For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].`type`::-The type of the metric, can be `COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM` or `UNTYPED`. For more details, see https://prometheus.io/docs/concepts/metric_types/[Prometheus metric types].+The type of the metric, can be `COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM` or `UNTYPED`.+For more details, see https://prometheus.io/docs/concepts/metric_types/[Prometheus metric types].`help`::Help text for the metric.`label_names`::-Label names for the metric. For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].+Label names for the metric.+For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].`label_values`::-Label values for the metric. For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].+Label values for the metric.+For more details, see https://prometheus.io/docs/practices/naming/[Prometheus naming best practices].`value`::-Value for the metric. Value must be set to Double type.+Value for the metric.+Value must be set to Double type.For example, `solr-exporter` converts the JSON in the previous section to the following:@@ -395,7 +427,8 @@ solr_ping{base_url="http://localhost:8983/solr",core="collection1"} 1.0== Prometheus Configuration-Prometheus is a separate server that you need to download and deploy. More information can be found at the Prometheus https://prometheus.io/docs/prometheus/latest/getting_started/[Getting Started] page.+Prometheus is a separate server that you need to download and deploy.+More information can be found at the Prometheus https://prometheus.io/docs/prometheus/latest/getting_started/[Getting Started] page.In order for Prometheus to know about the `solr-exporter`, the listen address must be added to the Prometheus server's `prometheus.yml` configuration file, as in this example:@@ -415,13 +448,13 @@ You can test that the Prometheus server, `solr-exporter`, and Solr are working tdoing a query for `solr_ping` metric in the Prometheus GUI:.Prometheus Solr Ping expression-image::images/monitoring-solr-with-prometheus-and-grafana/prometheus-solr-ping.png[image,width=800]-+image::images/monitoring-with-prometheus-and-grafana/prometheus-solr-ping.png[image,width=800]== Sample Grafana Dashboard-Grafana is another separate server that you need to download and deploy. More information can be found on the Grafana https://grafana.com/docs/grafana/latest/[Documentation] site.+To use Grafana for visualization, it must be downloaded and deployed separately.+More information can be found on the Grafana https://grafana.com/docs/grafana/latest/[Documentation] site.Grafana consumes data from many sources, including the Prometheus server that you previously set up.A Grafana sample dashboard is provided in the following JSON file: `contrib/prometheus-exporter/conf/grafana-solr-dashboard.json`.@@ -432,4 +465,4 @@ TIP: You can directly import the Solr dashboard https://grafana.com/grafana/dashThis screenshot shows what it might look like:.Grafana Dashboard-image::images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png[image,width=800]+image::images/monitoring-with-prometheus-and-grafana/grafana-solr-dashboard.png[image,width=800]diff --git a/solr/solr-ref-guide/src/morelikethis.adoc b/solr/solr-ref-guide/src/morelikethis.adocindex 72fb9aa06b5..1ec05c97f63 100644--- a/solr/solr-ref-guide/src/morelikethis.adoc+++ b/solr/solr-ref-guide/src/morelikethis.adoc@@ -25,7 +25,8 @@ The first, and most common, is to use it as a request handler.In this case, you would send text to the MoreLikeThis request handler as needed (as in when a user clicked on a "similar documents" link).The second is to use it as a search component.-This is less desirable since it performs the MoreLikeThis analysis on every document that matches a user query. This may slow search results.+This is less desirable since it performs the MoreLikeThis analysis on every document that matches a user query.+This may slow search results.Another approach is to use it as a request handler but with externally supplied text.This case, also referred to as the MoreLikeThisHandler, will supply information about similar documents in the index based on the text of the input document.@@ -38,7 +39,7 @@ This operates in much the same way as the request handler but since it is a quer`MoreLikeThis` constructs a Lucene query based on terms in a document.It does this by pulling terms from the list of fields provided with the request.-For best results, the fields should have stored term vectors (`termVectors=true`), which can be <<defining-fields.adoc#,configured in the schema>>.+For best results, the fields should have stored term vectors (`termVectors=true`), which can be <<fields.adoc#,configured in the schema>>.If term vectors are not stored, MoreLikeThis can generate terms from stored fields.The field used for the `uniqueKey` must also be stored in order for MoreLikeThis to work properly.@@ -156,7 +157,7 @@ Possible values are `true` or `false`.|Optional |Default: none|===+-Query fields and their boosts using the same format used by the <<the-dismax-query-parser.adoc#,DisMax Query Parser>>.+Query fields and their boosts using the same format used by the <<dismax-query-parser.adoc#,DisMax Query Parser>>.These fields must also be specified in `mlt.fl`.`mlt.interestingTerms`::@@ -176,7 +177,7 @@ Unless `mlt.boost=true`, all terms will have `boost=1.0`.+To use this parameter with the <<MoreLikeThis Search Component,search component>>, the query cannot be distributed.-In order to get interesting terms, the query must be sent to a single shard and limited to that shard only (with the <<distributed-requests.adoc#limiting-which-shards-are-queried,`shards`>> parameter).+In order to get interesting terms, the query must be sent to a single shard and limited to that shard only (with the <<solrcloud-distributed-requests.adoc#limiting-which-shards-are-queried,`shards`>> parameter).Multi-shard support is, however, available with the MoreLikeThis request handler.=== MoreLikeThis Request Handler@@ -220,7 +221,7 @@ curl -X POST -H 'Content-type:application/json' -d {Both of the above examples set the `mlt.fl` parameter to "body" for the request handler.This means that all requests to the handler will use that value for the parameter unless specifically overridden in an individual request.-For more about request handler configuration in general, see the section <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#default-components,RequestHandlers and SearchComponents in Solrconfig>>.+For more about request handler configuration in general, see the section <<requesthandlers-searchcomponents.adoc#default-components,Request Handlers and SearchComponents>>.==== Request Handler Parameters@@ -338,7 +339,7 @@ It's important to note this could incur a cost to search performance so should o==== Search Component Configuration-The MoreLikeThis search component is a default search component that works with all search handlers (see also <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#default-components,Default Components>>).+The MoreLikeThis search component is a default search component that works with all search handlers (see also <<requesthandlers-searchcomponents.adoc#default-components,Default Components>>).Since it is configured already, it doesn't need any additional configuration unless you'd like to set parameters for a particular collection that override the MoreLikeThis defaults.To do this, you could configure it like this:@@ -355,7 +356,7 @@ The above example would always enable MoreLikeThis for all queries and will alwaThis is probably not something you really want!But the example serves to show how you might define whichever parameters you would like to be default for MoreLikeThis.-If you gave the search component a name other than "mlt" as in the above example, you would need to explicitly add it to a request handler as described in the section <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#referencing-search-components,Referencing Search Components>>.+If you gave the search component a name other than "mlt" as in the above example, you would need to explicitly add it to a request handler as described in the section <<requesthandlers-searchcomponents.adoc#referencing-search-components,Referencing Search Components>>.Because the above example uses the same name as the default, the parameters defined there override Solr's default.==== Search Component Parameters@@ -586,11 +587,12 @@ Sets the maximum number of tokens to parse in each example document field that i|Optional |Default: `false`|===+-Specifies if the query will be boosted by the interesting term relevance. It can be either `true` or `false`.+Specifies if the query will be boosted by the interesting term relevance.+It can be either `true` or `false`.=== Query Parser Query and Response-The structure of a MoreLikeThis query parser request is like a query using <<local-parameters-in-queries.adoc#,local params>>, as in:+The structure of a MoreLikeThis query parser request is like a query using <<local-params.adoc#,local params>>, as in:[source,bash]----diff --git a/solr/solr-ref-guide/src/near-real-time-searching.adoc b/solr/solr-ref-guide/src/near-real-time-searching.adocdeleted file mode 100644index ba453c8361f..00000000000--- a/solr/solr-ref-guide/src/near-real-time-searching.adoc+++ /dev/null@@ -1,88 +0,0 @@-= Near Real Time Searching-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Near Real Time (NRT) search means that documents are available for search soon after being indexed. NRT searching is one of the main features of SolrCloud and is rarely attempted in leader/follower configurations.--Document durability and searchability are controlled by `commits`. The "Near" in "Near Real Time" is configurable to meet the needs of your application. Commits are either "hard" or "soft" and can be issued by a client (say SolrJ), via a REST call or configured to occur automatically in `solrconfig.xml`. The recommendation usually gives is to configure your commit strategy in `solrconfig.xml` (see below) and avoid issuing commits externally.--Typically in NRT applications, hard commits are configured with `openSearcher=false`, and soft commits are configured to make documents visible for search.--When a commit occurs, various background tasks are initiated, segment merging for example. These background tasks do not block additional updates to the index nor do they delay the availability of the documents for search.--When configuring for NRT, pay special attention to cache and autowarm settings as they can have a significant impact on NRT performance. For extremely short autoCommit intervals, consider disabling caching and autowarming completely.--== Commits and Searching--A *hard commit* calls `fsync` on the index files to ensure they have been flushed to stable storage. The current transaction log is closed and a new one is opened. See the "transaction log" discussion below for how data is recovered in the absence of a hard commit. Optionally a hard commit can also make documents visible for search, but this is not recommended for NRT searching as it is more expensive than a soft commit.--A *soft commit* is faster since it only makes index changes visible and does not `fsync` index files, start a new segment or start a new transaction log. Search collections that have NRT requirements will want to soft commit often enough to satisfy the visibility requirements of the application. A softCommit may be "less expensive" than a hard commit (openSearcher=true), but it is not free. It is recommended that this be set for as long as is reasonable given the application requirements.--Both hard and soft commits have two primary configuration parameters: `maxDocs` and `maxTime`.--`maxDocs`::-Integer. Defines the number of updates to process before activating.--`maxTime`::-Integer. The number of milliseconds to wait before activating.--If both of these parameters are specified, the first one to expire is honored. Generally, it is preferred to use `maxTime` rather than `maxDocs`, especially when indexing large numbers of documents in batches. Use `maxDocs` and `maxTime` judiciously to fine-tune your commit strategies.--Hard commit has an additional parameter `openSearcher`--`openSearcher`::-true|false, whether to make documents visible for search. For NRT applications this is usually set to `false` and `soft commit` is configured to control when documents are visible for search.--=== Transaction Logs (tlogs)--Transaction logs are a "rolling window" of updates since the last hard commit. The current transaction log is closed and a new one opened each time any variety of hard commit occurs. Soft commits have no effect on the transaction log.--When tlogs are enabled, documents being added to the index are written to the tlog before the indexing call returns to the client. In the event of an un-graceful shutdown (power loss, JVM crash, `kill -9`, etc.) any documents written to the tlog but not yet committed with a hard commit when Solr was stopped are replayed on startup. Therefore the data is not lost.--When Solr is shut down gracefully (using the `bin/solr stop` command) Solr will close the tlog file and index segments so no replay will be necessary on startup.--One point of confusion is how much data is contained in a transaction log. A tlog does not contain all documents, only the ones since the last hard commit. Older transaction log files are deleted when no longer needed.--WARNING: Implicit in the above is that transaction logs will grow forever if hard commits are disabled. Therefore it is important that hard commits be enabled when indexing.--=== Configuring Commits--As mentioned above, it is usually preferable to configure your commits (both hard and soft) in `solrconfig.xml` and avoid sending commits from an external source. Check your `solrconfig.xml` file since the defaults are likely not tuned to your needs. Here is an example NRT configuration for the two flavors of commit, a hard commit every 60 seconds and a soft commit every 30 seconds. Note that these are _not_ the values in some of the examples!--[source,xml]------<autoCommit>- <maxTime>${solr.autoCommit.maxTime:60000}</maxTime>- <openSearcher>false</openSearcher>-</autoCommit>--<autoSoftCommit>- <maxTime>${solr.autoSoftCommit.maxTime:30000}</maxTime>- </autoSoftCommit>-------TIP: These parameters can be overridden at run time by defining Java "system variables", for example specifying ``-Dsolr.autoCommit.maxTime=15000` would override the hard commit interval with a value of 15 seconds.--The choices for `autoCommit` (with `openSearcher=false`) and `autoSoftCommit` have different consequences. In the event of un-graceful shutdown, it can take up to the time specified in `autoCommit` for Solr to replay the uncommitted documents from the transaction log.--The time chosen for `autoSoftCommit` determines the maximum time after a document is sent to Solr before it becomes searchable and does not affect the transaction log. Choose as long an interval as your application can tolerate for this value, often 15-60 seconds is reasonable, or even longer depending on the requirements. In situations where the the time is set to a very short interval (say 1 second), consider disabling your caches (queryResultCache and filterCache especially) as they will have little utility.--TIP: For extremely high bulk indexing, especially for the initial load if there is no searching, consider turning off `autoSoftCommit` by specifying a value of `-1` for the maxTime parameter.--== Advanced Commit Options--All varieties of commits can be invoked from a SolrJ client or via a URL. The usual recommendation is to _not_ call commits externally. For those cases where it is desirable, see <<uploading-data-with-index-handlers.adoc#xml-update-commands,Update Commands>>. These options are listed for XML update commands that can be issued from a browser or curl, etc., and the equivalents are available from a SolrJ client.diff --git a/solr/solr-ref-guide/src/numerical-analysis.adoc b/solr/solr-ref-guide/src/numerical-analysis.adocindex dc3af58bccf..de43acd9841 100644--- a/solr/solr-ref-guide/src/numerical-analysis.adoc+++ b/solr/solr-ref-guide/src/numerical-analysis.adoc@@ -21,148 +21,114 @@ This section explores the interrelated math expressions for interpolation and nu== InterpolationInterpolation is used to construct new data points between a set of known control of points.-The ability to predict new data points allows for sampling along the curve defined by the-control points.+The ability to predict new data points allows for sampling along the curve defined by the control points.The interpolation functions described below all return an _interpolation function_that can be passed to other functions which make use of the sampling capability.-If returned directly the interpolation function returns an array containing predictions for each of the-control points. This is useful in the case of `loess` interpolation which first smooths the control points-and then interpolates the smoothed points. All other interpolation functions simply return the original-control points because interpolation predicts a curve that passes through the original control points.+If returned directly the interpolation function returns an array containing predictions for each of the control points.+This is useful in the case of `loess` interpolation which first smooths the control points and then interpolates the smoothed points.+All other interpolation functions simply return the original control points because interpolation predicts a curve that passes through the original control points.-There are different algorithms for interpolation that will result in different predictions-along the curve. The math expressions library currently supports the following-interpolation functions:+There are different algorithms for interpolation that will result in different predictions along the curve.+The math expressions library currently supports the following interpolation functions:-* `lerp`: Linear interpolation predicts points that pass through each control point and- form straight lines between control points.-* `spline`: Spline interpolation predicts points that pass through each control point-and form a smooth curve between control points.+* `lerp`: Linear interpolation predicts points that pass through each control point and form straight lines between control points.+* `spline`: Spline interpolation predicts points that pass through each control point and form a smooth curve between control points.* `akima`: Akima spline interpolation is similar to spline interpolation but is stable to outliers.-* `loess`: Loess interpolation first performs a non-linear local regression to smooth the original-control points. Then a spline is used to interpolate the smoothed control points.+* `loess`: Loess interpolation first performs a non-linear local regression to smooth the original control points.+Then a spline is used to interpolate the smoothed control points.=== Sampling Along the Curve-One way to better understand interpolation is to visualize what it means to sample along a curve. The example-below zooms in on a specific region of a curve by sampling the curve between a specific x-axis range.+One way to better understand interpolation is to visualize what it means to sample along a curve.+The example below zooms in on a specific region of a curve by sampling the curve between a specific x-axis range.image::images/math-expressions/interpolate1.png[]-The visualization above first creates two arrays with x and y-axis points. Notice that the x-axis ranges from- 0 to 9. Then the `akima`, `spline` and `lerp`-functions are applied to the vectors to create three interpolation functions.+The visualization above first creates two arrays with x and y-axis points.+Notice that the x-axis ranges from 0 to 9.+Then the `akima`, `spline` and `lerp` functions are applied to the vectors to create three interpolation functions.-Then 500 hundred random samples are drawn from a uniform distribution between 0 and 3. These are-the new zoomed in x-axis points, between 0 and 3. Notice that we are sampling a specific-area of the curve.+Then 500 hundred random samples are drawn from a uniform distribution between 0 and 3. These are the new zoomed in x-axis points, between 0 and 3. Notice that we are sampling a specific area of the curve.-Then the `predict` function is used to predict y-axis points for-the sampled x-axis, for all three interpolation functions. Finally all three prediction vectors-are plotted with the sampled x-axis points.+Then the `predict` function is used to predict y-axis points for the sampled x-axis, for all three interpolation functions.+Finally all three prediction vectors are plotted with the sampled x-axis points.-The red line is the `lerp` interpolation, the blue line is the `akima` and the purple line is-the `spline` interpolation. You can see they each produce different curves in between the control-points.+The red line is the `lerp` interpolation, the blue line is the `akima` and the purple line is the `spline` interpolation.+You can see they each produce different curves in between the control points.=== Smoothing Interpolation-The `loess` function is a smoothing interpolator which means it doesn't derive-a function that passes through the original control points. Instead the `loess` function-returns a function that smooths the original control points.+The `loess` function is a smoothing interpolator which means it doesn't derive a function that passes through the original control points.+Instead the `loess` function returns a function that smooths the original control points.-A technique known as local regression is used to compute the smoothed curve. The size of the-neighborhood of the local regression can be adjusted-to control how close the new curve conforms to the original control points.+A technique known as local regression is used to compute the smoothed curve.+The size of the neighborhood of the local regression can be adjusted to control how close the new curve conforms to the original control points.The `loess` function is passed x- and y-axes and fits a smooth curve to the data.-If only a single array is provided it is treated as the y-axis and a sequence is generated-for the x-axis.--The example below shows the `loess` function being used to model a monthly-time series. In the example the `timeseries` function is used to generate-a monthly time series of average closing prices for the stock ticker-*AMZN*. The `date_dt` and `avg(close_d)` fields from the time series-are then vectorized and stored in variables `x` and `y`. The `loess`-function is then applied to the *y* vector containing the average closing-prices. The `bandwidth` named parameter specifies the percentage-of the data set used to compute the local regression. The `loess` function-returns the fitted model of smoothed data points.--The `zplot` function is then used to plot the `x`, `y` and `y1`-variables.+If only a single array is provided it is treated as the y-axis and a sequence is generated for the x-axis.++The example below shows the `loess` function being used to model a monthly time series.+In the example the `timeseries` function is used to generate a monthly time series of average closing prices for the stock ticker *AMZN*.+The `date_dt` and `avg(close_d)` fields from the time series are then vectorized and stored in variables `x` and `y`.+The `loess` function is then applied to the *y* vector containing the average closing prices.+The `bandwidth` named parameter specifies the percentage of the data set used to compute the local regression.+The `loess` function returns the fitted model of smoothed data points.++The `zplot` function is then used to plot the `x`, `y` and `y1` variables.image::images/math-expressions/loess.png[]== Derivatives-The derivative of a function measures the rate of change of the `y` value in respects to the-rate of change of the `x` value.+The derivative of a function measures the rate of change of the `y` value in respect to the rate of change of the `x` value.-The `derivative` function can compute the derivative for any of the-interpolation functions described above. Each interpolation function-will produce different derivatives that match the characteristics-of the function.+The `derivative` function can compute the derivative for any of the interpolation functions described above.+Each interpolation function will produce different derivatives that match the characteristics of the function.=== The First Derivative (Velocity)-A simple example shows how the `derivative` function is used to calculate-the rate of change or *velocity*.+A simple example shows how the `derivative` function is used to calculate the rate of change or *velocity*.-In the example two vectors are created, one representing hours and-one representing miles traveled. The `lerp` function is then used to-create a linear interpolation of the `hours` and `miles` vectors.-The `derivative` function is then applied to the-linear interpolation. `zplot` is then used to plot the *`hours`*-on the x-axis and `miles` on the y-axis, and the `derivative` as `mph`,-at each x-axis point.+In the example two vectors are created, one representing hours and one representing miles traveled.+The `lerp` function is then used to create a linear interpolation of the `hours` and `miles` vectors.+The `derivative` function is then applied to the linear interpolation.+`zplot` is then used to plot the *`hours`* on the x-axis and `miles` on the y-axis, and the `derivative` as `mph`, at each x-axis point.image::images/math-expressions/derivative.png[]-Notice that the *miles_traveled* line has a slope of 10 until the-5th hour where-it changes to a slope of 50. The *mph* line, which is- the derivative, visualizes the *velocity* of the- *miles_traveled* line.+Notice that the *miles_traveled* line has a slope of 10 until the 5th hour where it changes to a slope of 50. The *mph* line, which is the derivative, visualizes the *velocity* of the *miles_traveled* line.-Also notice that the derivative is calculated along-straight lines showing immediate change from one point to the next. This-is because linear interpolation (`lerp`) is used as the interpolation-function. If the `spline` or `akima` functions had been used it would have produced-a derivative with rounded curves.+Also notice that the derivative is calculated along straight lines showing immediate change from one point to the next.+This is because linear interpolation (`lerp`) is used as the interpolation function.+If the `spline` or `akima` functions had been used it would have produced a derivative with rounded curves.=== The Second Derivative (Acceleration)-While the first derivative represents velocity, the second derivative-represents `acceleration`. The second the derivative is the derivative-of the first derivative.+While the first derivative represents velocity, the second derivative represents `acceleration`. The second the derivative is the derivative of the first derivative.The example below builds on the first example and adds the second derivative.-Notice that the second derivative `d2` is taken by applying the-derivative function to a linear interpolation of the first derivative.+Notice that the second derivative `d2` is taken by applying the derivative function to a linear interpolation of the first derivative.The second derivative is plotted as *acceleration* on the chart.image::images/math-expressions/derivatives.png[]-Notice that the acceleration line is 0 until the *mph* line increases from 10 to 50. At this-point the *acceleration* line moves to 40. As the *mph* line stays at 50, the acceleration-line drops to 0.+Notice that the acceleration line is 0 until the *mph* line increases from 10 to 50.+At this point the *acceleration* line moves to 40.+As the *mph* line stays at 50, the acceleration line drops to 0.=== Price Velocity-The example below shows how to plot the `derivative` for a time series generated-by the `timeseries` function. In the example a monthly time series is-generated for the average closing price for the stock ticker `amzn`.-The `avg(close)` column is vectorized and interpolated using linear-interpolation (`lerp`). The `zplot` function is then used to plot the derivative-of the time series.+The example below shows how to plot the `derivative` for a time series generated by the `timeseries` function.+In the example a monthly time series is generated for the average closing price for the stock ticker `amzn`.+The `avg(close)` column is vectorized and interpolated using linear interpolation (`lerp`).+The `zplot` function is then used to plot the derivative of the time series.image::images/math-expressions/derivative2.png[]@@ -172,18 +138,15 @@ Notice that the derivative plot clearly shows the rates of change in the stock p== IntegralsAn integral is a measure of the volume underneath a curve.-The `integral` function computes the cumulative integrals for a curve or the integral for a specific-range of an interpolated curve. Like the `derivative` function the `integral` function operates-over interpolation functions.+The `integral` function computes the cumulative integrals for a curve or the integral for a specific range of an interpolated curve.+Like the `derivative` function the `integral` function operates over interpolation functions.=== Single Integral-If the `integral` function is passed a *start* and *end* range it will compute the volume under the-curve within that specific range.+If the `integral` function is passed a *start* and *end* range it will compute the volume under the curve within that specific range.-In the example below the `integral` function computes an-integral for the entire range of the curve, 0 through 10. Notice that the `integral` function is passed-the interpolated curve and the start and end range, and returns the integral for the range.+In the example below the `integral` function computes an integral for the entire range of the curve, 0 through 10.+Notice that the `integral` function is passed the interpolated curve and the start and end range, and returns the integral for the range.[source,text]----@@ -193,8 +156,7 @@ let(x=array(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19integral=integral(curve, 0, 10))-----When this expression is sent to the `/stream` handler it-responds with:+When this expression is sent to the `/stream` handler it responds with:[source,json]----@@ -215,11 +177,10 @@ responds with:=== Cumulative Integral Plot-If the `integral` function is passed a single interpolated curve it returns a vector of the cumulative-integrals for the curve. The cumulative integrals vector contains a cumulative integral calculation-for each x-axis point. The cumulative integral is calculated by taking the-integral of the range between each x-axis point and the *first* x-axis point. In the example above this would-mean calculating a vector of integrals as such:+If the `integral` function is passed a single interpolated curve it returns a vector of the cumulative integrals for the curve.+The cumulative integrals vector contains a cumulative integral calculation for each x-axis point.+The cumulative integral is calculated by taking the integral of the range between each x-axis point and the *first* x-axis point.+In the example above this would mean calculating a vector of integrals as such:[source,text]----@@ -229,32 +190,25 @@ let(x=array(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19integrals=array(0, integral(curve, 0, 1), integral(curve, 0, 2), integral(curve, 0, 3), ...)-----The plot of cumulative integrals visualizes how much cumulative volume of the curve is under each point-x-axis point.+The plot of cumulative integrals visualizes how much cumulative volume of the curve is under each point x-axis point.-The example below shows the cumulative integral plot for a time series generated by-the `timeseries` function. In the example a monthly time series is-generated for the average closing price for the stock ticker `amzn`.+The example below shows the cumulative integral plot for a time series generated by the `timeseries` function.+In the example a monthly time series is generated for the average closing price for the stock ticker `amzn`.The `avg(close)` column is vectorized and interpolated using a `spline`.-The `zplot` function is then used to plot the cumulative integral-of the time series.+The `zplot` function is then used to plot the cumulative integral of the time series.image::images/math-expressions/integral.png[]-The plot above visualizes the volume under the curve as the *AMZN* stock-price changes over time. Because this plot is cumulative, the volume under-a stock price time series which stays the same over time, will-have a positive *linear* slope. A stock that has rising prices will have a *concave* shape and-a stock with falling prices will have a *convex* shape.+The plot above visualizes the volume under the curve as the *AMZN* stock price changes over time.+Because this plot is cumulative, the volume under a stock price time series which stays the same over time, will have a positive *linear* slope.+A stock that has rising prices will have a *concave* shape and a stock with falling prices will have a *convex* shape.-In this particular example the integral plot becomes more *concave* over time-showing accelerating increases in stock price.+In this particular example the integral plot becomes more *concave* over time showing accelerating increases in stock price.== Bicubic Spline-The `bicubicSpline` function can be used to interpolate and predict values-anywhere within a grid of data.+The `bicubicSpline` function can be used to interpolate and predict values anywhere within a grid of data.A simple example will make this more clear:@@ -272,15 +226,12 @@ let(years=array(1998, 2000, 2002, 2004, 2006),----In this example a bicubic spline is used to interpolate a matrix of real estate data.-Each row of the matrix represent specific `years`. Each column of the matrix-represents `floors` of the building. The grid of numbers is the average selling price of-an apartment for each year and floor. For example in 2002 the average selling price for-the 9th floor was `415000` (row 3, column 3).--The `bicubicSpline` function is then used to-interpolate the grid, and the `predict` function is used to predict a value for year 2003, floor 8.-Notice that the matrix does not include a data point for year 2003, floor 8. The `bicubicSpline`-function creates that data point based on the surrounding data in the matrix:+Each row of the matrix represent specific `years`. Each column of the matrix represents `floors` of the building.+The grid of numbers is the average selling price of an apartment for each year and floor.+For example in 2002 the average selling price for the 9th floor was `415000` (row 3, column 3).++The `bicubicSpline` function is then used to interpolate the grid, and the `predict` function is used to predict a value for year 2003, floor 8.+Notice that the matrix does not include a data point for year 2003, floor 8. The `bicubicSpline` function creates that data point based on the surrounding data in the matrix:[source,json]----diff --git a/solr/solr-ref-guide/src/other-parsers.adoc b/solr/solr-ref-guide/src/other-parsers.adocindex bbdd359a3c3..abf280201d3 100644--- a/solr/solr-ref-guide/src/other-parsers.adoc+++ b/solr/solr-ref-guide/src/other-parsers.adoc@@ -1,4 +1,4 @@-= Other Parsers+= Other Query Parsers// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,197 +16,22 @@// specific language governing permissions and limitations// under the License.-In addition to the main query parsers discussed earlier, there are several other query parsers that can be used instead of or in conjunction with the main parsers for specific purposes.+In addition to the main query parsers, there are several other query parsers that can be used instead of or in conjunction with the main parsers for specific purposes.This section details the other parsers, and gives examples for how they might be used.-Many of these parsers are expressed the same way as <<local-parameters-in-queries.adoc#,Local Parameters in Queries>>.+Many of these parsers are expressed the same way as <<local-params.adoc#,Local Params>>.== Block Join Query Parsers-There are two query parsers that support block joins. These parsers allow indexing and searching for relational content that has been <<indexing-nested-documents.adoc#, indexed as Nested Documents>>.--The example usage of the query parsers below assumes the following documents have been indexed:--[source,xml]------<add>- <doc>- <field name="id">1</field>- <field name="content_type">parent</field>- <field name="title">Solr has block join support</field>- <doc>- <field name="id">2</field>- <field name="content_type">child</field>- <field name="comments">SolrCloud supports it too!</field>- </doc>- </doc>- <doc>- <field name="id">3</field>- <field name="content_type">parent</field>- <field name="title">New Lucene and Solr release</field>- <doc>- <field name="id">4</field>- <field name="content_type">child</field>- <field name="comments">Lots of new features</field>- </doc>- </doc>-</add>-------=== Block Join Children Query Parser--This parser wraps a query that matches some parent documents and returns the children of those documents.--The syntax for this parser is: `q={!child of=<blockMask>}<someParents>`.--* The inner subordinate query string (`someParents`) must be a query that will match some parent documents-* The `of` parameter must be a query string to use as a <<#block-mask,Block Mask>> -- typically a query that matches the set of all possible parent documents--The resulting query will match all documents which do _not_ match the `<blockMask>` query and are children (or descendents) of the documents matched by `<someParents>`.--Using the example documents above, we can construct a query such as `q={!child of="content_type:parent"}title:lucene`. We only get one document in response:--[source,xml]------<result name="response" numFound="1" start="0">- <doc>- <str name="id">4</str>- <arr name="content_type"><str>child</str></arr>- <str name="comments">Lots of new features</str>- </doc>-</result>-------[CAUTION]-====-The query for `someParents` *MUST* match a strict subset of the documents matched by the <<#block-mask,Block Mask>> or your query may result in an Error:--[literal]-Parent query must not match any docs besides parent filter. Combine them as must (+) and must-not (-) clauses to find a problem doc.--You can search for `q=+(someParents) -(blockMask)` to find a cause if you encounter this type of error.-====--==== Filtering and Tagging--`{!child}` also supports `filters` and `excludeTags` local parameters like the following:--[source,text]-?q={!child of=<blockMask> filters=$parentfq excludeTags=certain}<someParents>-&parentfq=BRAND:Foo-&parentfq=NAME:Bar-&parentfq={!tag=certain}CATEGORY:Baz--This is equivalent to:--[source,text]-q={!child of=<blockMask>}+<someParents> +BRAND:Foo +NAME:Bar--Notice "$" syntax in `filters` for referencing queries; comma-separated tags `excludeTags` allows to exclude certain queries by tagging. Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>. Note, that filtering is applied to the subordinate clause (`<someParents>`), and the intersection result is joined to the children.--==== All Children Syntax--When subordinate clause (`<someParents>`) is omitted, it's parsed as a _segmented_ and _cached_ filter for children documents. More precisely, `q={!child of=<blockMask>}` is equivalent to `q=\*:* -<blockMask>`.--=== Block Join Parent Query Parser--This parser takes a query that matches child documents and returns their parents.--The syntax for this parser is similar to the `child` parser: `q={!parent which=<blockMask>}<someChildren>`.--* The inner subordinate query string (`someChildren`) must be a query that will match some child documents-* The `which` parameter must be a query string to use as a <<#block-mask,Block Mask>> -- typically a query that matches the set of all possible parent documents--The resulting query will match all documents which _do_ match the `<blockMask>` query and are parents (or ancestors) of the documents matched by `<someChildren>`.--Again using the example documents above, we can construct a query such as `q={!parent which="content_type:parent"}comments:SolrCloud`. We get this document in response:--[source,xml]------<result name="response" numFound="1" start="0">- <doc>- <str name="id">1</str>- <arr name="content_type"><str>parent</str></arr>- <arr name="title"><str>Solr has block join support</str></arr>- </doc>-</result>--------[CAUTION]-====-The query for `someChildren` *MUST NOT* match any documents matched by the <<#block-mask,Block Mask>> or your query may result in an Error:--[literal]-Child query must not match same docs with parent filter. Combine them as must clauses (+) to find a problem doc.--You can search for `q=+(blockMask) +(someChildren)` to find a cause.-====---==== Filtering and Tagging--The `{!parent}` query supports `filters` and `excludeTags` local parameters like the following:--[source,text]-?q={!parent which=<blockMask> filters=$childfq excludeTags=certain}<someChildren>-&childfq=COLOR:Red-&childfq=SIZE:XL-&childfq={!tag=certain}PRINT:Hatched--This is equivalent to:--[source,text]-q={!parent which=<blockMask>}+<someChildren> +COLOR:Red +SIZE:XL--Notice the "$" syntax in `filters` for referencing queries. Comma-separated tags in `excludeTags` allow excluding certain queries by tagging. Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>. Note that filtering is applied to the subordinate clause (`<someChildren>`) first, and the intersection result is joined to the parents.--==== Scoring with the Block Join Parent Query Parser--You can optionally use the `score` local parameter to return scores of the subordinate query. The values to use for this parameter define the type of aggregation, which are `avg` (average), `max` (maximum), `min` (minimum), `total (sum)`. Implicit default is `none` which returns `0.0`.--==== All Parents Syntax--When subordinate clause (`<someChildren>`) is omitted, it's parsed as a _segmented_ and _cached_ filter for all parent documents, or more precisely `q={!parent which=<blockMask>}` is equivalent to `q=<blockMask>`.--[#block-mask]-=== Block Masks: The `of` and `which` local params--The purpose of the "Block Mask" query specified as either an `of` or `which` parameter (depending on the parser used) is to identy the set of all documents in the index which should be treated as "parents" _(or their ancestors)_ and which documents should be treated as "children". This is important because in the "on disk" index, the relationships are flattened into "blocks" of documents, so the `of` / `which` params are needed to serve as a "mask" against the flat document blocks to identify the boundaries of every hierarchical relationship.--In the example queries above, we were able to use a very simple Block Mask of `doc_type:parent` because our data is very simple: every document is either a `parent` or a `child` So this query string easily distinguishes _all_ of our documents.--A common mistake is to try and use a `which` parameter that is more restrictive then the set of all parent documents, in order to filter the parents that are matched, as in this bad example:-------// BAD! DO NOT USE!-q={!parent which="title:join"}comments:support-------This type of query will frequently not work the way you might expect. Since the `which` parameter only identifies _some_ of the "parent" documents, the resulting query can match "parent" documents it should not, because it will mistakenly identify all documents which do _not_ match the `which="title:join"` Block Mask as children of the next "parent" document in the index (that does match this Mask).--A similar problematic situation can arise when mixing parent/child documents with "simple" documents that have no children _and do not match the query used to identify 'parent' documents_. For example, if we add the following document to our existing parent/child example documents:--[source,xml]------<add>- <doc>- <field name="id">0</field>- <field name="content_type">plain</field>- <field name="title">Lucene and Solr are cool</field>- </doc>-</add>-------...then our simple `doc_type:parent` Block Mask would no longer be adequate. We would instead need to use `\*:* -doc_type:child` or `doc_type:(simple parent)` to prevent our "simple" document from mistakenly being treated as a "child" of an adjacent "parent" document.--The <<searching-nested-documents#searching-nested-documents,Searching Nested Documents>> section contains more detailed examples of specifing Block Mask queries with non trivial hierarchicies of documents.+The Block Join query parsers are used with nested documents to query for parents and/or children.+These parsers are covered in detail in the section <<block-join-query-parser.adoc#,Block Join Query Parser>>.== Boolean Query Parser-The `BoolQParser` creates a Lucene `BooleanQuery` which is a boolean combination of other queries. Sub-queries along with their typed occurrences indicate how documents will be matched and scored.+The `BoolQParser` creates a Lucene `BooleanQuery` which is a boolean combination of other queries.+Sub-queries along with their typed occurrences indicate how documents will be matched and scored.*Parameters*@@ -217,13 +42,18 @@ A list of queries that *must* appear in matching documents and contribute to theA list of queries that *must not* appear in matching documents.`should`::-A list of queries *should* appear in matching documents. For a BooleanQuery with no `must` queries, one or more `should` queries must match a document for the BooleanQuery to match.+A list of queries *should* appear in matching documents.+For a BooleanQuery with no `must` queries, one or more `should` queries must match a document for the BooleanQuery to match.`filter`::-A list of queries that *must* appear in matching documents. However, unlike `must`, the score of filter queries is ignored. Also, these queries are cached in filter cache. To avoid caching add either `cache=false` as local parameter, or `"cache":"false"` property to underneath Query DLS Object.+A list of queries that *must* appear in matching documents.+However, unlike `must`, the score of filter queries is ignored.+Also, these queries are cached in filter cache.+To avoid caching add either `cache=false` as local parameter, or `"cache":"false"` property to underneath Query DLS Object.`excludeTags`::-Comma separated list of tags for excluding queries from parameters above. See explanation below.+Comma separated list of tags for excluding queries from parameters above.+See explanation below.*Examples*@@ -237,14 +67,16 @@ Comma separated list of tags for excluding queries from parameters above. See ex{!bool filter=foo should=bar}-----Parameters might also be multivalue references. The former example above is equivalent to:+Parameters might also be multivalue references.+The former example above is equivalent to:[source,text]----q={!bool must=$ref}&ref=foo&ref=bar-----Referred queries might be excluded via tags. Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>.+Referred queries might be excluded via tags.+Overall the idea is similar to <<faceting.adoc#tagging-and-excluding-filters, excluding fq in facets>>.[source,text]----@@ -260,7 +92,8 @@ q={!bool must=foo}== Boost Query Parser-`BoostQParser` extends the `QParserPlugin` and creates a boosted query from the input value. The main value is any query to be "wrapped" and "boosted" -- only documents which match that query will match the final query produced by this parser.+`BoostQParser` extends the `QParserPlugin` and creates a boosted query from the input value.+The main value is any query to be "wrapped" and "boosted" -- only documents which match that query will match the final query produced by this parser.Parameter `b` is a <<function-queries.adoc#available-functions,function>> to be evaluated against each document that matches the original query, and the result of the function will be multiplied into into the final score for that document.=== Boost Query Parser Examples@@ -282,7 +115,7 @@ q={!boost b=div(1,add(1,price))}name:fooThe `<<function-queries.adoc#query-function,query(...)>>` function is particularly useful for situations where you want to multiply (or divide) the score for each document matching your main query by the score that document would have from another query.-This example uses <<local-parameters-in-queries.adoc#parameter-dereferencing,local parameter variables>> to create a query for `name:foo` which is boosted by the scores from the independently specified query `category:electronics`:+This example uses <<local-params.adoc#parameter-dereferencing,local param variables>> to create a query for `name:foo` which is boosted by the scores from the independently specified query `category:electronics`:[source,text]----@@ -295,7 +128,8 @@ my_boost=category:electronicsThe `CollapsingQParser` is really a _post filter_ that provides more performant field collapsing than Solr's standard approach when the number of distinct groups in the result set is high.-This parser collapses the result set to a single document per group before it forwards the result set to the rest of the search components. So all downstream components (faceting, highlighting, etc.) will work with the collapsed result set.+This parser collapses the result set to a single document per group before it forwards the result set to the rest of the search components.+So all downstream components (faceting, highlighting, etc.) will work with the collapsed result set.Details about using the `CollapsingQParser` can be found in the section <<collapse-and-expand-results.adoc#,Collapse and Expand Results>>.@@ -308,7 +142,9 @@ Under the covers, this query parser makes use of the Span group of queries, e.g.*Parameters*`inOrder`::-Set to true to force phrase queries to match terms in the order specified. The default is `true`.+Set to true to force phrase queries to match terms in the order specified.+The default is `true`.+`df`::The default search field.@@ -333,9 +169,12 @@ A mix of ordered and unordered complex phrase queries:=== Complex Phrase Parser Limitations-Performance is sensitive to the number of unique terms that are associated with a pattern. For instance, searching for "a*" will form a large OR clause (technically a SpanOr with many terms) for all of the terms in your index for the indicated field that start with the single letter 'a'. It may be prudent to restrict wildcards to at least two or preferably three letters as a prefix. Allowing very short prefixes may result in to many low-quality documents being returned.+Performance is sensitive to the number of unique terms that are associated with a pattern.+For instance, searching for "a*" will form a large OR clause (technically a SpanOr with many terms) for all of the terms in your index for the indicated field that start with the single letter 'a'. It may be prudent to restrict wildcards to at least two or preferably three letters as a prefix.+Allowing very short prefixes may result in to many low-quality documents being returned.-Notice that it also supports leading wildcards "*a" as well with consequent performance implications. Applying <<filter-descriptions.adoc#reversed-wildcard-filter,ReversedWildcardFilterFactory>> in index-time analysis is usually a good idea.+Notice that it also supports leading wildcards "*a" as well with consequent performance implications.+Applying <<filters.adoc#reversed-wildcard-filter,ReversedWildcardFilterFactory>> in index-time analysis is usually a good idea.==== MaxBooleanClauses with Complex Phrase Parser@@ -346,7 +185,7 @@ You may need to increase MaxBooleanClauses in `solrconfig.xml` as a result of th<maxBooleanClauses>4096</maxBooleanClauses>-----This property is described in more detail in the section <<query-settings-in-solrconfig.adoc#query-sizing-and-warming,Query Sizing and Warming>>.+This property is described in more detail in the section <<caches-warming.adoc#query-sizing-and-warming,Query Sizing and Warming>>.==== Stopwords with Complex Phrase Parser@@ -361,14 +200,16 @@ While the query below does not use this parser:q=features:"Stores up to 15,000"-----the document is returned. The next query that _does_ use the Complex Phrase Query Parser, as in this query:+the document is returned.+The next query that _does_ use the Complex Phrase Query Parser, as in this query:[source,text]----q=features:"sto* up to 15*"&defType=complexphrase-----does _not_ return that document because SpanNearQuery has no good way to handle stopwords in a way analogous to PhraseQuery. If you must remove stopwords for your use case, use a custom filter factory or perhaps a customized synonyms filter that reduces given stopwords to some impossible token.+does _not_ return that document because SpanNearQuery has no good way to handle stopwords in a way analogous to PhraseQuery.+If you must remove stopwords for your use case, use a custom filter factory or perhaps a customized synonyms filter that reduces given stopwords to some impossible token.==== Escaping with Complex Phrase Parser@@ -376,7 +217,8 @@ Special care has to be given when escaping: clauses between double quotes (usual== Field Query Parser-The `FieldQParser` extends the `QParserPlugin` and creates a field query from the input value, applying text analysis and constructing a phrase query if appropriate. The parameter `f` is the field to be queried.+The `FieldQParser` extends the `QParserPlugin` and creates a field query from the input value, applying text analysis and constructing a phrase query if appropriate.+The parameter `f` is the field to be queried.Example:@@ -385,7 +227,8 @@ Example:{!field f=myfield}Foo Bar-----This example creates a phrase query with "foo" followed by "bar" (assuming the analyzer for `myfield` is a text field with an analyzer that splits on whitespace and lowercase terms). This is generally equivalent to the Lucene query parser expression `myfield:"Foo Bar"`.+This example creates a phrase query with "foo" followed by "bar" (assuming the analyzer for `myfield` is a text field with an analyzer that splits on whitespace and lowercase terms).+This is generally equivalent to the Lucene query parser expression `myfield:"Foo Bar"`.== Filters Query Parser@@ -406,7 +249,8 @@ q=+field:text +COLOR:Red +SIZE:XL== Function Query Parser-The `FunctionQParser` extends the `QParserPlugin` and creates a function query from the input value. This is only one way to use function queries in Solr; for another, more integrated, approach, see the section on <<function-queries.adoc#,Function Queries>>.+The `FunctionQParser` extends the `QParserPlugin` and creates a function query from the input value.+This is only one way to use function queries in Solr; for another, more integrated, approach, see the section on <<function-queries.adoc#,Function Queries>>.Example:@@ -417,21 +261,28 @@ Example:== Function Range Query Parser-The `FunctionRangeQParser` extends the `QParserPlugin` and creates a range query over a function. This is also referred to as `frange`, as seen in the examples below.+The `FunctionRangeQParser` extends the `QParserPlugin` and creates a range query over a function.+This is also referred to as `frange`, as seen in the examples below.*Parameters*`l`::-The lower bound. This parameter is optional.+The lower bound.+This parameter is optional.`u`::-The upper bound. This parameter is optional.+The upper bound.+This parameter is optional.`incl`::-Include the lower bound. This parameter is optional. The default is `true`.+Include the lower bound.+This parameter is optional.+The default is `true`.`incu`::-Include the upper bound. This parameter is optional. The default is `true`.+Include the upper bound.+This parameter is optional.+The default is `true`.*Examples*@@ -445,7 +296,8 @@ Include the upper bound. This parameter is optional. The default is `true`.fq={!frange l=0 u=2.2} sum(user_ranking,editor_ranking)-----Both of these examples restrict the results by a range of values found in a declared field or a function query. In the second example, we're doing a sum calculation, and then defining only values between 0 and 2.2 should be returned to the user.+Both of these examples restrict the results by a range of values found in a declared field or a function query.+In the second example, we're doing a sum calculation, and then defining only values between 0 and 2.2 should be returned to the user.For more information about range queries over functions, see Yonik Seeley's introductory blog post https://lucidworks.com/2009/07/06/ranges-over-functions-in-solr-14/[Ranges over Functions in Solr 1.4].@@ -463,28 +315,35 @@ for its performance characteristics so `indexed=true` will perform better for mo=== Graph Query Parameters`to`::-The field name of matching documents to inspect to identify outgoing edges for graph traversal. Defaults to `edge_ids`.+The field name of matching documents to inspect to identify outgoing edges for graph traversal.+Defaults to `edge_ids`.`from`::-The field name to of candidate documents to inspect to identify incoming graph edges. Defaults to `node_id`.+The field name to of candidate documents to inspect to identify incoming graph edges.+Defaults to `node_id`.`traversalFilter`::An optional query that can be supplied to limit the scope of documents that are traversed.`maxDepth`::-Integer specifying how deep the breadth first search of the graph should go beginning with the initial query. Defaults to `-1` (unlimited).+Integer specifying how deep the breadth first search of the graph should go beginning with the initial query.+Defaults to `-1` (unlimited).`returnRoot`::-Boolean to indicate if the documents that matched the original query (to define the starting points for graph) should be included in the final results. Defaults to `true`.+Boolean to indicate if the documents that matched the original query (to define the starting points for graph) should be included in the final results.+Defaults to `true`.`returnOnlyLeaf`::-Boolean that indicates if the results of the query should be filtered so that only documents with no outgoing edges are returned. Defaults to `false`.+Boolean that indicates if the results of the query should be filtered so that only documents with no outgoing edges are returned.+Defaults to `false`.-`useAutn`:: Boolean that indicates if an Automatons should be compiled for each iteration of the breadth first search, which may be faster for some graphs. Defaults to `false`.+`useAutn`::+Boolean that indicates if an Automatons should be compiled for each iteration of the breadth first search, which may be faster for some graphs.+Defaults to `false`.=== Graph Query Limitations-The `graph` parser only works in single node Solr installations, or with <<solrcloud.adoc#,SolrCloud>> collections that use exactly 1 shard.+The `graph` parser only works in single-node Solr installations, or with SolrCloud and user-managed clusters that use exactly 1 shard.=== Graph Query Examples@@ -527,7 +386,8 @@ http://localhost:8983/solr/my_graph/query?fl=id&q={!graph+from=in_edge+to=out_ed}-----We can also use the `traversalFilter` to limit the graph traversal to only nodes with maximum value of 15 in the `foo` field. In this case that means D, E, and F are excluded – F has a value of `foo=11`, but it is unreachable because the traversal skipped D:+We can also use the `traversalFilter` to limit the graph traversal to only nodes with maximum value of 15 in the `foo` field.+In this case that means D, E, and F are excluded – F has a value of `foo=11`, but it is unreachable because the traversal skipped D:[source,text]----@@ -544,7 +404,8 @@ http://localhost:8983/solr/my_graph/query?fl=id&q={!graph+from=in_edge+to=out_ed}-----The examples shown so far have all used a query for a single document (`"id:A"`) as the root node for the graph traversal, but any query can be used to identify multiple documents to use as root nodes. The next example demonstrates using the `maxDepth` parameter to find all nodes that are at most one edge away from an root node with a value in the `foo` field less then or equal to 10:+The examples shown so far have all used a query for a single document (`"id:A"`) as the root node for the graph traversal, but any query can be used to identify multiple documents to use as root nodes.+The next example demonstrates using the `maxDepth` parameter to find all nodes that are at most one edge away from an root node with a value in the `foo` field less then or equal to 10:[source,text]----@@ -566,7 +427,8 @@ http://localhost:8983/solr/my_graph/query?fl=id&q={!graph+from=in_edge+to=out_ed=== Simplified Models-The Document & Field modeling used in the above examples enumerated all of the outgoing and income edges for each node explicitly, to help demonstrate exactly how the "from" and "to" parameters work, and to give you an idea of what is possible. With multiple sets of fields like these for identifying incoming and outgoing edges, it's possible to model many independent Directed Graphs that contain some or all of the documents in your collection.+The Document & Field modeling used in the above examples enumerated all of the outgoing and income edges for each node explicitly, to help demonstrate exactly how the "from" and "to" parameters work, and to give you an idea of what is possible.+With multiple sets of fields like these for identifying incoming and outgoing edges, it's possible to model many independent Directed Graphs that contain some or all of the documents in your collection.But in many cases it can also be possible to drastically simplify the model used.@@ -605,24 +467,31 @@ http://localhost:8983/solr/alt_graph/query?fl=id&q={!graph+from=id+to=out_edge+m{ "id":"H" } ]}----+== Hash Range Query Parser-The hash range query parser will return documents that have a field that contains a value that would be hashed to a particular range. This is used by the join query when using method=crossCollection. The hash rang query parser has a per segment cache for each field that this query parser will operate on.+The hash range query parser will return documents that have a field that contains a value that would be hashed to a particular range.+This is used by the join query when using method=crossCollection.+The hash rang query parser has a per segment cache for each field that this query parser will operate on.-When specifying a min/max hash range and a field name with the hash range query parser, only documents who contain a field value that hashes into that range will be returned. If you want to query for a very large result set, you can query for various hash ranges to return a fraction of the documents with each range request. In the cross collection join case, the hash_range query parser is used to ensure that each shard only gets the set of join keys that would end up on that shard.+When specifying a min/max hash range and a field name with the hash range query parser, only documents who contain a field value that hashes into that range will be returned.+If you want to query for a very large result set, you can query for various hash ranges to return a fraction of the documents with each range request.+In the cross collection join case, the hash_range query parser is used to ensure that each shard only gets the set of join keys that would end up on that shard.-This query parser uses the MurmurHash3_x86_32. This is the same as the default hashing for the default composite ID router in Solr.+This query parser uses the MurmurHash3_x86_32.+This is the same as the default hashing for the default composite ID router in Solr.=== Hash Range Parameters`f`::-The field name to operate on. This field should have docValues enabled and should be single-valued+The field name to operate on.+This field should have docValues enabled and should be single-valued`l`::-The lower bound of the hash range for the query+The lower bound of the hash range for the query.`u`::-The upper bound for the hash range for the query+The upper bound for the hash range for the query.=== Hash Range Example@@ -633,7 +502,9 @@ The upper bound for the hash range for the query=== Hash Range Cache Config-The hash range query parser uses a special cache to improve the speedup of the queries. The following should be added to the `solrconfig.xml` for the various fields that you want to perform the hash range query on. Note the name of the cache should be the field name prefixed by "hash_".+The hash range query parser uses a special cache to improve the speedup of the queries.+The following should be added to the `solrconfig.xml` for the various fields that you want to perform the hash range query on.+Note the name of the cache should be the field name prefixed by "hash_".[source,xml]----@@ -647,237 +518,9 @@ The hash range query parser uses a special cache to improve the speedup of the q== Join Query Parser-The Join query parser allows users to run queries that normalize relationships between documents.-Solr runs a subquery of the user's choosing (the `v` parameter), identifies all the values that matching documents have in a field of interest (the `from` parameter), and then returns documents where those values are contained in a second field of interest (the `to` parameter).--In practice, these semantics are much like "inner queries" in a SQL engine.-As an example, consider the Solr query below:--[source,text]------/solr/techproducts/select?q={!join from=manu_id_s to=id}title:ipod-------This query, which returns a document for each manufacturer that makes a product with "ipod" in the title, is semantically identical to the SQL query below:--[source,text]------SELECT *-FROM techproducts-WHERE id IN (- SELECT manu_id_s- FROM techproducts- WHERE title='ipod'- )-------The join operation is done on a term basis, so the `from` and `to` fields must use compatible field types.-For example: joining between a `StrField` and a `IntPointField` will not work.-Likewise joining between a `StrField` and a `TextField` that uses `LowerCaseFilterFactory` will only work for values that are already lower cased in the string field.--=== Parameters--This query parser takes the following parameters:--`from`::-The name of a field which contains values to look for in the "to" field.-Can be single or multi-valued, but must have a field type compatible with the field represented in the "to" field.-This parameter is required.--`to`::-The name of a field whose value(s) will be checked against those found in the "from" field.-Can be single or multi-valued, but must have a field type compatible with the "from" field.-This parameter is required.+The Join Query Parser allows users to run queries that normalize relationships between documents, similar to SQL-style joins.-`fromIndex`::-The name of the index to run the "from" query (`v` parameter) on and where "from" values are gathered.-Must be located on the same node as the core processing the request.-This parameter is optional; it defaults to the value of the processing core if not specified.-See <<Joining Across Single Shard Collections,Joining Across Single Shard Collections>> or <<Cross Collection Join,Cross Collection Join>> below for more information.--`score`::-An optional parameter that instructs Solr to return information about the "from" query scores.-The value of this parameter controls what type of aggregation information is returned.-Options include `avg` (average), `max` (maximum), `min` (minimum), `total` (total), or `none`.-+-If `method` is not specified but `score` is, then the `dvWithScore` method is used.-If `method` is specified and is not `dvWithScore`, then the `score` value is ignored.-See the `method` parameter documentation below for more details.---`method`::-An optional parameter used to determine which of several query implementations should be used by Solr.-Options are restricted to: `index`, `dvWithScore`, and `topLevelDV`.-If unspecified the default value is `index`, unless the `score` parameter is present which overrides it to `dvWithScore`.-Each implementation has its own performance characteristics, and users are encouraged to experiment to determine which implementation is most performant for their use-case.-Details and performance heuristics are given below.-+-`index` the default `method` unless the `score` parameter is specified.-Uses the terms index structures to process the request.-Performance scales with the cardinality and number of postings (term occurrences) in the "from" field.-Consider this method when the "from" field has low cardinality, when the "to" side returns a large number of documents, or when sporadic post-commit slowdowns cannot be tolerated (this is a disadvantage of other methods that `index` avoids).--+-`dvWithScore` returns an optional "score" statistic alongside result documents.-Uses docValues structures if available, but falls back to the field cache when necessary.-The first access to the field cache slows down the initial requests following a commit and takes up additional space on the JVM heap, so docValues are recommended in most situations.-Performance scales linearly with the number of values matched in the "from" field.-This method must be used if score information is required, and should also be considered when the "from" query matches few documents, regardless of the number of "to" side documents returned.-+-.dvWithScore and single value numerics-[WARNING]-====-The `dvWithScore` method doesn't support single value numeric fields. Users migrating from versions prior to 7.0 are encouraged to change field types to string and rebuild indexes during migration.-====-+-`topLevelDV` can only be used when `to` and `from` fields have docValues data, and does not currently support numeric fields.-Uses top-level docValues data structures to find results.-These data structures outperform other methods as the number of values matched in the `from` field grows high.-But they are also expensive to build and need to be lazily populated after each commit, causing a sometimes-noticeable slowdown on the first query to use them after each commit.-If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.-Consider this method when the "from" query matches a large number of documents and the "to" result set is small to moderate in size, but only if sporadic post-commit slowness is tolerable.--=== Joining Across Single Shard Collections--You can also specify a `fromIndex` parameter to join with a field from another core or a single shard collection. If running in SolrCloud mode, then the collection specified in the `fromIndex` parameter must have a single shard and a replica on all Solr nodes where the collection you're joining to has a replica.--Let's consider an example where you want to use a Solr join query to filter movies by directors that have won an Oscar. Specifically, imagine we have two collections with the following fields:--*movies*: id, title, director_id, ...--*movie_directors*: id, name, has_oscar, ...--To filter movies by directors that have won an Oscar using a Solr join on the *movie_directors* collection, you can send the following filter query to the *movies* collection:--[source,text]------fq={!join from=id fromIndex=movie_directors to=director_id}has_oscar:true-------Notice that the query criteria of the filter (`has_oscar:true`) is based on a field in the collection specified using `fromIndex`. Keep in mind that you cannot return fields from the `fromIndex` collection using join queries, you can only use the fields for filtering results in the "to" collection (movies).--Next, let's understand how these collections need to be deployed in your cluster. Imagine the *movies* collection is deployed to a four node SolrCloud cluster and has two shards with a replication factor of two. Specifically, the *movies* collection has replicas on the following four nodes:--node 1: movies_shard1_replica1--node 2: movies_shard1_replica2--node 3: movies_shard2_replica1--node 4: movies_shard2_replica2--To use the *movie_directors* collection in Solr join queries with the *movies* collection, it needs to have a replica on each of the four nodes. In other words, *movie_directors* must have one shard and replication factor of four:--node 1: movie_directors_shard1_replica1--node 2: movie_directors_shard1_replica2--node 3: movie_directors_shard1_replica3--node 4: movie_directors_shard1_replica4--At query time, the `JoinQParser` will access the local replica of the *movie_directors* collection to perform the join. If a local replica is not available or active, then the query will fail. At this point, it should be clear that since you're limited to a single shard and the data must be replicated across all nodes where it is needed, this approach works better with smaller data sets where there is a one-to-many relationship between the from collection and the to collection. Moreover, if you add a replica to the to collection, then you also need to add a replica for the from collection.--For more information, Erick Erickson has written a blog post about join performance titled https://lucidworks.com/2012/06/20/solr-and-joins/[Solr and Joins].--=== Cross Collection Join-The Cross Collection Join Filter is a method for the join parser that will execute a query against a remote Solr collection to get back a set of join keys that will be used to as a filter query against the local Solr collection.--The crossCollection join query will create an CrossCollectionQuery object.-The CrossCollectionQuery will first query a remote Solr collection and get back a streaming expression result of the join keys.-As the join keys are streamed to the node, a bitset of the matching documents in the local index is built up.-This avoids keeping the full set of join keys in memory at any given time.-This bitset is then inserted into the filter cache upon successful execution as with the normal behavior of the Solr filter cache.--If the local index is sharded according to the join key field, the cross collection join can leverage a secondary query parser called the "hash_range" query parser.-The hash_range query parser is responsible for returning only the documents that hash to a given range of values.-This allows the CrossCollectionQuery to query the remote Solr collection and return only the join keys that would match a specific shard in the local Solr collection.-This has the benefit of making sure that network traffic doesn't increase as the number of shards increases and allows for much greater scalability.--The CrossCollection join query works with both String and Point types of fields.-The fields that are being used for the join key must be single-valued and have docValues enabled.--It's advised to shard the local collection by the join key as this allows for the optimization mentioned above to be utilized.--The cross collection join queries should not be generally used as part of the `q` parameter, but rather it is designed to be used as a filter query (`fq` parameter) to ensure proper caching.--The remote Solr collection that is being queried should have a single-valued field for the join key with docValues enabled.--The remote Solr collection does not have any specific sharding requirements.--==== Join Query Parser Definition in solrconfig.xml--The cross collection join has some configuration options that can be specified in `solrconfig.xml`.--`routerField`::-If the documents are routed to shards using the CompositeID router by the join field, then that field name should be specified in the configuration here. This will allow the parser to optimize the resulting HashRange query.--`solrUrl`::-If specified, this array of strings specifies the white listed Solr URLs that you can pass to the solrUrl query parameter. Without this configuration the solrUrl parameter cannot be used. This restriction is necessary to prevent an attacker from using Solr to explore the network.--[source,xml]------ <queryParser name="join" class="org.apache.solr.search.JoinQParserPlugin">- <str name="routerField">product_id_s</str>- <arr name="allowSolrUrls">- <str>http://othersolr.example.com:8983/solr</str>- </arr>- </queryParser>-------==== Cross Collection Join Query Parameters--`fromIndex`::-The name of the external Solr collection to be queried to retrieve the set of join key values (required).--`zkHost`::-The connection string to be used to connect to ZooKeeper. `zkHost` and `solrUrl` are both optional parameters, and at most one of them should be specified. If neither `zkHost` nor `solrUrl` are specified, the local ZooKeeper cluster will be used. (optional).--`solrUrl`::-The URL of the external Solr node to be queried. Must be a character for character exact match of a whitelisted url. (optional, disabled by default for security).--`from`::-The join key field name in the external collection (required).--`to`::-The join key field name in the local collection.--`v`::-The query substituted in as a local param. This is the query string that will match documents in the remote collection.--`routed`::-If `true`, the cross collection join query will use each shard's hash range to determine the set of join keys to retrieve for that shard.-This parameter improves the performance of the cross-collection join, but it depends on the local collection being routed by the `to` field.-If this parameter is not specified, the cross collection join query will try to determine the correct value automatically.--`ttl`::-The length of time that a cross collection join query in the cache will be considered valid, in seconds.-Defaults to `3600` (one hour).-The cross collection join query will not be aware of changes to the remote collection, so if the remote collection is updated, cached cross collection queries may give inaccurate results.-After the `ttl` period has expired, the cross collection join query will re-execute the join against the remote collection.--Other Parameters::-Any normal Solr query parameter can also be specified/passed through as a local param.--==== Cross Collection Query Examples--[source,text]------http://localhost:8983/solr/localCollection/query?fl=id&q={!join method="crossCollection" fromIndex="otherCollection" from="fromField" to="toField" v="*:*"}-------== Lucene Query Parser--The `LuceneQParser` extends the `QParserPlugin` by parsing Solr's variant on the Lucene QueryParser syntax. This is effectively the same query parser that is used in Lucene. It uses the operators `q.op`, the default operator ("OR" or "AND") and `df`, the default field name.--Example:--[source,text]------{!lucene q.op=AND df=text}myfield:foo +bar -baz-------For more information about the syntax for the Lucene Query Parser, see the {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/classic/package-summary.html[Classic QueryParser javadocs].+Details of this query parser are in the section <<join-query-parser.adoc#,Join Query Parser>>.== Learning To Rank Query Parser@@ -894,7 +537,10 @@ Details about using the `LTRQParserPlugin` can be found in the <<learning-to-ran== Max Score Query Parser-The `MaxScoreQParser` extends the `LuceneQParser` but returns the Max score from the clauses. It does this by wrapping all `SHOULD` clauses in a `DisjunctionMaxQuery` with `tie=1.0`. Any `MUST` or `PROHIBITED` clauses are passed through as-is. Non-boolean queries, e.g., NumericRange falls-through to the `LuceneQParser` parser behavior.+The `MaxScoreQParser` extends the `LuceneQParser` but returns the Max score from the clauses.+It does this by wrapping all `SHOULD` clauses in a `DisjunctionMaxQuery` with `tie=1.0`.+Any `MUST` or `PROHIBITED` clauses are passed through as-is.+Non-boolean queries, e.g., NumericRange falls-through to the `LuceneQParser` parser behavior.Example:@@ -911,7 +557,8 @@ Information about how to use this query parser is with the documentation about M== Nested Query Parser-The `NestedParser` extends the `QParserPlugin` and creates a nested query, with the ability for that query to redefine its type via local parameters. This is useful in specifying defaults in configuration and letting clients indirectly reference them.+The `NestedParser` extends the `QParserPlugin` and creates a nested query, with the ability for that query to redefine its type via local params.+This is useful in specifying defaults in configuration and letting clients indirectly reference them.Example:@@ -920,7 +567,9 @@ Example:{!query defType=func v=$q1}-----If the `q1` parameter is price, then the query would be a function query on the price field. If the `q1` parameter is \{!lucene}inStock:true}} then a term query is created from the Lucene syntax string that matches documents with `inStock=true`. These parameters would be defined in `solrconfig.xml`, in the `defaults` section:+If the `q1` parameter is price, then the query would be a function query on the price field.+If the `q1` parameter is \{!lucene}inStock:true}} then a term query is created from the Lucene syntax string that matches documents with `inStock=true`.+These parameters would be defined in `solrconfig.xml`, in the `defaults` section:[source,xml]----@@ -934,27 +583,34 @@ For more information about the possibilities of nested queries, see Yonik Seeley== Payload Query Parsers-These query parsers utilize payloads encoded on terms during indexing. Payloads can be encoded on terms using either the `DelimitedPayloadTokenFilter` or the `NumericPayloadTokenFilter`.+These query parsers utilize payloads encoded on terms during indexing.+Payloads can be encoded on terms using either the `DelimitedPayloadTokenFilter` or the `NumericPayloadTokenFilter`.=== Payload Score Parser-`PayloadScoreQParser` incorporates each matching term's numeric (integer or float) payloads into the scores. The main query is parsed from the field type's query analysis into a `SpanQuery` based on the value of the `operator` parameter below.+`PayloadScoreQParser` incorporates each matching term's numeric (integer or float) payloads into the scores.+The main query is parsed from the field type's query analysis into a `SpanQuery` based on the value of the `operator` parameter below.This parser accepts the following parameters:`f`::-The field to use. This parameter is required.+The field to use.+This parameter is required.`func`::-The payload function. The options are: `min`, `max`, `average`, or `sum`. This parameter is required.+The payload function.+The options are: `min`, `max`, `average`, or `sum`.+This parameter is required.`operator`::-A search operator. The options are+A search operator.+The options are:* `or` will generate either a `SpanTermQuery` or a `SpanOrQuery` depending on the number of tokens emitted.* `phrase` will generate either `SpanTermQuery` or an ordered, zero slop `SpanNearQuery`, depending on how many tokens are emitted.`includeSpanScore`::-If `true`, multiples the computed payload factor by the score of the original query. If `false`, the default, the computed payload factor is the score.+If `true`, multiples the computed payload factor by the score of the original query.+If `false`, the default, the computed payload factor is the score.*Examples*@@ -966,9 +622,14 @@ If `true`, multiples the computed payload factor by the score of the original qu=== Payload Check Parser-`PayloadCheckQParser` only matches when the matching terms also have the specified relationship to the payloads. The default relationship is equals, however, inequality matching can also be performed. The main query, for both of these parsers, is parsed straightforwardly from the field type's query analysis into a `SpanQuery`. The generated `SpanQuery` will be either a `SpanTermQuery` or an ordered, zero slop `SpanNearQuery`, depending on how many tokens are emitted. The net effect is that the main query always operates in a manner similar to a phrase query in the standard Lucene parser (thus ignoring any value for `q.op`).+`PayloadCheckQParser` only matches when the matching terms also have the specified relationship to the payloads.+The default relationship is equals, however, inequality matching can also be performed.+The main query, for both of these parsers, is parsed straightforwardly from the field type's query analysis into a `SpanQuery`.+The generated `SpanQuery` will be either a `SpanTermQuery` or an ordered, zero slop `SpanNearQuery`, depending on how many tokens are emitted.+The net effect is that the main query always operates in a manner similar to a phrase query in the standard Lucene parser (thus ignoring any value for `q.op`).-NOTE: If when the field analysis is applied to the query, it alters the number of tokens, the final number of tokens must match the number of payloads supplied in the `payloads` parameter. If there is a mismatch between the number of query tokens, and the number of payload values supplied with this query, the query will not match.+NOTE: If when the field analysis is applied to the query, it alters the number of tokens, the final number of tokens must match the number of payloads supplied in the `payloads` parameter.+If there is a mismatch between the number of query tokens, and the number of payload values supplied with this query, the query will not match.This parser accepts the following parameters:@@ -976,10 +637,13 @@ This parser accepts the following parameters:The field to use (required).`payloads`::-A space-separated list of payloads to be compared with payloads in the matching tokens from the document (required). Each specified payload will be encoded using the encoder determined from the field type prior to matching. Integer, float and identity (string) encodings are supported with the same meanings as for DelimitedPayloadTokenFilter.+A space-separated list of payloads to be compared with payloads in the matching tokens from the document (required).+Each specified payload will be encoded using the encoder determined from the field type prior to matching.+Integer, float and identity (string) encodings are supported with the same meanings as for DelimitedPayloadTokenFilter.`op`::-The inequality operation to apply to the payload check. All operations require that consecutive tokens derived from the analysis of the query match consecutive tokens in the document, and additionally the payloads on the document tokens must be:+The inequality operation to apply to the payload check.+All operations require that consecutive tokens derived from the analysis of the query match consecutive tokens in the document, and additionally the payloads on the document tokens must be:* `eq` - equal to the specified payloads (default)* `gt` - greater than the specified payloads* `lt` - less than the specified payloads@@ -1003,9 +667,11 @@ Find all documents with the phrase "foo bar" where term "foo" has a payload grea== Prefix Query Parser-`PrefixQParser` extends the `QParserPlugin` by creating a prefix query from the input value. Currently no analysis or value transformation is done to create this prefix query.+`PrefixQParser` extends the `QParserPlugin` by creating a prefix query from the input value.+Currently no analysis or value transformation is done to create this prefix query.-The parameter is `f`, the field. The string after the prefix declaration is treated as a wildcard query.+The parameter is `f`, the field.+The string after the prefix declaration is treated as a wildcard query.Example:@@ -1018,7 +684,8 @@ This would be generally equivalent to the Lucene query parser expression `myfiel== Raw Query Parser-`RawQParser` extends the `QParserPlugin` by creating a term query from the input value without any text analysis or transformation. This is useful in debugging, or when raw terms are returned from the terms component (this is not the default).+`RawQParser` extends the `QParserPlugin` by creating a term query from the input value without any text analysis or transformation.+This is useful in debugging, or when raw terms are returned from the terms component (this is not the default).The only parameter is `f`, which defines the field to search.@@ -1054,16 +721,17 @@ Details about using the `ReRankQParserPlugin` can be found in the <<query-re-ran== Simple Query Parser-The Simple query parser in Solr is based on Lucene's SimpleQueryParser. This query parser is designed to allow users to enter queries however they want, and it will do its best to interpret the query and return results.+The Simple query parser in Solr is based on Lucene's SimpleQueryParser.+This query parser is designed to allow users to enter queries however they want, and it will do its best to interpret the query and return results.This parser takes the following parameters:q.operators::-Comma-separated list of names of parsing operators to enable. By default, all operations are enabled, and this parameter can be used to effectively disable specific operators as needed, by excluding them from the list. Passing an empty string with this parameter disables all operators.-+-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed+Comma-separated list of names of parsing operators to enable.+By default, all operations are enabled, and this parameter can be used to effectively disable specific operators as needed, by excluding them from the list.+Passing an empty string with this parameter disables all operators.+-[cols="15,20,50,15",options="header"]+[%autowidth.stretch,options="header"]|===|Name |Operator |Description |Example query|`AND` |`+` |Specifies AND |`token1+token2`@@ -1090,7 +758,9 @@ At the end of terms, specifies a fuzzy query.|===q.op::-Defines the default operator to use if none is defined by the user. Allowed values are `AND` and `OR`. `OR` is used if none is specified.+Defines the default operator to use if none is defined by the user.+Allowed values are `AND` and `OR`.+`OR` is used if none is specified.qf::A list of query fields and boosts to use when building the query.@@ -1098,17 +768,22 @@ A list of query fields and boosts to use when building the query.df::Defines the default field if none is defined in the Schema, or overrides the default field if it is already defined.-Any errors in syntax are ignored and the query parser will interpret queries as best it can. However, this can lead to odd results in some cases.+Any errors in syntax are ignored and the query parser will interpret queries as best it can.+However, this can lead to odd results in some cases.== Spatial Query Parsers-There are two spatial QParsers in Solr: `geofilt` and `bbox`. But there are other ways to query spatially: using the `frange` parser with a distance function, using the standard (lucene) query parser with the range syntax to pick the corners of a rectangle, or with RPT and BBoxField you can use the standard query parser but use a special syntax within quotes that allows you to pick the spatial predicate.+There are two spatial QParsers in Solr: `geofilt` and `bbox`.+But there are other ways to query spatially: using the `frange` parser with a distance function, using the standard (lucene) query parser with the range syntax to pick the corners of a rectangle, or with RPT and BBoxField you can use the standard query parser but use a special syntax within quotes that allows you to pick the spatial predicate.All these options are documented further in the section <<spatial-search.adoc#,Spatial Search>>.== Surround Query Parser-The `SurroundQParser` enables the Surround query syntax, which provides proximity search functionality. There are two positional operators: `w` creates an ordered span query and `n` creates an unordered one. Both operators take a numeric value to indicate distance between two terms. The default is 1, and the maximum is 99.+The `SurroundQParser` enables the Surround query syntax, which provides proximity search functionality.+There are two positional operators: `w` creates an ordered span query and `n` creates an unordered one.+Both operators take a numeric value to indicate distance between two terms.+The default is `1`, and the maximum is `99`.Note that the query string is not analyzed in any way.@@ -1121,7 +796,8 @@ Example:This example finds documents where the terms "foo" and "bar" are no more than 3 terms away from each other (i.e., no more than 2 terms between them).-This query parser will also accept boolean operators (`AND`, `OR`, and `NOT`, in either upper- or lowercase), wildcards, quoting for phrase searches, and boosting. The `w` and `n` operators can also be expressed in upper- or lowercase.+This query parser will also accept boolean operators (`AND`, `OR`, and `NOT`, in either upper- or lowercase), wildcards, quoting for phrase searches, and boosting.+The `w` and `n` operators can also be expressed in upper- or lowercase.The non-unary operators (everything but `NOT`) support both infix `(a AND b AND c)` and prefix `AND(a, b, c)` notation.@@ -1129,9 +805,12 @@ The non-unary operators (everything but `NOT`) support both infix `(a AND b AND`SwitchQParser` is a `QParserPlugin` that acts like a "switch" or "case" statement.-The primary input string is trimmed and then prefixed with `case.` for use as a key to lookup a "switch case" in the parser's local params. If a matching local param is found the resulting parameter value will then be parsed as a subquery, and returned as the parse result.+The primary input string is trimmed and then prefixed with `case.` for use as a key to lookup a "switch case" in the parser's local params.+If a matching local param is found the resulting parameter value will then be parsed as a subquery, and returned as the parse result.-The `case` local param can be optionally be specified as a switch case to match missing (or blank) input strings. The `default` local param can optionally be specified as a default case to use if the input string does not match any other switch case local params. If default is not specified, then any input which does not match a switch case local param will result in a syntax error.+The `case` local param can be optionally be specified as a switch case to match missing (or blank) input strings.+The `default` local param can optionally be specified as a default case to use if the input string does not match any other switch case local params.+If default is not specified, then any input which does not match a switch case local param will result in a syntax error.In the examples below, the result of each query is "XXX":@@ -1183,7 +862,9 @@ Using the example configuration below, clients can optionally specify the custom== Term Query Parser-`TermQParser` extends the `QParserPlugin` by creating a single term query from the input value equivalent to `readableToIndexed()`. This is useful for generating filter queries from the external human readable terms returned by the faceting or terms components. The only parameter is `f`, for the field.+`TermQParser` extends the `QParserPlugin` by creating a single term query from the input value equivalent to `readableToIndexed()`.+This is useful for generating filter queries from the external human readable terms returned by the faceting or terms components.+The only parameter is `f`, for the field.Example:@@ -1192,7 +873,8 @@ Example:{!term f=weight}1.5-----For text fields, no analysis is done since raw terms are already returned from the faceting and terms components. To apply analysis to text fields as well, see the <<Field Query Parser>>, above.+For text fields, no analysis is done since raw terms are already returned from the faceting and terms components.+To apply analysis to text fields as well, see the <<Field Query Parser>>, above.If no analysis or transformation is desired for any type of field, see the <<Raw Query Parser>>, above.@@ -1200,30 +882,56 @@ If no analysis or transformation is desired for any type of field, see the <<Raw`TermsQParser` functions similarly to the <<Term Query Parser,Term Query Parser>> but takes in multiple values separated by commas and returns documents matching any of the specified values.-This can be useful for generating filter queries from the external human readable terms returned by the faceting or terms components, and may be more efficient in some cases than using the <<the-standard-query-parser.adoc#,Standard Query Parser>> to generate a boolean query since the default implementation `method` avoids scoring.+This can be useful for generating filter queries from the external human readable terms returned by the faceting or terms components, and may be more efficient in some cases than using the <<standard-query-parser.adoc#,Standard Query Parser>> to generate a boolean query since the default implementation `method` avoids scoring.This query parser takes the following parameters:`f`::-The field on which to search. This parameter is required.+The field on which to search.+This parameter is required.`separator`::-Separator to use when parsing the input. If set to " " (a single blank space), will trim additional white space from the input terms. Defaults to a comma (`,`).+Separator to use when parsing the input.+If set to " " (a single blank space), will trim additional white space from the input terms.+Defaults to a comma (`,`).`method`::-An optional parameter used to determine which of several query implementations should be used by Solr. Options are restricted to: `termsFilter`, `booleanQuery`, `automaton`, `docValuesTermsFilterPerSegment`, `docValuesTermsFilterTopLevel` or `docValuesTermsFilter`. If unspecified, the default value is `termsFilter`. Each implementation has its own performance characteristics, and users are encouraged to experiment to determine which implementation is most performant for their use-case. Heuristics are given below.+An optional parameter used to determine which of several query implementations should be used by Solr.+-`booleanQuery` creates a `BooleanQuery` representing the request. Scales well with index size, but poorly with the number of terms being searched for.+Options are restricted to: `termsFilter`, `booleanQuery`, `automaton`, `docValuesTermsFilterPerSegment`, `docValuesTermsFilterTopLevel` or `docValuesTermsFilter`.+If unspecified, the default value is `termsFilter`.+-`termsFilter` the default `method`. Uses a `BooleanQuery` or a `TermInSetQuery` depending on the number of terms. Scales well with index size, but only moderately with the number of query terms.+Each implementation has its own performance characteristics, and users are encouraged to experiment to determine which implementation is most performant for their use-case.+Heuristics are given below.+-`docValuesTermsFilter` can only be used on fields with docValues data. The `cache` parameter is false by default. Chooses between the `docValuesTermsFilterTopLevel` and `docValuesTermsFilterPerSegment` methods using the number of query terms as a rough heuristic. Users should typically use this method instead of using `docValuesTermsFilterTopLevel` or `docValuesTermsFilterPerSegment` directly, unless they've done performance testing to validate one of the methods on queries of all sizes. Depending on the implementation picked, this method may rely on expensive data structures which are lazily populated after each commit. If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.+`booleanQuery` creates a `BooleanQuery` representing the request.+Scales well with index size, but poorly with the number of terms being searched for.+-`docValuesTermsFilterTopLevel` can only be used on fields with docValues data. The `cache` parameter is false by default. Uses top-level docValues data structures to find results. These data structures are more efficient as the number of query terms grows high (over several hundred). But they are also expensive to build and need to be populated lazily after each commit, causing a sometimes-noticeable slowdown on the first query after each commit. If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.+`termsFilter` the default `method`.+Uses a `BooleanQuery` or a `TermInSetQuery` depending on the number of terms.+Scales well with index size, but only moderately with the number of query terms.+-`docValuesTermsFilterPerSegment` can only be used on fields with docValues data. The `cache` parameter is false by default. It is more efficient than the "top-level" alternative with small to medium (~500) numbers of query terms, and doesn't suffer a slowdown on queries immediately following a commit (as `docValuesTermsFilterTopLevel` does - see above). But it is less performant on very large numbers of query terms.+`docValuesTermsFilter` can only be used on fields with docValues data.+The `cache` parameter is false by default.+Chooses between the `docValuesTermsFilterTopLevel` and `docValuesTermsFilterPerSegment` methods using the number of query terms as a rough heuristic.+Users should typically use this method instead of using `docValuesTermsFilterTopLevel` or `docValuesTermsFilterPerSegment` directly, unless they've done performance testing to validate one of the methods on queries of all sizes.+Depending on the implementation picked, this method may rely on expensive data structures which are lazily populated after each commit.+If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.+-`automaton` creates an `AutomatonQuery` representing the request with each term forming a union. Scales well with index size and moderately with the number of query terms.+`docValuesTermsFilterTopLevel` can only be used on fields with docValues data.+The `cache` parameter is false by default.+Uses top-level docValues data structures to find results.+These data structures are more efficient as the number of query terms grows high (over several hundred).+But they are also expensive to build and need to be populated lazily after each commit, causing a sometimes-noticeable slowdown on the first query after each commit.+If you commit frequently and your use-case can tolerate a static warming query, consider adding one to `solrconfig.xml` so that this work is done as a part of the commit itself and not attached directly to user requests.+++`docValuesTermsFilterPerSegment` can only be used on fields with docValues data.+The `cache` parameter is false by default.+It is more efficient than the "top-level" alternative with small to medium (~500) numbers of query terms, and doesn't suffer a slowdown on queries immediately following a commit (as `docValuesTermsFilterTopLevel` does - see above).+But it is less performant on very large numbers of query terms.+++`automaton` creates an `AutomatonQuery` representing the request with each term forming a union.+Scales well with index size and moderately with the number of query terms.*Examples*@@ -1239,11 +947,10 @@ An optional parameter used to determine which of several query implementations s== XML Query Parser-The {solr-javadocs}/core/org/apache/solr/search/XmlQParserPlugin.html[XmlQParserPlugin] extends the {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin] and supports the creation of queries from XML. Example:--// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed+The {solr-javadocs}/core/org/apache/solr/search/XmlQParserPlugin.html[XmlQParserPlugin] extends the {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin] and supports the creation of queries from XML.+Example:-[cols="30,70",options="header"]+[%autowidth.stretch,options="header"]|===|Parameter |Value|defType |`xmlparser`@@ -1271,11 +978,10 @@ The {solr-javadocs}/core/org/apache/solr/search/XmlQParserPlugin.html[XmlQParser----|===-The XmlQParser implementation uses the {solr-javadocs}/core/org/apache/solr/search/SolrCoreParser.html[SolrCoreParser] class which extends Lucene's {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/CoreParser.html[CoreParser] class. XML elements are mapped to {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/QueryBuilder.html[QueryBuilder] classes as follows:--// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed+The XmlQParser implementation uses the {solr-javadocs}/core/org/apache/solr/search/SolrCoreParser.html[SolrCoreParser] class which extends Lucene's {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/CoreParser.html[CoreParser] class.+XML elements are mapped to {lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/QueryBuilder.html[QueryBuilder] classes as follows:-[width="100%",cols="30,70",options="header"]+[%autowidth.stretch,cols="30,70",options="header"]|===|XML element |QueryBuilder class|<BooleanQuery> |{lucene-javadocs}/queryparser/org/apache/lucene/queryparser/xml/builders/BooleanQueryBuilder.html[BooleanQueryBuilder]@@ -1299,7 +1005,9 @@ The XmlQParser implementation uses the {solr-javadocs}/core/org/apache/solr/sear=== Customizing XML Query Parser-You can configure your own custom query builders for additional XML elements. The custom builders need to extend the {solr-javadocs}/core/org/apache/solr/search/SolrQueryBuilder.html[SolrQueryBuilder] or the {solr-javadocs}/core/org/apache/solr/search/SolrSpanQueryBuilder.html[SolrSpanQueryBuilder] class. Example `solrconfig.xml` snippet:+You can configure your own custom query builders for additional XML elements.+The custom builders need to extend the {solr-javadocs}/core/org/apache/solr/search/SolrQueryBuilder.html[SolrQueryBuilder] or the {solr-javadocs}/core/org/apache/solr/search/SolrSpanQueryBuilder.html[SolrSpanQueryBuilder] class.+Example `solrconfig.xml` snippet:[source,xml]----diff --git a/solr/solr-ref-guide/src/other-schema-elements.adoc b/solr/solr-ref-guide/src/other-schema-elements.adocdeleted file mode 100644index 04c0ff09c13..00000000000--- a/solr/solr-ref-guide/src/other-schema-elements.adoc+++ /dev/null@@ -1,95 +0,0 @@-= Other Schema Elements-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section describes several other important elements of `schema.xml` not covered in earlier sections.--== Unique Key--The `uniqueKey` element specifies which field is a unique identifier for documents. Although `uniqueKey` is not required, it is nearly always warranted by your application design. For example, `uniqueKey` should be used if you will ever update a document in the index.--You can define the unique key field by naming it:--[source,xml]------<uniqueKey>id</uniqueKey>-------Schema defaults and `copyFields` cannot be used to populate the `uniqueKey` field. The `fieldType` of `uniqueKey` must not be analyzed and must not be any of the `*PointField` types. You can use `UUIDUpdateProcessorFactory` to have `uniqueKey` values generated automatically.--Further, the operation will fail if the `uniqueKey` field is used, but is multivalued (or inherits the multivalue-ness from the `fieldtype`). However, `uniqueKey` will continue to work, as long as the field is properly used.---== Similarity--Similarity is a Lucene class used to score a document in searching.--Each collection has one "global" Similarity, and by default Solr uses an implicit {solr-javadocs}/core/org/apache/solr/search/similarities/SchemaSimilarityFactory.html[`SchemaSimilarityFactory`] which allows individual field types to be configured with a "per-type" specific Similarity and implicitly uses `BM25Similarity` for any field type which does not have an explicit Similarity.--This default behavior can be overridden by declaring a top level `<similarity/>` element in your `schema.xml`, outside of any single field type. This similarity declaration can either refer directly to the name of a class with a no-argument constructor, such as in this example showing `BM25Similarity`:--[source,xml]------<similarity class="solr.BM25SimilarityFactory"/>-------or by referencing a `SimilarityFactory` implementation, which may take optional initialization parameters:--[source,xml]------<similarity class="solr.DFRSimilarityFactory">- <str name="basicModel">P</str>- <str name="afterEffect">L</str>- <str name="normalization">H2</str>- <float name="c">7</float>-</similarity>-------In most cases, specifying global level similarity like this will cause an error if your `schema.xml` also includes field type specific `<similarity/>` declarations. One key exception to this is that you may explicitly declare a {solr-javadocs}/core/org/apache/solr/search/similarities/SchemaSimilarityFactory.html[`SchemaSimilarityFactory`] and specify what that default behavior will be for all field types that do not declare an explicit Similarity using the name of field type (specified by `defaultSimFromFieldType`) that _is_ configured with a specific similarity:--[source,xml]------<similarity class="solr.SchemaSimilarityFactory">- <str name="defaultSimFromFieldType">text_dfr</str>-</similarity>-<fieldType name="text_dfr" class="solr.TextField">- <analyzer ... />- <similarity class="solr.DFRSimilarityFactory">- <str name="basicModel">I(F)</str>- <str name="afterEffect">B</str>- <str name="normalization">H3</str>- <float name="mu">900</float>- </similarity>-</fieldType>-<fieldType name="text_ib" class="solr.TextField">- <analyzer ... />- <similarity class="solr.IBSimilarityFactory">- <str name="distribution">SPL</str>- <str name="lambda">DF</str>- <str name="normalization">H2</str>- </similarity>-</fieldType>-<fieldType name="text_other" class="solr.TextField">- <analyzer ... />-</fieldType>-------In the example above `IBSimilarityFactory` (using the Information-Based model) will be used for any fields of type `text_ib`, while `DFRSimilarityFactory` (divergence from random) will be used for any fields of type `text_dfr`, as well as any fields using a type that does not explicitly specify a `<similarity/>`.--If `SchemaSimilarityFactory` is explicitly declared without configuring a `defaultSimFromFieldType`, then `BM25Similarity` is implicitly used as the default for `luceneMatchVersion >= 8.0.0` and otherwise `LegacyBM25Similarity` is used to mimic the same BM25 formula that was the default in those versions.--In addition to the various factories mentioned on this page, there are several other similarity implementations that can be used such as the `SweetSpotSimilarityFactory`, `ClassicSimilarityFactory`, `LegacyBM25SimilarityFactory` etc. For details, see the Solr Javadocs for the {solr-javadocs}/core/org/apache/solr/schema/SimilarityFactory.html[similarity factories].diff --git a/solr/solr-ref-guide/src/overview-of-documents-fields-and-schema-design.adoc b/solr/solr-ref-guide/src/overview-of-documents-fields-and-schema-design.adocdeleted file mode 100644index 8fd9bdf98c9..00000000000--- a/solr/solr-ref-guide/src/overview-of-documents-fields-and-schema-design.adoc+++ /dev/null@@ -1,61 +0,0 @@-= Overview of Documents, Fields, and Schema Design-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The fundamental premise of Solr is simple. You give it a lot of information, then later you can ask it questions and find the piece of information you want. The part where you feed in all the information is called _indexing_ or _updating_. When you ask a question, it's called a _query_.--One way to understand how Solr works is to think of a loose-leaf book of recipes. Every time you add a recipe to the book, you update the index at the back. You list each ingredient and the page number of the recipe you just added. Suppose you add one hundred recipes. Using the index, you can very quickly find all the recipes that use garbanzo beans, or artichokes, or coffee, as an ingredient. Using the index is much faster than looking through each recipe one by one. Imagine a book of one thousand recipes, or one million.--Solr allows you to build an index with many different fields, or types of entries. The example above shows how to build an index with just one field, `ingredients`. You could have other fields in the index for the recipe's cooking style, like `Asian`, `Cajun`, or `vegan`, and you could have an index field for preparation times. Solr can answer questions like "What Cajun-style recipes that have blood oranges as an ingredient can be prepared in fewer than 30 minutes?"--The schema is the place where you tell Solr how it should build indexes from input documents.--== How Solr Sees the World--Solr's basic unit of information is a _document_, which is a set of data that describes something. A recipe document would contain the ingredients, the instructions, the preparation time, the cooking time, the tools needed, and so on. A document about a person, for example, might contain the person's name, biography, favorite color, and shoe size. A document about a book could contain the title, author, year of publication, number of pages, and so on.--In the Solr universe, documents are composed of _fields_, which are more specific pieces of information. Shoe size could be a field. First name and last name could be fields.--Fields can contain different kinds of data. A name field, for example, is text (character data). A shoe size field might be a floating point number so that it could contain values like 6 and 9.5. Obviously, the definition of fields is flexible (you could define a shoe size field as a text field rather than a floating point number, for example), but if you define your fields correctly, Solr will be able to interpret them correctly and your users will get better results when they perform a query.--You can tell Solr about the kind of data a field contains by specifying its _field type_. The field type tells Solr how to interpret the field and how it can be queried.--When you add a document, Solr takes the information in the document's fields and adds that information to an index. When you perform a query, Solr can quickly consult the index and return the matching documents.--== Field Analysis--_Field analysis_ tells Solr what to do with incoming data when building an index. A more accurate name for this process would be _processing_ or even _digestion_, but the official name is _analysis_.--Consider, for example, a biography field in a person document. Every word of the biography must be indexed so that you can quickly find people whose lives have had anything to do with ketchup, or dragonflies, or cryptography.--However, a biography will likely contains lots of words you don't care about and don't want clogging up your index—words like "the", "a", "to", and so forth. Furthermore, suppose the biography contains the word "Ketchup", capitalized at the beginning of a sentence. If a user makes a query for "ketchup", you want Solr to tell you about the person even though the biography contains the capitalized word.--The solution to both these problems is field analysis. For the biography field, you can tell Solr how to break apart the biography into words. You can tell Solr that you want to make all the words lower case, and you can tell Solr to remove accents marks.--Field analysis is an important part of a field type. <<understanding-analyzers-tokenizers-and-filters.adoc#,Understanding Analyzers, Tokenizers, and Filters>> is a detailed description of field analysis.--== Solr's Schema File--Solr stores details about the field types and fields it is expected to understand in a schema file. The name and location of this file may vary depending on how you initially configured Solr or if you modified it later.--* `managed-schema` is the name for the schema file Solr uses by default to support making Schema changes at runtime via the <<schema-api.adoc#,Schema API>>, or <<schemaless-mode.adoc#,Schemaless Mode>> features. You may <<schema-factory-definition-in-solrconfig.adoc#,explicitly configure the managed schema features>> to use an alternative filename if you choose, but the contents of the files are still updated automatically by Solr.-* `schema.xml` is the traditional name for a schema file which can be edited manually by users who use the <<schema-factory-definition-in-solrconfig.adoc#,`ClassicIndexSchemaFactory`>>.-* If you are using SolrCloud you may not be able to find any file by these names on the local filesystem. You will only be able to see the schema through the Schema API (if enabled) or through the Solr Admin UI's <<cloud-screens.adoc#,Cloud Screens>>.--Whichever name of the file in use in your installation, the structure of the file is not changed. However, the way you interact with the file will change. If you are using the managed schema, it is expected that you only interact with the file with the Schema API, and never make manual edits. If you do not use the managed schema, you will only be able to make manual edits to the file, the Schema API will not support any modifications.--Note that if you are not using the Schema API yet you do use SolrCloud, you will need to interact with `schema.xml` through ZooKeeper using upconfig and downconfig commands to make a local copy and upload your changes. The options for doing this are described in <<solr-control-script-reference.adoc#,Solr Control Script Reference>> and <<using-zookeeper-to-manage-configuration-files.adoc#,Using ZooKeeper to Manage Configuration Files>>.diff --git a/solr/solr-ref-guide/src/overview-of-searching-in-solr.adoc b/solr/solr-ref-guide/src/overview-of-searching-in-solr.adocdeleted file mode 100644index 7ed075d9e63..00000000000--- a/solr/solr-ref-guide/src/overview-of-searching-in-solr.adoc+++ /dev/null@@ -1,59 +0,0 @@-= Overview of Searching in Solr-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr offers a rich, flexible set of features for search. To understand the extent of this flexibility, it's helpful to begin with an overview of the steps and components involved in a Solr search.--When a user runs a search in Solr, the search query is processed by a *request handler*. A request handler is a Solr plug-in that defines the logic to be used when Solr processes a request. Solr supports a variety of request handlers. Some are designed for processing search queries, while others manage tasks such as index replication.--Search applications select a particular request handler by default. In addition, applications can be configured to allow users to override the default selection in preference of a different request handler.--To process a search query, a request handler calls a *query parser*, which interprets the terms and parameters of a query. Different query parsers support different syntax. Solr's default query parser is known as the <<the-standard-query-parser.adoc#,Standard Query Parser>>,or more commonly just the "lucene" query parser. Solr also includes the <<the-dismax-query-parser.adoc#,DisMax>>query parser, and the <<the-extended-dismax-query-parser.adoc#,Extended DisMax>> (eDisMax) query parser. The <<the-standard-query-parser.adoc#,standard>> query parser's syntax allows for greater precision in searches, but the DisMax query parser is much more tolerant of errors. The DisMax query parser is designed to provide an experience similar to that of popular search engines such as Google, which rarely display syntax errors to users. The Extended DisMax query parser is an improved version of DisMax that handles the full Lucene query syntax while still tolerating syntax errors. It also includes several additional features.--In addition, there are <<common-query-parameters.adoc#,common query parameters>> that are accepted by all query parsers.--Input to a query parser can include:--* search strings---that is, _terms_ to search for in the index-* _parameters for fine-tuning the query_ by increasing the importance of particular strings or fields, by applying Boolean logic among the search terms, or by excluding content from the search results-* _parameters for controlling the presentation of the query response_, such as specifying the order in which results are to be presented or limiting the response to particular fields of the search application's schema.--Search parameters may also specify a *filter query*. As part of a search response, a filter query runs a query against the entire index and caches the results. Because Solr allocates a separate cache for filter queries, the strategic use of filter queries can improve search performance. (Despite their similar names, query filters are not related to analysis filters. Filter queries perform queries at search time against data already in the index, while analysis filters, such as Tokenizers, parse content for indexing, following specified rules).--A search query can request that certain terms be highlighted in the search response; that is, the selected terms will be displayed in colored boxes so that they "jump out" on the screen of search results. <<highlighting.adoc#,*Highlighting*>> can make it easier to find relevant passages in long documents returned in a search. Solr supports multi-term highlighting. Solr includes a rich set of search parameters for controlling how terms are highlighted.--Search responses can also be configured to include *snippets* (document excerpts) featuring highlighted text. Popular search engines such as Google and Yahoo! return snippets in their search results: 3-4 lines of text offering a description of a search result.--To help users zero in on the content they're looking for, Solr supports two special ways of grouping search results to aid further exploration: faceting and clustering.--<<faceting.adoc#,*Faceting*>> is the arrangement of search results into categories (which are based on indexed terms). Within each category, Solr reports on the number of hits for relevant term, which is called a facet constraint. Faceting makes it easy for users to explore search results on sites such as movie sites and product review sites, where there are many categories and many items within a category.--The screen shot below shows an example of faceting from the CNET Web site (CBS Interactive Inc.), which was the first site to use Solr.--image::images/overview-of-searching-in-solr/worddav88969a784fb8a63d8c46e9c043f5f953.png[image,width=600,height=300]--Faceting makes use of fields defined when the search applications were indexed. In the example above, these fields include categories of information that are useful for describing digital cameras: manufacturer, resolution, and zoom range.--*Clustering* groups search results by similarities discovered when a search is executed, rather than when content is indexed. The results of clustering often lack the neat hierarchical organization found in faceted search results, but clustering can be useful nonetheless. It can reveal unexpected commonalities among search results, and it can help users rule out content that isn't pertinent to what they're really searching for.--Solr also supports a feature called <<morelikethis.adoc#,MoreLikeThis>>, which enables users to submit new queries that focus on particular terms returned in an earlier query. MoreLikeThis queries can make use of faceting or clustering to provide additional aid to users.--A Solr component called a <<response-writers.adoc#,*response writer*>> manages the final presentation of the query response. Solr includes a variety of response writers, including an <<response-writers.adoc#standard-xml-response-writer,XML Response Writer>> and a <<response-writers.adoc#json-response-writer,JSON Response Writer>>.--The diagram below summarizes some key elements of the search process.--image::images/overview-of-searching-in-solr/worddav16392965e726e04513a21641fabad474.png[image,width=624,height=401]diff --git a/solr/solr-ref-guide/src/overview-of-the-solr-admin-ui.adoc b/solr/solr-ref-guide/src/overview-of-the-solr-admin-ui.adocdeleted file mode 100644index 1eb93780e55..00000000000--- a/solr/solr-ref-guide/src/overview-of-the-solr-admin-ui.adoc+++ /dev/null@@ -1,83 +0,0 @@-= Overview of the Solr Admin UI-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr features a Web interface that provides online access to many Solr configuration options and features.--== Dashboard--Accessing the URL `\http://hostname:8983/solr/` will show the main dashboard, which is divided into two parts.--.Solr Dashboard-image::images/overview-of-the-solr-admin-ui/dashboard.png[image,height=400]--The left-side of the screen is a menu under the Solr logo that provides the navigation through the screens of the UI.--The first set of links are for system-level information and configuration and provide access to <<logging.adoc#,Logging>>, <<collections-core-admin.adoc#,Collection/Core Administration>>, and <<java-properties.adoc#,Java Properties>>, among other things.--At the end of this information is at least one pulldown listing Solr cores configured for this instance.-On <<solrcloud.adoc#,SolrCloud>> nodes, an additional pulldown list shows all collections in this cluster.-Clicking on a collection or core name shows secondary menus of information for the specified collection or core, such as a <<schema-browser-screen.adoc#,Schema Browser>>, <<files-screen.adoc#,Config Files>>, <<plugins-stats-screen.adoc#,Plugins & Statistics>>, and an ability to perform <<query-screen.adoc#,Queries>> on indexed data.--The left-side navigation appears on every screen, while the center changes to the detail of the option selected.-The Dashboard shows several information items in the center of the screen, including system uptime, the version being run, system-level data, JVM arguments, and the security plugins enabled (if any).--Under the covers, the Solr Admin UI uses the same HTTP APIs available to all clients to access Solr-related data to drive an external interface.--[TIP]-====-The path to the Solr Admin UI given above is `\http://hostname:port/solr`, which redirects to `\http://hostname:port/solr/\#/`. A convenience redirect is also supported, so simply accessing the Admin UI at `\http://hostname:port/` will also redirect to `\http://hostname:port/solr/#/`.-====--== Login Screen--If authentication has been enabled, Solr will present a login screen to unauthenticated users before allowing them further access to the Admin UI.--.Login Screen-image::images/overview-of-the-solr-admin-ui/login.png[]--This login screen currently only works with Basic Authentication.-See the section <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>> for- details on how to configure Solr to use this method of authentication.--Once logged in, the left-hand navigation will show the current user with an option to logout.--If Kerberos is enabled and the user has a valid ticket, the login screen will be skipped.-However, if the user does not have a valid ticket, they will see a message that they need to obtain a valid ticket before continuing.--== Getting Assistance--At the bottom of each screen of the Admin UI is a set of links that can be used to get more assistance with configuring and using Solr.--.Assistance icons-image::images/getting-assistance/Assistance.png[image]--These icons include the following links.--// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="25,75",options="header"]-|===-|Link |Description-|Documentation |Navigates to the Apache Solr documentation hosted on https://solr.apache.org/.-|Issue Tracker |Navigates to the JIRA issue tracking server for the Apache Solr project. This server resides at https://issues.apache.org/jira/browse/SOLR.-|IRC Channel |Navigates to Solr's http://en.wikipedia.org/wiki/Internet_Relay_Chat[IRC] live-chat room: http://webchat.freenode.net/?channels=#solr.-|Community forum |Navigates to the Apache Wiki page which has further information about ways to engage in the Solr User community mailing lists: https://cwiki.apache.org/confluence/display/solr/UsingMailingLists.-|Solr Query Syntax |Navigates to the section <<query-syntax-and-parsing.adoc#,Query Syntax and Parsing>> in this Reference Guide.-|===--These links cannot be modified without editing the `index.html` in the `server/solr/solr-webapp` directory that contains the Admin UI files.diff --git a/solr/solr-ref-guide/src/pagination-of-results.adoc b/solr/solr-ref-guide/src/pagination-of-results.adocindex f76eca9a9ea..ca7db7bb9c1 100644--- a/solr/solr-ref-guide/src/pagination-of-results.adoc+++ b/solr/solr-ref-guide/src/pagination-of-results.adoc@@ -21,7 +21,8 @@ In most search applications, the "top" matching results (sorted by score, or somIn many applications the UI for these sorted results are displayed to the user in "pages" containing a fixed number of matching results, and users don't typically look at results past the first few pages worth of results.== Basic Pagination-In Solr, this basic paginated searching is supported using the `start` and `rows` parameters, and performance of this common behaviour can be tuned by utilizing the <<query-settings-in-solrconfig.adoc#queryresultcache,`queryResultCache`>> and adjusting the <<query-settings-in-solrconfig.adoc#queryresultwindowsize,`queryResultWindowSize`>> configuration options based on your expected page sizes.++In Solr, this basic paginated searching is supported using the `start` and `rows` parameters, and performance of this common behaviour can be tuned by utilizing the <<caches-warming.adoc#query-result-cache,`queryResultCache`>> and adjusting the <<caches-warming.adoc#queryresultwindowsize-element,`queryResultWindowSize`>> configuration options based on your expected page sizes.=== Basic Pagination Examples@@ -72,9 +73,11 @@ In typical situations these impacts from index changes on paginated searching doIn some situations, the results of a Solr search are not destined for a simple paginated user interface.-When you wish to fetch a very large number of sorted results from Solr to feed into an external system, using very large values for the `start` or `rows` parameters can be very inefficient. Pagination using `start` and `rows` not only require Solr to compute (and sort) in memory all of the matching documents that should be fetched for the current page, but also all of the documents that would have appeared on previous pages.+When you wish to fetch a very large number of sorted results from Solr to feed into an external system, using very large values for the `start` or `rows` parameters can be very inefficient.+Pagination using `start` and `rows` not only require Solr to compute (and sort) in memory all of the matching documents that should be fetched for the current page, but also all of the documents that would have appeared on previous pages.-While a request for `start=0&rows=1000000` may be obviously inefficient because it requires Solr to maintain & sort in memory a set of 1 million documents, likewise a request for `start=999000&rows=1000` is equally inefficient for the same reasons. Solr can't compute which matching document is the 999001st result in sorted order, without first determining what the first 999000 matching sorted results are.+While a request for `start=0&rows=1000000` may be obviously inefficient because it requires Solr to maintain & sort in memory a set of 1 million documents, likewise a request for `start=999000&rows=1000` is equally inefficient for the same reasons.+Solr can't compute which matching document is the 999001st result in sorted order, without first determining what the first 999000 matching sorted results are.If the index is distributed, which is common when running in SolrCloud mode, then 1 million documents are retrieved from *each shard*. For a ten shard index, ten million entries must be retrieved and sorted to figure out the 1000 documents that match those query parameters.@@ -82,13 +85,18 @@ If the index is distributed, which is common when running in SolrCloud mode, theAs an alternative to increasing the "start" parameter to request subsequent pages of sorted results, Solr supports using a "Cursor" to scan through results.-Cursors in Solr are a logical concept that doesn't involve caching any state information on the server. Instead the sort values of the last document returned to the client are used to compute a "mark" representing a logical point in the ordered space of sort values. That "mark" can be specified in the parameters of subsequent requests to tell Solr where to continue.+Cursors in Solr are a logical concept that doesn't involve caching any state information on the server.+Instead the sort values of the last document returned to the client are used to compute a "mark" representing a logical point in the ordered space of sort values.+That "mark" can be specified in the parameters of subsequent requests to tell Solr where to continue.=== Using Cursors-To use a cursor with Solr, specify a `cursorMark` parameter with the value of `*`. You can think of this being analogous to `start=0` as a way to tell Solr "start at the beginning of my sorted results" except that it also informs Solr that you want to use a Cursor.+To use a cursor with Solr, specify a `cursorMark` parameter with the value of `*`.+You can think of this being analogous to `start=0` as a way to tell Solr "start at the beginning of my sorted results" except that it also informs Solr that you want to use a Cursor.-In addition to returning the top N sorted results (where you can control N using the `rows` parameter) the Solr response will also include an encoded String named `nextCursorMark`. You then take the `nextCursorMark` String value from the response, and pass it back to Solr as the `cursorMark` parameter for your next request. You can repeat this process until you've fetched as many docs as you want, or until the `nextCursorMark` returned matches the `cursorMark` you've already specified -- indicating that there are no more results.+In addition to returning the top N sorted results (where you can control N using the `rows` parameter) the Solr response will also include an encoded String named `nextCursorMark`.+You then take the `nextCursorMark` String value from the response, and pass it back to Solr as the `cursorMark` parameter for your next request.+You can repeat this process until you've fetched as many docs as you want, or until the `nextCursorMark` returned matches the `cursorMark` you've already specified -- indicating that there are no more results.=== Constraints when using Cursors@@ -105,11 +113,14 @@ When the `responseHeader` no longer includes `"partialResults": true`, and `curs. `sort` clauses must include the uniqueKey field (either `asc` or `desc`).+If `id` is your uniqueKey field, then sort parameters like `id asc` and `name asc, id desc` would both work fine, but `name asc` by itself would not-. Sorts including <<working-with-dates.adoc#,Date Math>> based functions that involve calculations relative to `NOW` will cause confusing results, since every document will get a new sort value on every subsequent request. This can easily result in cursors that never end, and constantly return the same documents over and over – even if the documents are never updated.+. Sorts including <<date-formatting-math.adoc#,Date Math>> based functions that involve calculations relative to `NOW` will cause confusing results, since every document will get a new sort value on every subsequent request.+This can easily result in cursors that never end, and constantly return the same documents over and over – even if the documents are never updated.+-In this situation, choose & re-use a fixed value for the <<working-with-dates.adoc#now,`NOW` request parameter>> in all of your cursor requests.+In this situation, choose & re-use a fixed value for the <<date-formatting-math.adoc#now,`NOW` request parameter>> in all of your cursor requests.-Cursor mark values are computed based on the sort values of each document in the result, which means multiple documents with identical sort values will produce identical Cursor mark values if one of them is the last document on a page of results. In that situation, the subsequent request using that `cursorMark` would not know which of the documents with the identical mark values should be skipped. Requiring that the uniqueKey field be used as a clause in the sort criteria guarantees that a deterministic ordering will be returned, and that every `cursorMark` value will identify a unique point in the sequence of documents.+Cursor mark values are computed based on the sort values of each document in the result, which means multiple documents with identical sort values will produce identical Cursor mark values if one of them is the last document on a page of results.+In that situation, the subsequent request using that `cursorMark` would not know which of the documents with the identical mark values should be skipped.+Requiring that the uniqueKey field be used as a clause in the sort criteria guarantees that a deterministic ordering will be returned, and that every `cursorMark` value will identify a unique point in the sequence of documents.=== Cursor Examples@@ -209,7 +220,10 @@ while (! done) {=== How Cursors are Affected by Index Updates-Unlike basic pagination, Cursor pagination does not rely on using an absolute "offset" into the completed sorted list of matching documents. Instead, the `cursorMark` specified in a request encapsulates information about the *relative* position of the last document returned, based on the *absolute* sort values of that document. This means that the impact of index modifications is much smaller when using a cursor compared to basic pagination. Consider the same example index described when discussing basic pagination:+Unlike basic pagination, Cursor pagination does not rely on using an absolute "offset" into the completed sorted list of matching documents.+Instead, the `cursorMark` specified in a request encapsulates information about the *relative* position of the last document returned, based on the *absolute* sort values of that document.+This means that the impact of index modifications is much smaller when using a cursor compared to basic pagination.+Consider the same example index described when discussing basic pagination:[options="header",%autowidth,width="30%"]|===@@ -246,9 +260,12 @@ In this situation, you will be guaranteed that each document is only returned on=== "Tailing" a Cursor-Because Cursor requests are stateless, and the cursorMark values encapsulate the absolute sort values of the last document returned from a search, it's possible to "continue" fetching additional results from a cursor that has already reached its end. If new documents are added (or existing documents are updated) to the end of the results.+Because Cursor requests are stateless, and the cursorMark values encapsulate the absolute sort values of the last document returned from a search, it's possible to "continue" fetching additional results from a cursor that has already reached its end.+If new documents are added (or existing documents are updated) to the end of the results.-You can think of this as similar to using something like "tail -f" in Unix. The most common examples of how this can be useful is when you have a "timestamp" field recording when a document has been added/updated in your index. Client applications can continuously poll a cursor using a `sort=timestamp asc, id asc` for documents matching a query, and always be notified when a document is added or updated matching the request criteria.+You can think of this as similar to using something like "tail -f" in Unix.+The most common examples of how this can be useful is when you have a "timestamp" field recording when a document has been added/updated in your index.+Client applications can continuously poll a cursor using a `sort=timestamp asc, id asc` for documents matching a query, and always be notified when a document is added or updated matching the request criteria.Another common example is when you have uniqueKey values that always increase as new documents are created, and you can continuously poll a cursor using `sort=id asc` to be notified about new documents.diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adocindex 1c5b30052b6..8f8511a1656 100644--- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc+++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc@@ -1,5 +1,9 @@= Parallel SQL Interface-:page-children: solr-jdbc-dbvisualizer, solr-jdbc-squirrel-sql, solr-jdbc-apache-zeppelin, solr-jdbc-python-jython, solr-jdbc-r+:page-children: jdbc-dbvisualizer, \+ jdbc-squirrel, \+ jdbc-zeppelin, \+ jdbc-python-jython, \+ jdbc-r// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -19,19 +23,25 @@Solr's Parallel SQL Interface brings the power of SQL to SolrCloud.-The SQL interface seamlessly combines SQL with Solr's full-text search capabilities. Both MapReduce style and JSON Facet API aggregations are supported, which means the SQL interface can be used to support both *high query volume* and *high cardinality* use cases.+The SQL interface seamlessly combines SQL with Solr's full-text search capabilities.+Both MapReduce style and JSON Facet API aggregations are supported, which means the SQL interface can be used to support both *high query volume* and *high cardinality* use cases.== SQL Architecture-The SQL interface allows sending a SQL query to Solr and getting documents streamed back in response. Under the covers, Solr's SQL interface uses the Apache Calcite SQL engine to translate SQL queries to physical query plans implemented as <<streaming-expressions.adoc#,Streaming Expressions>>.+The SQL interface allows sending a SQL query to Solr and getting documents streamed back in response.+Under the covers, Solr's SQL interface uses the Apache Calcite SQL engine to translate SQL queries to physical query plans implemented as <<streaming-expressions.adoc#,Streaming Expressions>>.=== Solr Collections and DB Tables-In a standard `SELECT` statement such as `SELECT <expressions> FROM <table>`, the table names correspond to Solr collection names. Table names are case insensitive.+In a standard `SELECT` statement such as `SELECT <expressions> FROM <table>`, the table names correspond to Solr collection names.+Table names are case insensitive.-Column names in the SQL query map directly to fields in the Solr index for the collection being queried. These identifiers are case sensitive. Aliases are supported, and can be referenced in the `ORDER BY` clause.+Column names in the SQL query map directly to fields in the Solr index for the collection being queried.+These identifiers are case sensitive.+Aliases are supported, and can be referenced in the `ORDER BY` clause.-The `*` syntax to indicate all fields is not supported in either limited or unlimited queries. The `score` field can be used only with queries that contain a `LIMIT` clause.+The `*` syntax to indicate all fields is not supported in either limited or unlimited queries.+The `score` field can be used only with queries that contain a `LIMIT` clause.For example, we could index Solr's sample documents and then construct an SQL query like this:@@ -40,7 +50,8 @@ For example, we could index Solr's sample documents and then construct an SQL quSELECT manu as mfr, price as retail FROM techproducts-----The collection in Solr we are using is "techproducts", and we've asked for the "manu" and "price" fields to be returned and aliased with new names. While this example does not use those aliases, we could build on this to ORDER BY one or more of those fields.+The collection in Solr we are using is "techproducts", and we've asked for the "manu" and "price" fields to be returned and aliased with new names.+While this example does not use those aliases, we could build on this to ORDER BY one or more of those fields.More information about how to structure SQL queries for Solr is included in the section <<Solr SQL Syntax>> below.@@ -48,12 +59,22 @@ More information about how to structure SQL queries for Solr is included in theThe SQL feature of Solr can work with aggregations (grouping of results) in two ways:-* `facet`: This is the *default* aggregation mode, which uses the JSON Facet API or StatsComponent for aggregations. In this scenario the aggregations logic is pushed down into the search engine and only the aggregates are sent across the network. This is Solr's normal mode of operation. This is fast when the cardinality of GROUP BY fields is low to moderate. But it breaks down when you have high cardinality fields in the GROUP BY field.-* `map_reduce`: This implementation shuffles tuples to worker nodes and performs the aggregation on the worker nodes. It involves sorting and partitioning the entire result set and sending it to worker nodes. In this approach the tuples arrive at the worker nodes sorted by the GROUP BY fields. The worker nodes can then rollup the aggregates one group at a time. This allows for unlimited cardinality aggregation, but you pay the price of sending the entire result set across the network to worker nodes.+* `facet`: This is the *default* aggregation mode, which uses the JSON Facet API or StatsComponent for aggregations.+In this scenario the aggregations logic is pushed down into the search engine and only the aggregates are sent across the network.+This is Solr's normal mode of operation.+This is fast when the cardinality of GROUP BY fields is low to moderate.+But it breaks down when you have high cardinality fields in the GROUP BY field.+* `map_reduce`: This implementation shuffles tuples to worker nodes and performs the aggregation on the worker nodes.+It involves sorting and partitioning the entire result set and sending it to worker nodes.+In this approach the tuples arrive at the worker nodes sorted by the GROUP BY fields.+The worker nodes can then rollup the aggregates one group at a time.+This allows for unlimited cardinality aggregation, but you pay the price of sending the entire result set across the network to worker nodes.These modes are defined with the `aggregationMode` property when sending the request to Solr.-As noted, the choice between aggregation modes depends on the cardinality of the fields you are working with. If you have low-to-moderate cardinality in the fields you are grouping by, the 'facet' aggregation mode will give you a higher performance because only the final groups are returned, very similar to how facets work today. If, however, you have high cardinality in the fields, the "map_reduce" aggregation mode with worker nodes provide a much more performant option.+The choice between aggregation modes depends on the cardinality of the fields you are working with.+If you have low-to-moderate cardinality in the fields you are grouping by, the 'facet' aggregation mode will give you a higher performance because only the final groups are returned, very similar to how facets work today.+If, however, you have high cardinality in the fields, the "map_reduce" aggregation mode with worker nodes provide a much more performant option.== Configuration@@ -62,7 +83,11 @@ The request handlers used for the SQL interface are configured to load implicitl[[sql-request-handler]]=== /sql Request Handler-The `/sql` handler is the front end of the Parallel SQL interface. All SQL queries are sent to the `/sql` handler to be processed. The handler also coordinates the distributed MapReduce jobs when running `GROUP BY` and `SELECT DISTINCT` queries in `map_reduce` mode. By default the `/sql` handler will choose worker nodes from its own collection to handle the distributed operations. In this default scenario the collection where the `/sql` handler resides acts as the default worker collection for MapReduce queries.+The `/sql` handler is the front end of the Parallel SQL interface.+All SQL queries are sent to the `/sql` handler to be processed.+The handler also coordinates the distributed MapReduce jobs when running `GROUP BY` and `SELECT DISTINCT` queries in `map_reduce` mode.+By default the `/sql` handler will choose worker nodes from its own collection to handle the distributed operations.+In this default scenario the collection where the `/sql` handler resides acts as the default worker collection for MapReduce queries.By default, the `/sql` request handler is configured as an implicit handler, meaning that it is always enabled in every Solr installation and no further configuration is required.@@ -77,20 +102,27 @@ Consequently, you do not need to grant explicit permission to the `/admin/luke`[IMPORTANT]====-As described below in the section <<Best Practices>>, you may want to set up a separate collection for parallelized SQL queries. If you have high cardinality fields and a large amount of data, please be sure to review that section and consider using a separate collection.+As described below in the section <<Best Practices>>, you may want to set up a separate collection for parallelized SQL queries.+If you have high cardinality fields and a large amount of data, please be sure to review that section and consider using a separate collection.======= /stream and /export Request Handlers-The Streaming API is an extensible parallel computing framework for SolrCloud. <<streaming-expressions.adoc#,Streaming Expressions>> provide a query language and a serialization format for the Streaming API.+The Streaming API is an extensible parallel computing framework for SolrCloud.+<<streaming-expressions.adoc#,Streaming Expressions>> provide a query language and a serialization format for the Streaming API.-The Streaming API provides support for fast MapReduce allowing it to perform parallel relational algebra on extremely large data sets. Under the covers the SQL interface parses SQL queries using the Apache Calcite SQL Parser. It then translates the queries to the parallel query plan. The parallel query plan is expressed using the Streaming API and Streaming Expressions.+The Streaming API provides support for fast MapReduce allowing it to perform parallel relational algebra on extremely large data sets.+Under the covers the SQL interface parses SQL queries using the Apache Calcite SQL Parser.+It then translates the queries to the parallel query plan.+The parallel query plan is expressed using the Streaming API and Streaming Expressions.Like the `/sql` request handler, the `/stream` and `/export` request handlers are configured as implicit handlers, and no further configuration is required.=== Fields-In some cases, fields used in SQL queries must be configured as DocValue fields. If queries are unlimited, all fields must be DocValue fields. If queries are limited (with the `limit` clause) then fields do not have to be have DocValues enabled.+In some cases, fields used in SQL queries must be configured as DocValue fields.+If queries are unlimited, all fields must be DocValue fields.+If queries are limited (with the `limit` clause) then fields do not have to be have DocValues enabled.=== Sending Queries@@ -98,7 +130,8 @@ The SQL Interface provides a basic JDBC driver and an HTTP interface to perform=== JDBC Driver-The JDBC Driver ships with SolrJ. Below is sample code for creating a connection and executing a query with the JDBC driver:+The JDBC Driver ships with SolrJ.+Below is sample code for creating a connection and executing a query with the JDBC driver:[source,java]----@@ -119,7 +152,10 @@ try {}-----The connection URL must contain the `zkHost` and the `collection` parameters. The collection must be a valid SolrCloud collection at the specified ZooKeeper host. The collection must also be configured with the `/sql` handler. The `aggregationMode` and `numWorkers` parameters are optional.+The connection URL must contain the `zkHost` and the `collection` parameters.+The collection must be a valid SolrCloud collection at the specified ZooKeeper host.+The collection must also be configured with the `/sql` handler.+The `aggregationMode` and `numWorkers` parameters are optional.=== HTTP Interface@@ -151,7 +187,8 @@ Below is sample result set:}-----Notice that the result set is an array of tuples with key/value pairs that match the SQL column list. The final tuple contains the EOF flag which signals the end of the stream.+Notice that the result set is an array of tuples with key/value pairs that match the SQL column list.+The final tuple contains the EOF flag which signals the end of the stream.== Solr SQL Syntax@@ -160,7 +197,8 @@ Solr supports a broad range of SQL syntax..SQL Parser is Case Insensitive[IMPORTANT]====-The SQL parser being used by Solr to translate the SQL statements is case insensitive. However, for ease of reading, all examples on this page use capitalized keywords.+The SQL parser being used by Solr to translate the SQL statements is case insensitive.+However, for ease of reading, all examples on this page use capitalized keywords.====.SELECT * is not supported@@ -171,7 +209,9 @@ The SQL parser being used by Solr does not support the SELECT * syntax, you must=== Escaping Reserved Words-The SQL parser will return an error if a reserved word is used in the SQL query. Reserved words can be escaped and included in the query using the back tick. For example:+The SQL parser will return an error if a reserved word is used in the SQL query.+Reserved words can be escaped and included in the query using the back tick.+For example:[source,sql]----@@ -180,7 +220,10 @@ select `from` from emails=== SELECT Statements-Solr supports limited and unlimited select queries. The syntax between the two types of queries are identical except for the `LIMIT` clause in the SQL statement. However, they have very different execution plans and different requirements for how the data is stored. The sections below explores both types of queries.+Solr supports limited and unlimited select queries.+The syntax between the two types of queries are identical except for the `LIMIT` clause in the SQL statement.+However, they have very different execution plans and different requirements for how the data is stored.+The sections below explores both types of queries.==== Basic SELECT statement with LIMIT@@ -197,10 +240,13 @@ We've covered many syntax options with this example, so let's walk through what'[IMPORTANT]====-The WHERE clause must have a field on one side of the predicate. Two constants `(5 < 10`) or two fields `(fielda > fieldb)` is not supported. Subqueries are also not supported.+The WHERE clause must have a field on one side of the predicate.+Two constants `(5 < 10`) or two fields `(fielda > fieldb)` is not supported.+Subqueries are also not supported.====-The `WHERE` clause allows Solr's search syntax to be injected into the SQL query. In the example:+The `WHERE` clause allows Solr's search syntax to be injected into the SQL query.+In the example:[source,sql]----@@ -209,7 +255,8 @@ WHERE fieldC = 'term1 term2'The predicate above will execute a full text search for the phrase 'term1 term2' in fieldC.-To execute a non-phrase query, simply add parens inside of the single quotes. For example:+To execute a non-phrase query, simply add parens inside of the single quotes.+For example:[source,sql]----@@ -267,11 +314,14 @@ The parallel SQL interface supports and pushes down most common SQL operators, s=== ORDER BY Clause-The `ORDER BY` clause maps directly to Solr fields. Multiple `ORDER BY` fields and directions are supported.+The `ORDER BY` clause maps directly to Solr fields.+Multiple `ORDER BY` fields and directions are supported.The `score` field is accepted in the `ORDER BY` clause in queries where a limit is specified.-If the `ORDER BY` clause contains the exact fields in the `GROUP BY` clause, then there is no-limit placed on the returned results. If the `ORDER BY` clause contains different fields than the `GROUP BY` clause, a limit of 100 is automatically applied. To increase this limit you must specify a value in the `LIMIT` clause.+If the `ORDER BY` clause contains the exact fields in the `GROUP BY` clause, then there is no-limit placed on the returned results.+If the `ORDER BY` clause contains different fields than the `GROUP BY` clause, a limit of 100 is automatically applied.+To increase this limit you must specify a value in the `LIMIT` clause.Order by fields are case sensitive.@@ -291,23 +341,30 @@ SQL `OFFSET` is not intended for deep-paging type use cases.=== LIMIT Clause-Limits the result set to the specified size. In the example above the clause `LIMIT 100` will limit the result set to 100 records.+Limits the result set to the specified size.+In the example above the clause `LIMIT 100` will limit the result set to 100 records.There are a few differences to note between limited and unlimited queries:-* Limited queries support `score` in the field list and `ORDER BY`. Unlimited queries do not.-* Limited queries allow any stored field in the field list. Unlimited queries require the fields to be stored as a DocValues field.-* Limited queries allow any indexed field in the `ORDER BY` list. Unlimited queries require the fields to be stored as a DocValues field.+* Limited queries support `score` in the field list and `ORDER BY`.+Unlimited queries do not.+* Limited queries allow any stored field in the field list.+Unlimited queries require the fields to be stored as a DocValues field.+* Limited queries allow any indexed field in the `ORDER BY` list.+Unlimited queries require the fields to be stored as a DocValues field.=== SELECT DISTINCT QueriesThe SQL interface supports both MapReduce and Facet implementations for `SELECT DISTINCT` queries.-The MapReduce implementation shuffles tuples to worker nodes where the Distinct operation is performed. This implementation can perform the Distinct operation over extremely high cardinality fields.+The MapReduce implementation shuffles tuples to worker nodes where the Distinct operation is performed.+This implementation can perform the Distinct operation over extremely high cardinality fields.-The Facet implementation pushes down the Distinct operation into the search engine using the JSON Facet API. This implementation is designed for high performance, high QPS scenarios on low-to-moderate cardinality fields.+The Facet implementation pushes down the Distinct operation into the search engine using the JSON Facet API.+This implementation is designed for high performance, high QPS scenarios on low-to-moderate cardinality fields.-The `aggregationMode` parameter is available in the both the JDBC driver and HTTP interface to choose the underlying implementation (`map_reduce` or `facet`). The SQL syntax is identical for both implementations:+The `aggregationMode` parameter is available in the both the JDBC driver and HTTP interface to choose the underlying implementation (`map_reduce` or `facet`).+The SQL syntax is identical for both implementations:[source,sql]----@@ -319,7 +376,7 @@ SELECT distinct fieldA as fa, fieldB as fb FROM tableA ORDER BY fa desc, fb descThe SQL interface supports simple statistics calculated on numeric fields.The supported functions are `COUNT(*)`, `COUNT(DISTINCT field)`, `APPROX_COUNT_DISTINCT(field)`, `MIN`, `MAX`, `SUM`, and `AVG`.-Because these functions never require data to be shuffled, the aggregations are pushed down into the search engine and are generated by the <<the-stats-component.adoc#,StatsComponent>>.+Because these functions never require data to be shuffled, the aggregations are pushed down into the search engine and are generated by the <<stats-component.adoc#,StatsComponent>>.[source,sql]----@@ -332,7 +389,9 @@ The `APPROX_COUNT_DISTINCT` metric uses Solr's HyperLogLog (hll) statistical funThe SQL interface also supports `GROUP BY` aggregate queries.-As with `SELECT DISTINCT` queries, the SQL interface supports both a MapReduce implementation and a Facet implementation. The MapReduce implementation can build aggregations over extremely high cardinality fields. The Facet implementations provides high performance aggregation over fields with moderate levels of cardinality.+As with `SELECT DISTINCT` queries, the SQL interface supports both a MapReduce implementation and a Facet implementation.+The MapReduce implementation can build aggregations over extremely high cardinality fields.+The Facet implementations provides high performance aggregation over fields with moderate levels of cardinality.==== Basic GROUP BY with Aggregates@@ -353,7 +412,8 @@ Let's break this down into pieces:==== Column Identifiers and Aliases-The Column Identifiers can contain both fields in the Solr index and aggregate functions. The supported aggregate functions are:+The Column Identifiers can contain both fields in the Solr index and aggregate functions.+The supported aggregate functions are:* `COUNT(*)`: Counts the number of records over a set of buckets.* `SUM(field)`: Sums a numeric field over over a set of buckets.@@ -368,7 +428,8 @@ only `COUNT(*)` can be computed for each GROUP BY dimension.=== HAVING Clause-The `HAVING` clause may contain any function listed in the field list. Complex `HAVING` clauses such as this are supported:+The `HAVING` clause may contain any function listed in the field list.+Complex `HAVING` clauses such as this are supported:[source,sql]----@@ -385,58 +446,92 @@ ORDER BY SUM(fieldC) ASC=== Separate Collections-It makes sense to create a separate SolrCloud collection just for the `/sql` handler. This collection can be created using SolrCloud's standard collection API.+It makes sense to create a separate SolrCloud collection just for the `/sql` handler.+This collection can be created using SolrCloud's standard collection API.-Since this collection only exists to handle `/sql` requests and provide a pool of worker nodes, this collection does not need to hold any data. Worker nodes are selected randomly from the entire pool of available nodes in the `/sql` handler's collection. So to grow this collection dynamically replicas can be added to existing shards. New replicas will automatically be put to work after they've been added.+Since this collection only exists to handle `/sql` requests and provide a pool of worker nodes, this collection does not need to hold any data.+Worker nodes are selected randomly from the entire pool of available nodes in the `/sql` handler's collection.+So to grow this collection dynamically replicas can be added to existing shards.+New replicas will automatically be put to work after they've been added.== Parallel SQL Queries-An earlier section describes how the SQL interface translates the SQL statement to a streaming expression. One of the parameters of the request is the `aggregationMode`, which defines if the query should use a MapReduce-like shuffling technique or push the operation down into the search engine.+An earlier section describes how the SQL interface translates the SQL statement to a streaming expression.+One of the parameters of the request is the `aggregationMode`, which defines if the query should use a MapReduce-like shuffling technique or push the operation down into the search engine.=== Parallelized Queries-The Parallel SQL architecture consists of three logical tiers: a *SQL* tier, a *Worker* tier, and a *Data Table* tier. By default the SQL and Worker tiers are collapsed into the same physical SolrCloud collection.+The Parallel SQL architecture consists of three logical tiers: a *SQL* tier, a *Worker* tier, and a *Data Table* tier.+By default the SQL and Worker tiers are collapsed into the same physical SolrCloud collection.==== SQL Tier-The SQL tier is where the `/sql` handler resides. The `/sql` handler takes the SQL query and translates it to a parallel query plan. It then selects worker nodes to execute the plan and sends the query plan to each worker node to be run in parallel.+The SQL tier is where the `/sql` handler resides.+The `/sql` handler takes the SQL query and translates it to a parallel query plan.+It then selects worker nodes to execute the plan and sends the query plan to each worker node to be run in parallel.Once the query plan has been executed by the worker nodes, the `/sql` handler then performs the final merge of the tuples returned by the worker nodes.==== Worker Tier-The workers in the worker tier receive the query plan from the `/sql` handler and execute the parallel query plan. The parallel execution plan includes the queries that need to be made on the Data Table tier and the relational algebra needed to satisfy the query. Each worker node assigned to the query is shuffled 1/N of the tuples from the Data Tables. The worker nodes execute the query plan and stream tuples back to the worker nodes.+The workers in the worker tier receive the query plan from the `/sql` handler and execute the parallel query plan.+The parallel execution plan includes the queries that need to be made on the Data Table tier and the relational algebra needed to satisfy the query.+Each worker node assigned to the query is shuffled 1/N of the tuples from the Data Tables.+The worker nodes execute the query plan and stream tuples back to the worker nodes.==== Data Table Tier-The Data Table tier is where the tables reside. Each table is its own SolrCloud collection. The Data Table layer receives queries from the worker nodes and emits tuples (search results). The Data Table tier also handles the initial sorting and partitioning of tuples sent to the workers. This means the tuples are always sorted and partitioned before they hit the network. The partitioned tuples are sent directly to the correct worker nodes in the proper sort order, ready to be reduced.+The Data Table tier is where the tables reside.+Each table is its own SolrCloud collection.+The Data Table layer receives queries from the worker nodes and emits tuples (search results).+The Data Table tier also handles the initial sorting and partitioning of tuples sent to the workers.+This means the tuples are always sorted and partitioned before they hit the network.+The partitioned tuples are sent directly to the correct worker nodes in the proper sort order, ready to be reduced..How Parallel SQL Queries are Distributedimage::images/parallel-sql-interface/cluster.png[image,width=492,height=250]-The image above shows the three tiers broken out into different SolrCloud collections for clarity. In practice the `/sql` handler and worker collection by default share the same collection.+The image above shows the three tiers broken out into different SolrCloud collections for clarity.+In practice the `/sql` handler and worker collection by default share the same collection.-*Note:* The image shows the network flow for a single Parallel SQL Query (SQL over MapReduce). This network flow is used when `map_reduce` aggregation mode is used for `GROUP BY` aggregations or the `SELECT DISTINCT` query. The traditional SolrCloud network flow (without workers) is used when the `facet` aggregation mode is used.+NOTE: The image shows the network flow for a single Parallel SQL Query (SQL over MapReduce).+This network flow is used when `map_reduce` aggregation mode is used for `GROUP BY` aggregations or the `SELECT DISTINCT` query.+The traditional SolrCloud network flow (without workers) is used when the `facet` aggregation mode is used.Below is a description of the flow:-. The client sends a SQL query to the `/sql` handler. The request is handled by a single `/sql` handler instance.+. The client sends a SQL query to the `/sql` handler.+The request is handled by a single `/sql` handler instance.. The `/sql` handler parses the SQL query and creates the parallel query plan.. The query plan is sent to worker nodes (in green).-. The worker nodes execute the plan in parallel. The diagram shows each worker node contacting a collection in the Data Table tier (in blue).-. The collection in the Data Table tier is the table from the SQL query. Notice that the collection has five shards each with 3 replicas.-. Notice that each worker contacts one replica from each shard. Because there are 5 workers, each worker is returned 1/5 of the search results from each shard. The partitioning is done inside of the Data Table tier so there is no duplication of data across the network.-. Also notice with this design ALL replicas in the data layer are shuffling (sorting & partitioning) data simultaneously. As the number of shards, replicas and workers grows this design allows for a massive amount of computing power to be applied to a single query.-. The worker nodes process the tuples returned from the Data Table tier in parallel. The worker nodes perform the relational algebra needed to satisfy the query plan.+. The worker nodes execute the plan in parallel.+The diagram shows each worker node contacting a collection in the Data Table tier (in blue).+. The collection in the Data Table tier is the table from the SQL query.+Notice that the collection has five shards each with 3 replicas.+. Notice that each worker contacts one replica from each shard.+Because there are 5 workers, each worker is returned 1/5 of the search results from each shard.+The partitioning is done inside of the Data Table tier so there is no duplication of data across the network.+. Also notice with this design ALL replicas in the data layer are shuffling (sorting & partitioning) data simultaneously.+As the number of shards, replicas and workers grows this design allows for a massive amount of computing power to be applied to a single query.+. The worker nodes process the tuples returned from the Data Table tier in parallel.+The worker nodes perform the relational algebra needed to satisfy the query plan.. The worker nodes stream tuples back to the `/sql` handler where the final merge is done, and finally the tuples are streamed back to the client.== SQL Clients and Database Visualization Tools-The SQL interface supports queries sent from SQL clients and database visualization tools such as DbVisualizer and Apache Zeppelin.+The SQL interface supports queries sent from SQL clients and database visualization tools.++This Guide contains documentation to configure the following tools and clients:++* <<jdbc-zeppelin.adoc#,Apache Zeppelin>>+* <<jdbc-dbvisualizer.adoc#,DbVisualizer>>+* <<jdbc-squirrel.adoc#,SQuirreL SQL>>+* <<jdbc-r.adoc#,R>>+* <<jdbc-python-jython.adoc#,Python/Jython>>=== Generic Clients-For most Java based clients, the following jars will need to be placed on the client classpath:+For most Java-based clients, the following jars will need to be placed on the client classpath:* all .jars found in `$SOLR_HOME/dist/solrj-libs`* the SolrJ .jar found at `$SOLR_HOME/dist/solr-solrj-<version>.jar`@@ -451,23 +546,3 @@ jdbc:solr://SOLR_ZK_CONNECTION_STRING?collection=COLLECTION_NAME----There are other parameters that can be optionally added to the connection string like `aggregationMode` and `numWorkers`.--=== DbVisualizer--A step-by-step guide for setting up https://www.dbvis.com/[DbVisualizer] is in the section <<solr-jdbc-dbvisualizer.adoc#,Solr JDBC - DbVisualizer>>.--=== SQuirreL SQL--A step-by-step guide for setting up http://squirrel-sql.sourceforge.net[SQuirreL SQL] is in the section <<solr-jdbc-squirrel-sql.adoc#,Solr JDBC - SQuirreL SQL>>.--=== Apache Zeppelin--A step-by-step guide for setting up http://zeppelin.apache.org/[Apache Zeppelin] is in the section <<solr-jdbc-apache-zeppelin.adoc#,Solr JDBC - Apache Zeppelin>>.--=== Python/Jython--Examples of using Python and Jython for connecting to Solr with the Solr JDBC driver are available in the section <<solr-jdbc-python-jython.adoc#,Solr JDBC - Python/Jython>>.--=== R--Examples of using R for connecting to Solr with the Solr JDBC driver are available in the section <<solr-jdbc-r.adoc#,Solr JDBC - R>>.diff --git a/solr/solr-ref-guide/src/parameter-reference.adoc b/solr/solr-ref-guide/src/parameter-reference.adocdeleted file mode 100644index 2b2a6fd9818..00000000000--- a/solr/solr-ref-guide/src/parameter-reference.adoc+++ /dev/null@@ -1,56 +0,0 @@-= Parameter Reference-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--== Cluster Parameters--`numShards`::-Defaults to `1`. The number of shards to hash documents to. There must be one leader per shard and each leader can have _N_ replicas.---== SolrCloud Instance Parameters--These are set in `solr.xml`, but by default the `host` and `hostContext` parameters are set up to also work with system properties.--`host`::-Defaults to the first local host address found. If the wrong host address is found automatically, you can override the host address with this parameter.--`hostPort`::-Defaults to the port specified via `bin/solr -p <port>`, or `8983` if not specified. The port that Solr is running on. This value is only used when `-DzkRun` is specified without a value (see below), to calculate the default port on which embedded ZooKeeper will run. In the `solr.xml` shipped with Solr, the `hostPort` system property is not referenced, and so is ignored. If you want to run Solr on a non-default port, use `bin/solr -p <port>` rather than specifying `-DhostPort`.--`hostContext`::-Defaults to `solr`. The context path for the Solr web application.--== SolrCloud Instance ZooKeeper Parameters--`zkRun`::-Defaults to `localhost:<hostPort+1000>`. Causes Solr to run an embedded version of ZooKeeper. Set to the address of ZooKeeper on this node; this allows us to know who you are in the list of addresses in the `zkHost` connect string. Use `-DzkRun` (with no value) to get the default value.--`zkHost`::-The host address for ZooKeeper. Usually this is a comma-separated list of addresses to each node in your ZooKeeper ensemble.--`zkClientTimeout`::-Defaults to 15000. The time a client is allowed to not talk to ZooKeeper before its session expires.--`zkRun` and `zkHost` are set up using system properties. `zkClientTimeout` is set up in `solr.xml` by default, but can also be set using a system property.--== SolrCloud Core Parameters--`shard`::-Defaults to being automatically assigned based on numShards. Specifies which shard this core acts as a replica of. `shard` can be specified in the <<defining-core-properties.adoc#,`core.properties`>> for each core.--Additional cloud related parameters are discussed in <<format-of-solr-xml.adoc#,Format of solr.xml>>diff --git a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc b/solr/solr-ref-guide/src/partial-document-updates.adocsimilarity index 77%rename from solr/solr-ref-guide/src/updating-parts-of-documents.adocrename to solr/solr-ref-guide/src/partial-document-updates.adocindex 03d16ac472b..ae71bbdfeba 100644--- a/solr/solr-ref-guide/src/updating-parts-of-documents.adoc+++ b/solr/solr-ref-guide/src/partial-document-updates.adoc@@ -1,4 +1,4 @@-= Updating Parts of Documents+= Partial Document Updates// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -16,21 +16,25 @@// specific language governing permissions and limitations// under the License.-Once you have indexed the content you need in your Solr index, you will want to start thinking about your strategy for dealing with changes to those documents. Solr supports three approaches to updating documents that have only partially changed.+Once you have indexed the content you need in your Solr index, you will want to start thinking about your strategy for dealing with changes to those documents.+Solr supports three approaches to updating documents that have only partially changed.The first is _<<Atomic Updates,atomic updates>>_. This approach allows changing only one or more fields of a document without having to reindex the entire document.The second approach is known as _<<In-Place Updates,in-place updates>>_. This approach is similar to atomic updates (is a subset of atomic updates in some sense), but can be used only for updating single valued non-indexed and non-stored docValue-based numeric fields.-The third approach is known as _<<Optimistic Concurrency,optimistic concurrency>>_ or _optimistic locking_. It is a feature of many NoSQL databases, and allows conditional updating a document based on its version. This approach includes semantics and rules for how to deal with version matches or mis-matches.+The third approach is known as _<<Optimistic Concurrency,optimistic concurrency>>_ or _optimistic locking_. It is a feature of many NoSQL databases, and allows conditional updating a document based on its version.+This approach includes semantics and rules for how to deal with version matches or mis-matches.Atomic Updates (and in-place updates) and Optimistic Concurrency may be used as independent strategies for managing changes to documents, or they may be combined: you can use optimistic concurrency to conditionally apply an atomic update.== Atomic Updates-Solr supports several modifiers that atomically update values of a document. This allows updating only specific fields, which can help speed indexing processes in an environment where speed of index additions is critical to the application.+Solr supports several modifiers that atomically update values of a document.+This allows updating only specific fields, which can help speed indexing processes in an environment where speed of index additions is critical to the application.-To use atomic updates, add a modifier to the field that needs to be updated. The content can be updated, added to, or incrementally increased if the field has a numeric type.+To use atomic updates, add a modifier to the field that needs to be updated.+The content can be updated, added to, or incrementally increased if the field has a numeric type.`set`::Set or replace the field value(s) with the specified value(s), or remove the values if 'null' or empty list is specified as the new value.@@ -38,27 +42,38 @@ Set or replace the field value(s) with the specified value(s), or remove the valMay be specified as a single value, or as a list for multiValued fields.`add`::-Adds the specified values to a multiValued field. May be specified as a single value, or as a list.+Adds the specified values to a multiValued field.+May be specified as a single value, or as a list.`add-distinct`::-Adds the specified values to a multiValued field, only if not already present. May be specified as a single value, or as a list.+Adds the specified values to a multiValued field, only if not already present.+May be specified as a single value, or as a list.`remove`::-Removes (all occurrences of) the specified values from a multiValued field. May be specified as a single value, or as a list.+Removes (all occurrences of) the specified values from a multiValued field.+May be specified as a single value, or as a list.`removeregex`::-Removes all occurrences of the specified regex from a multiValued field. May be specified as a single value, or as a list.+Removes all occurrences of the specified regex from a multiValued field.+May be specified as a single value, or as a list.`inc`::-Increments a numeric value by a specific amount. Must be specified as a single numeric value.+Increments a numeric value by a specific amount.+Must be specified as a single numeric value.=== Field Storage-The core functionality of atomically updating a document requires that all fields in your schema must be configured as stored (`stored="true"`) or docValues (`docValues="true"`) except for fields which are `<copyField/>` destinations, which must be configured as `stored="false"`. Atomic updates are applied to the document represented by the existing stored field values. All data in copyField destinations fields must originate from ONLY copyField sources.+The core functionality of atomically updating a document requires that all fields in your schema must be configured as stored (`stored="true"`) or docValues (`docValues="true"`) except for fields which are `<copyField/>` destinations, which must be configured as `stored="false"`. Atomic updates are applied to the document represented by the existing stored field values.+All data in copyField destinations fields must originate from ONLY copyField sources.-If `<copyField/>` destinations are configured as stored, then Solr will attempt to index both the current value of the field as well as an additional copy from any source fields. If such fields contain some information that comes from the indexing program and some information that comes from copyField, then the information which originally came from the indexing program will be lost when an atomic update is made.+If `<copyField/>` destinations are configured as stored, then Solr will attempt to index both the current value of the field as well as an additional copy from any source fields.+If such fields contain some information that comes from the indexing program and some information that comes from copyField, then the information which originally came from the indexing program will be lost when an atomic update is made.-There are other kinds of derived fields that must also be set so they aren't stored. Some spatial field types, such as BBoxField and LatLonType, use derived fields. CurrencyFieldType also uses derived fields. These types create additional fields which are normally specified by a dynamic field definition. That dynamic field definition must be not stored, or indexing will fail.+There are other kinds of derived fields that must also be set so they aren't stored.+Some spatial field types, such as BBoxField and LatLonType, use derived fields.+CurrencyFieldType also uses derived fields.+These types create additional fields which are normally specified by a dynamic field definition.+That dynamic field definition must be not stored, or indexing will fail.=== Example Updating Part of a Document@@ -109,27 +124,32 @@ Solr supports modifying, adding and removing child documents as part of atomic uSyntactically, updates changing the children of a document are very similar to regular atomic updates of simple fields, as demonstrated by the examples below.Schema and configuration requirements for updating child documents use the same-<<updating-parts-of-documents#field-storage,Field Storage>> requirements for atomic updates mentioned above.+<<Field Storage>> requirements for atomic updates mentioned above.-Under the hood, Solr conceptually behaves similarly for nested documents as for non-nested documents, it's just that it applies to entire trees (from the root) of nested documents instead of stand-alone documents. You can expect more overhead because of this. In-place updates avoid that.+Under the hood, Solr conceptually behaves similarly for nested documents as for non-nested documents, it's just that it applies to entire trees (from the root) of nested documents instead of stand-alone documents.+You can expect more overhead because of this.+In-place updates avoid that.[IMPORTANT]====.Routing Updates using child document Ids in SolrCloudWhen SolrCloud receives document updates, the-<<shards-and-indexing-data-in-solrcloud#document-routing,document routing>> rules for the collection is used to determine which shard should process the update based on the `id` of the document.+<<solrcloud-shards-indexing.adoc#document-routing,document routing>> rules for the collection is used to determine which shard should process the update based on the `id` of the document.When sending an update that specifies the `id` of a _child document_ this will not work by default: the correct shard to send the document to is based on the `id` of the "Root" document for the block the child document is in, *not* the `id` of the child document being updated.Solr offers two solutions to address this:-* Clients may specify a <<shards-and-indexing-data-in-solrcloud#document-routing,`\_route_` parameter>>, with the `id` of the Root document as the parameter value, on each update to tell Solr which shard should process the update.-* Clients can use the (default) `compositeId` router's "prefix routing" feature when indexing all documents to ensure that all child/descendent documents in a Block use the same `id` prefix as the Root level document. This will cause Solr's default routing logic to automatically send child document updates to the correct shard.+* Clients may specify a <<solrcloud-shards-indexing.adoc#document-routing,`\_route_` parameter>>, with the `id` of the Root document as the parameter value, on each update to tell Solr which shard should process the update.+* Clients can use the (default) `compositeId` router's "prefix routing" feature when indexing all documents to ensure that all child/descendent documents in a Block use the same `id` prefix as the Root level document.+This will cause Solr's default routing logic to automatically send child document updates to the correct shard.Furthermore, you _should_ (sometimes _must_) specify the Root document's ID in the `\_root_`-field of this partial update. This is how Solr understands that you are updating a child-document, and not a Root document. Without it, Solr only guesses that the `\_route_` parameter is+field of this partial update.+This is how Solr understands that you are updating a child+document, and not a Root document.+Without it, Solr only guesses that the `\_route_` parameter isequivalent, but it may be absent or not equivalent (e.g., when using the `implicit` router).All of the examples below use `id` prefixes, so no `\_route_` parameter will be necessary for these examples.@@ -211,7 +231,10 @@ curl -X POST 'http://localhost:8983/solr/gettingstarted/update?commit=true' -H '== In-Place Updates-In-place updates are very similar to atomic updates; in some sense, this is a subset of atomic updates. In regular atomic updates, the entire document is reindexed internally during the application of the update. However, in this approach, only the fields to be updated are affected and the rest of the documents are not reindexed internally. Hence, the efficiency of updating in-place is unaffected by the size of the documents that are updated (i.e., number of fields, size of fields, etc.). Apart from these internal differences in efficiency, there is no functional difference between atomic updates and in-place updates.+In-place updates are very similar to atomic updates; in some sense, this is a subset of atomic updates.+In regular atomic updates, the entire document is reindexed internally during the application of the update.+However, in this approach, only the fields to be updated are affected and the rest of the documents are not reindexed internally.+Hence, the efficiency of updating in-place is unaffected by the size of the documents that are updated (i.e., number of fields, size of fields, etc.). Apart from these internal differences in efficiency, there is no functional difference between atomic updates and in-place updates.An atomic update operation is performed using this In-Place approach only when the fields to be updated meet these three conditions:@@ -219,13 +242,15 @@ An atomic update operation is performed using this In-Place approach only when t* the `\_version_` field is also a non-indexed, non-stored single valued docValues field; and,* copy targets of updated fields, if any, are also non-indexed, non-stored single valued numeric docValues fields.-To use in-place updates, add a modifier to the field that needs to be updated. The content can be updated or incrementally increased.+To use in-place updates, add a modifier to the field that needs to be updated.+The content can be updated or incrementally increased.`set`::Set or replace the field value(s) with the specified value(s). May be specified as a single value.`inc`::-Increments a numeric value by a specific amount. Must be specified as a single numeric value.+Increments a numeric value by a specific amount.+Must be specified as a single numeric value.[TIP]====@@ -234,7 +259,6 @@ Increments a numeric value by a specific amount. Must be specified as a single nSince it can be tricky to ensure that all of the necessary conditions are satisfied to ensure that an update can be done In-Place, Solr supports a request parameter option named `update.partial.requireInPlace`.When set to `true`, an atomic update that can not be done In-Place will fail.Users can specify this option when they would prefer that an update request "fail fast" if it can't be done In-Place.-======= In-Place Update Example@@ -286,27 +310,36 @@ The resulting document in our collection will be:== Optimistic Concurrency-Optimistic Concurrency is a feature of Solr that can be used by client applications which update/replace documents to ensure that the document they are replacing/updating has not been concurrently modified by another client application. This feature works by requiring a `\_version_` field on all documents in the index, and comparing that to a `\_version_` specified as part of the update command. By default, Solr's Schema includes a `\_version_` field, and this field is automatically added to each new document.+Optimistic Concurrency is a feature of Solr that can be used by client applications which update/replace documents to ensure that the document they are replacing/updating has not been concurrently modified by another client application.+This feature works by requiring a `\_version_` field on all documents in the index, and comparing that to a `\_version_` specified as part of the update command.+By default, Solr's Schema includes a `\_version_` field, and this field is automatically added to each new document.In general, using optimistic concurrency involves the following work flow:-. A client reads a document. In Solr, one might retrieve the document with the `/get` handler to be sure to have the latest version.+. A client reads a document.+In Solr, one might retrieve the document with the `/get` handler to be sure to have the latest version.. A client changes the document locally.. The client resubmits the changed document to Solr, for example, perhaps with the `/update` handler.. If there is a version conflict (HTTP error code 409), the client starts the process over.-When the client resubmits a changed document to Solr, the `\_version_` can be included with the update to invoke optimistic concurrency control. Specific semantics are used to define when the document should be updated or when to report a conflict.+When the client resubmits a changed document to Solr, the `\_version_` can be included with the update to invoke optimistic concurrency control.+Specific semantics are used to define when the document should be updated or when to report a conflict.* If the content in the `\_version_` field is greater than '1' (i.e., '12345'), then the `\_version_` in the document must match the `\_version_` in the index.-* If the content in the `\_version_` field is equal to '1', then the document must simply exist. In this case, no version matching occurs, but if the document does not exist, the updates will be rejected.-* If the content in the `\_version_` field is less than '0' (i.e., '-1'), then the document must *not* exist. In this case, no version matching occurs, but if the document exists, the updates will be rejected.-* If the content in the `\_version_` field is equal to '0', then it doesn't matter if the versions match or if the document exists or not. If it exists, it will be overwritten; if it does not exist, it will be added.+* If the content in the `\_version_` field is equal to '1', then the document must simply exist.+In this case, no version matching occurs, but if the document does not exist, the updates will be rejected.+* If the content in the `\_version_` field is less than '0' (i.e., '-1'), then the document must *not* exist.+In this case, no version matching occurs, but if the document exists, the updates will be rejected.+* If the content in the `\_version_` field is equal to '0', then it doesn't matter if the versions match or if the document exists or not.+If it exists, it will be overwritten; if it does not exist, it will be added.-When documents are added/updated in batches even a single version conflict may lead to rejecting the entire batch. Use the parameter `failOnVersionConflicts=false` to avoid failure of the entire batch when version constraints fail for one or more documents in a batch.+When documents are added/updated in batches even a single version conflict may lead to rejecting the entire batch.+Use the parameter `failOnVersionConflicts=false` to avoid failure of the entire batch when version constraints fail for one or more documents in a batch.If the document being updated does not include the `\_version_` field, and atomic updates are not being used, the document will be treated by normal Solr rules, which is usually to discard the previous version.-When using Optimistic Concurrency, clients can include an optional `versions=true` request parameter to indicate that the _new_ versions of the documents being added should be included in the response. This allows clients to immediately know what the `\_version_` is of every document added without needing to make a redundant <<realtime-get.adoc#,`/get` request>>.+When using Optimistic Concurrency, clients can include an optional `versions=true` request parameter to indicate that the _new_ versions of the documents being added should be included in the response.+This allows clients to immediately know what the `\_version_` is of every document added without needing to make a redundant <<realtime-get.adoc#,`/get` request>>.Following are some examples using `versions=true` in queries:@@ -344,7 +377,8 @@ $ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/t-----In this example, we've attempted to update document "aaa" but specified the wrong version in the request: `_version_=999999` doesn't match the document version we just got when we added the document. We get an error in response.+In this example, we've attempted to update document "aaa" but specified the wrong version in the request: `_version_=999999` doesn't match the document version we just got when we added the document.+We get an error in response.[source,bash]----@@ -359,7 +393,8 @@ $ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/t"aaa",1632740462042284032]}-----Now we've sent an update with a value for `\_version_` that matches the value in the index, and it succeeds. Because we included `versions=true` to the update request, the response includes a different value for the `\_version_` field.+Now we've sent an update with a value for `\_version_` that matches the value in the index, and it succeeds.+Because we included `versions=true` to the update request, the response includes a different value for the `\_version_` field.[source,bash]----$ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true&omitHeader=true' --data-binary '@@ -377,7 +412,9 @@ $ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/t"code":409}}-----Now we've sent an update with a value for `\_version_` embedded in the document itself. this request fails because we have specified the wrong version. This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc.+Now we've sent an update with a value for `\_version_` embedded in the document itself.+This request fails because we have specified the wrong version.+This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc.[source,bash]----@@ -392,7 +429,9 @@ $ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/t"aaa",1632741942747987968]}-----Now we've sent an update with a value for `\_version_` embedded in the document itself. this request fails because we have specified the wrong version. This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc.+Now we've sent an update with a value for `\_version_` embedded in the document itself.+This request fails because we have specified the wrong version.+This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc.[source,bash]@@ -427,7 +466,9 @@ $ curl -X POST -H 'Content-Type: application/json' 'http://localhost:8983/solr/t"ccc",1632740949182382080]}-----In this example, we have added 2 documents "aaa" and "ccc". As we have specified the parameter `\_version_=-1`, this request should not add the document with the id `aaa` because it already exists. The request succeeds & does not throw any error because the `failOnVersionConflicts=false` parameter is specified. The response shows that only document `ccc` is added and `aaa` is silently ignored.+In this example, we have added 2 documents "aaa" and "ccc". As we have specified the parameter `\_version_=-1`, this request should not add the document with the id `aaa` because it already exists.+The request succeeds & does not throw any error because the `failOnVersionConflicts=false` parameter is specified.+The response shows that only document `ccc` is added and `aaa` is silently ignored.For more information, please also see Yonik Seeley's presentation on https://www.youtube.com/watch?v=WYVM6Wz-XTw[NoSQL features in Solr 4] from Apache Lucene EuroCon 2012.@@ -453,7 +494,8 @@ Once configured, this update processor will reject (HTTP error code 409) any att.versionField vs `\_version_`[IMPORTANT]====-The `\_version_` field used by Solr for its normal optimistic concurrency also has important semantics in how updates are distributed to replicas in SolrCloud, and *MUST* be assigned internally by Solr. Users can not re-purpose that field and specify it as the `versionField` for use in the `DocBasedVersionConstraintsProcessorFactory` configuration.+The `\_version_` field used by Solr for its normal optimistic concurrency also has important semantics in how updates are distributed to replicas in SolrCloud, and *MUST* be assigned internally by Solr.+Users can not re-purpose that field and specify it as the `versionField` for use in the `DocBasedVersionConstraintsProcessorFactory` configuration.====`DocBasedVersionConstraintsProcessorFactory` supports the following additional configuration parameters, which are all optional:diff --git a/solr/solr-ref-guide/src/performance-statistics-reference.adoc b/solr/solr-ref-guide/src/performance-statistics-reference.adocindex 34694cfa2ba..883319e310b 100644--- a/solr/solr-ref-guide/src/performance-statistics-reference.adoc+++ b/solr/solr-ref-guide/src/performance-statistics-reference.adoc@@ -50,9 +50,7 @@ To get request times, specifically, you can send an API request such as:* `\http://localhost:8983/solr/admin/metrics?group=core&prefix=UPDATE./update.requestTimes`* `\http://localhost:8983/solr/admin/metrics?group=core&prefix=QUERY./select.requestTimes`-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="25,75",options="header"]+[%autowidth.stretch,options="header"]|===|Attribute |Description|15minRate |Requests per second received over the past 15 minutes.@@ -118,9 +116,7 @@ You can get all update handler statistics shown in the table below with an API rThe following describes the specific statistics you can get:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="40,60",options="header"]+[%autowidth.stretch,options="header"]|===|Attribute |Description|`UPDATE.updateHandler.adds` |Total number of “add” requests since last commit.@@ -191,9 +187,7 @@ You can get the statistics shown in the table below with an API request such asThe following statistics are available for each of the caches mentioned above:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="25,75",options="header"]+[%autowidth.stretch,options="header"]|===|Attribute |Description|cumulative_evictions |Number of cache evictions across all caches since this node has been running.@@ -220,4 +214,4 @@ When eviction by heap usage is enabled, the following additional statistics are|evictionsRamUsage| Number of cache evictions for the current index searcher because heap usage exceeded maxRamMB.|===-More information on Solr caches is available in the section <<query-settings-in-solrconfig.adoc#,Query Settings in SolrConfig>>.+More information on Solr caches is available in the section <<caches-warming.adoc#,Caches and Query Warming>>.diff --git a/solr/solr-ref-guide/src/phonetic-matching.adoc b/solr/solr-ref-guide/src/phonetic-matching.adocindex f316f1c1f55..566988fd09b 100644--- a/solr/solr-ref-guide/src/phonetic-matching.adoc+++ b/solr/solr-ref-guide/src/phonetic-matching.adoc@@ -22,15 +22,24 @@ For overviews of and comparisons between algorithms, see http://en.wikipedia.org== Beider-Morse Phonetic Matching (BMPM)-For examples of how to use this encoding in your analyzer, see <<filter-descriptions.adoc#beider-morse-filter,Beider Morse Filter>> in the Filter Descriptions section.+For examples of how to use this encoding in your analyzer, see <<filters.adoc#beider-morse-filter,Beider Morse Filter>> in the Filter Descriptions section.-Beider-Morse Phonetic Matching (BMPM) is a "soundalike" tool that lets you search using a new phonetic matching system. BMPM helps you search for personal names (or just surnames) in a Solr/Lucene index, and is far superior to the existing phonetic codecs, such as regular soundex, metaphone, caverphone, etc.+Beider-Morse Phonetic Matching (BMPM) is a "soundalike" tool that lets you search using a new phonetic matching system.+BMPM helps you search for personal names (or just surnames) in a Solr/Lucene index, and is far superior to the existing phonetic codecs, such as regular soundex, metaphone, caverphone, etc.-In general, phonetic matching lets you search a name list for names that are phonetically equivalent to the desired name. BMPM is similar to a soundex search in that an exact spelling is not required. Unlike soundex, it does not generate a large quantity of false hits.+In general, phonetic matching lets you search a name list for names that are phonetically equivalent to the desired name.+BMPM is similar to a soundex search in that an exact spelling is not required.+Unlike soundex, it does not generate a large quantity of false hits.-From the spelling of the name, BMPM attempts to determine the language. It then applies phonetic rules for that particular language to transliterate the name into a phonetic alphabet. If it is not possible to determine the language with a fair degree of certainty, it uses generic phonetic instead. Finally, it applies language-independent rules regarding such things as voiced and unvoiced consonants and vowels to further insure the reliability of the matches.+From the spelling of the name, BMPM attempts to determine the language.+It then applies phonetic rules for that particular language to transliterate the name into a phonetic alphabet.+If it is not possible to determine the language with a fair degree of certainty, it uses generic phonetic instead.+Finally, it applies language-independent rules regarding such things as voiced and unvoiced consonants and vowels to further insure the reliability of the matches.-For example, assume that the matches found when searching for Stephen in a database are "Stefan", "Steph", "Stephen", "Steve", "Steven", "Stove", and "Stuffin". "Stefan", "Stephen", and "Steven" are probably relevant, and are names that you want to see. "Stuffin", however, is probably not relevant. Also rejected were "Steph", "Steve", and "Stove". Of those, "Stove" is probably not one that we would have wanted. But "Steph" and "Steve" are possibly ones that you might be interested in.+For example, assume that the matches found when searching for Stephen in a database are "Stefan", "Steph", "Stephen", "Steve", "Steven", "Stove", and "Stuffin". "Stefan", "Stephen", and "Steven" are probably relevant, and are names that you want to see.+"Stuffin", however, is probably not relevant.+Also rejected were "Steph", "Steve", and "Stove". Of those, "Stove" is probably not one that we would have wanted.+But "Steph" and "Steve" are possibly ones that you might be interested in.For Solr, BMPM searching is available for the following languages:@@ -55,7 +64,7 @@ For more information, see here: http://stevemorse.org/phoneticinfo.htm and http:== Daitch-Mokotoff Soundex-To use this encoding in your analyzer, see <<filter-descriptions.adoc#daitch-mokotoff-soundex-filter,Daitch-Mokotoff Soundex Filter>> in the Filter Descriptions section.+To use this encoding in your analyzer, see <<filters.adoc#daitch-mokotoff-soundex-filter,Daitch-Mokotoff Soundex Filter>> in the Filter Descriptions section.The Daitch-Mokotoff Soundex algorithm is a refinement of the Russel and American Soundex algorithms, yielding greater accuracy in matching especially Slavic and Yiddish surnames with similar pronunciation but differences in spelling.@@ -72,13 +81,15 @@ For more information, see http://en.wikipedia.org/wiki/Daitch%E2%80%93Mokotoff_S== Double Metaphone-To use this encoding in your analyzer, see <<filter-descriptions.adoc#double-metaphone-filter,Double Metaphone Filter>> in the Filter Descriptions section. Alternatively, you may specify `encoder="DoubleMetaphone"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>, but note that the Phonetic Filter version will *not* provide the second ("alternate") encoding that is generated by the Double Metaphone Filter for some tokens.+To use this encoding in your analyzer, see <<filters.adoc#double-metaphone-filter,Double Metaphone Filter>> in the Filter Descriptions section.+Alternatively, you may specify `encoder="DoubleMetaphone"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>, but note that the Phonetic Filter version will *not* provide the second ("alternate") encoding that is generated by the Double Metaphone Filter for some tokens.-Encodes tokens using the double metaphone algorithm by Lawrence Philips. See the original article at http://www.drdobbs.com/the-double-metaphone-search-algorithm/184401251?pgno=2+Encodes tokens using the double metaphone algorithm by Lawrence Philips.+See the original article at http://www.drdobbs.com/the-double-metaphone-search-algorithm/184401251?pgno=2== Metaphone-To use this encoding in your analyzer, specify `encoder="Metaphone"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="Metaphone"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.Encodes tokens using the Metaphone algorithm by Lawrence Philips, described in "Hanging on the Metaphone" in Computer Language, Dec. 1990.@@ -87,7 +98,7 @@ Another reference for more information is http://www.drdobbs.com/the-double-meta== Soundex-To use this encoding in your analyzer, specify `encoder="Soundex"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="Soundex"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.Encodes tokens using the Soundex algorithm, which is used to relate similar names, but can also be used as a general purpose scheme to find words with similar phonemes.@@ -95,7 +106,7 @@ See also http://en.wikipedia.org/wiki/Soundex.== Refined Soundex-To use this encoding in your analyzer, specify `encoder="RefinedSoundex"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="RefinedSoundex"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.Encodes tokens using an improved version of the Soundex algorithm.@@ -103,15 +114,16 @@ See http://en.wikipedia.org/wiki/Soundex.== Caverphone-To use this encoding in your analyzer, specify `encoder="Caverphone"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="Caverphone"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.-Caverphone is an algorithm created by the Caversham Project at the University of Otago. The algorithm is optimised for accents present in the southern part of the city of Dunedin, New Zealand.+Caverphone is an algorithm created by the Caversham Project at the University of Otago.+The algorithm is optimised for accents present in the southern part of the city of Dunedin, New Zealand.See http://en.wikipedia.org/wiki/Caverphone and the Caverphone 2.0 specification at http://caversham.otago.ac.nz/files/working/ctp150804.pdf== Kölner Phonetik a.k.a. Cologne Phonetic-To use this encoding in your analyzer, specify `encoder="ColognePhonetic"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="ColognePhonetic"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.The Kölner Phonetik, an algorithm published by Hans Joachim Postel in 1969, is optimized for the German language.@@ -119,7 +131,7 @@ See http://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik== NYSIIS-To use this encoding in your analyzer, specify `encoder="Nysiis"` with the <<filter-descriptions.adoc#phonetic-filter,Phonetic Filter>>.+To use this encoding in your analyzer, specify `encoder="Nysiis"` with the <<filters.adoc#phonetic-filter,Phonetic Filter>>.NYSIIS is an encoding used to relate similar names, but can also be used as a general purpose scheme to find words with similar phonemes.diff --git a/solr/solr-ref-guide/src/ping.adoc b/solr/solr-ref-guide/src/ping.adocindex 39e8a07c80b..38cff84b467 100644--- a/solr/solr-ref-guide/src/ping.adoc+++ b/solr/solr-ref-guide/src/ping.adoc@@ -21,7 +21,7 @@ Choosing Ping under a core name issues a `ping` request to check whether the cor.Ping Option in Core Dropdownimage::images/ping/ping.png[image,width=171,height=195]-The search executed by a Ping is configured with the <<request-parameters-api.adoc#,Request Parameters API>>. See <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>> for the paramset to use for the `/admin/ping` endpoint.+The search executed by a Ping is configured with the <<request-parameters-api.adoc#,Request Parameters API>>. See <<implicit-requesthandlers.adoc#,Implicit Request Handlers>> for the paramset to use for the `/admin/ping` endpoint.The Ping option doesn't open a page, but the status of the request can be seen on the core overview page shown when clicking on a collection name. The length of time the request has taken is displayed next to the Ping option, in milliseconds.diff --git a/solr/solr-ref-guide/src/post-tool.adoc b/solr/solr-ref-guide/src/post-tool.adocindex dd11cdfb843..0cda48dd6fb 100644--- a/solr/solr-ref-guide/src/post-tool.adoc+++ b/solr/solr-ref-guide/src/post-tool.adoc@@ -116,7 +116,7 @@ Index a tab-separated file into `gettingstarted`:bin/post -c signals -params "separator=%09" -type text/csv data.tsv-----The content type (`-type`) parameter is required to treat the file as the proper type, otherwise it will be ignored and a WARNING logged as it does not know what type of content a .tsv file is. The <<uploading-data-with-index-handlers.adoc#csv-formatted-index-updates,CSV handler>> supports the `separator` parameter, and is passed through using the `-params` setting.+The content type (`-type`) parameter is required to treat the file as the proper type, otherwise it will be ignored and a WARNING logged as it does not know what type of content a .tsv file is. The <<indexing-with-update-handlers.adoc#csv-formatted-index-updates,CSV handler>> supports the `separator` parameter, and is passed through using the `-params` setting.=== Indexing JSONdiff --git a/solr/solr-ref-guide/src/property-substitution.adoc b/solr/solr-ref-guide/src/property-substitution.adocnew file mode 100644index 00000000000..b4fe68184f7--- /dev/null+++ b/solr/solr-ref-guide/src/property-substitution.adoc@@ -0,0 +1,146 @@+= Property Substitution in Configuration Files+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++// *TODO* merge this page with core-discovery.adoc?++Solr supports variable substitution of property values in configuration files, which allows runtime specification of various configuration options in `solrconfig.xml`.++The syntax is `${propertyname[:option default value]`}.+This allows defining a default that can be overridden when Solr is launched.+If a default value is not specified, then the property _must_ be specified at runtime or the configuration file will generate an error when parsed.++There are multiple methods for specifying properties that can be used in configuration files.+Of those below, strongly consider "config overlay" as the preferred approach, as it stays local to the configset and is easy to modify.++== JVM System Properties++Any JVM system property, usually specified using the `-D` flag when starting the JVM, can be used as variables in any XML configuration file in Solr.++For example, in the sample `solrconfig.xml` files, you will see this value which defines the locking type to use:++[source,xml]+----+<lockType>${solr.lock.type:native}</lockType>+----++Which means the lock type defaults to "native" but when starting Solr you could override this using a JVM system property by launching Solr with:++[source,bash]+----+bin/solr start -Dsolr.lock.type=none+----++In general, any Java system property that you want to set can be passed through the `bin/solr` script using the standard `-Dproperty=value` syntax.++Alternatively, you can add common system properties to the `SOLR_OPTS` environment variable defined in the Solr include file (`bin/solr.in.sh` or `bin/solr.in.cmd`).+For more information about how the Solr include file works, refer to: <<taking-solr-to-production.adoc#,Taking Solr to Production>>.++== Config API to Override solrconfig.xml++The <<config-api.adoc#,Config API>> allows you to use an API to modify Solr's configuration, specifically user defined properties.+Changes made with this API are stored in a file named `configoverlay.json`.+This file should only be edited with the API, but will look like this example:++[source,json]+----+{+ "userProps":{"update.autoCreateFields":"false"},+ "requestHandler":{"/myterms":{+ "name":"/myterms",+ "class":"solr.SearchHandler",+ "defaults":{+ "terms":true,+ "distrib":false},+ "components":["terms"]}}}+----++For more details, see the section <<config-api.adoc#,Config API>>.++== User-Defined Properties in core.properties++Every Solr core has a `core.properties` file, automatically created when using the APIs.+When you create a SolrCloud collection, you can pass through custom parameters by prefixing the parameter name with `_property.name_` as a parameter.++For example, to add a property named "my.custom.prop":++[.dynamic-tabs]+--+[example.tab-pane#v1customprop]+====+[.tab-label]*V1 API*++[source,bash]+----+http://localhost:8983/solr/admin/collections?action=CREATE&name=gettingstarted&numShards=1&property.my.custom.prop=edismax+----+====++[example.tab-pane#v2]+====+[.tab-label]*V2 API*++[source,bash]+----+curl -X POST -H 'Content-type: application/json' -d '{"create": {"name": "gettingstarted", "numShards": "1", "property.my.custom.prop": "edismax"}}' http://localhost:8983/api/collections+----+====+--++This will create a `core.properties` file that has at least the following properties (others omitted for brevity):++[source,properties]+----+#core.properties+name=gettingstarted+my.custom.prop=edismax+----++The `my.custom.prop` property can then be used as a variable, such as in `solrconfig.xml`:++[source,xml]+----+<requestHandler name="/select">+ <lst name="defaults">+ <str name="defType">${my.custom.prop}</str>+ </lst>+</requestHandler>+----++== Implicit Core Properties++Several attributes of a Solr core are available as "implicit" properties that can be used in variable substitution, independent of where or how the underlying value is initialized.++For example, regardless of whether the name for a particular Solr core is explicitly configured in `core.properties` or inferred from the name of the instance directory, the implicit property `solr.core.name` is available for use as a variable in that core's configuration file:++[source,xml]+----+<requestHandler name="/select">+ <lst name="defaults">+ <str name="collection_name">${solr.core.name}</str>+ </lst>+</requestHandler>+----++All implicit properties use the `solr.core.` name prefix, and reflect the runtime value of the equivalent <<core-discovery.adoc#,`core.properties` property>>:++* `solr.core.name`+* `solr.core.config`+* `solr.core.schema`+* `solr.core.dataDir`+* `solr.core.transient`+* `solr.core.loadOnStartup`diff --git a/solr/solr-ref-guide/src/putting-the-pieces-together.adoc b/solr/solr-ref-guide/src/putting-the-pieces-together.adocdeleted file mode 100644index 8cc175c4764..00000000000--- a/solr/solr-ref-guide/src/putting-the-pieces-together.adoc+++ /dev/null@@ -1,73 +0,0 @@-= Putting the Pieces Together-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--At the highest level, `schema.xml` is structured as follows.--This example is not real XML, but it gives you an idea of the structure of the file.--[source,xml]------<schema>- <types>- <fields>- <uniqueKey>- <copyField>-</schema>-------Obviously, most of the excitement is in `types` and `fields`, where the field types and the actual field definitions live.--These are supplemented by `copyFields`.--The `uniqueKey` must always be defined.--.Types and fields are optional tags-[NOTE]-====-Note that the `types` and `fields` sections are optional, meaning you are free to mix `field`, `dynamicField`, `copyField` and `fieldType` definitions on the top level. This allows for a more logical grouping of related tags in your schema.-====--== Choosing Appropriate Numeric Types--For general numeric needs, consider using one of the `IntPointField`, `LongPointField`, `FloatPointField`, or `DoublePointField` classes, depending on the specific values you expect. These "Dimensional Point" based numeric classes use specially encoded data structures to support efficient range queries regardless of the size of the ranges used. Enable <<docvalues.adoc#,DocValues>> on these fields as needed for sorting and/or faceting.--Some Solr features may not yet work with "Dimensional Points", in which case you may want to consider the equivalent `TrieIntField`, `TrieLongField`, `TrieFloatField`, and `TrieDoubleField` classes. These field types are deprecated and are likely to be removed in a future major Solr release, but they can still be used if necessary. Configure a `precisionStep="0"` if you wish to minimize index size, but if you expect users to make frequent range queries on numeric types, use the default `precisionStep` (by not specifying it) or specify it as `precisionStep="8"` (which is the default). This offers faster speed for range queries at the expense of increasing index size.--== Working With Text--Handling text properly will make your users happy by providing them with the best possible results for text searches.--One technique is using a text field as a catch-all for keyword searching. Most users are not sophisticated about their searches and the most common search is likely to be a simple keyword search. You can use `copyField` to take a variety of fields and funnel them all into a single text field for keyword searches.--In the `schema.xml` file for the "```techproducts```" example included with Solr, `copyField` declarations are used to dump the contents of `cat`, `name`, `manu`, `features`, and `includes` into a single field, `text`. In addition, it could be a good idea to copy `ID` into `text` in case users wanted to search for a particular product by passing its product number to a keyword search.--Another technique is using `copyField` to use the same field in different ways. Suppose you have a field that is a list of authors, like this:--`Schildt, Herbert; Wolpert, Lewis; Davies, P.`--For searching by author, you could tokenize the field, convert to lower case, and strip out punctuation:--`schildt / herbert / wolpert / lewis / davies / p`--For sorting, just use an untokenized field, converted to lower case, with punctuation stripped:--`schildt herbert wolpert lewis davies p`--Finally, for faceting, use the primary author only via a `StrField`:--`Schildt, Herbert`diff --git a/solr/solr-ref-guide/src/using-python.adoc b/solr/solr-ref-guide/src/python.adocsimilarity index 80%rename from solr/solr-ref-guide/src/using-python.adocrename to solr/solr-ref-guide/src/python.adocindex d5dc465726d..900979fdaee 100644--- a/solr/solr-ref-guide/src/using-python.adoc+++ b/solr/solr-ref-guide/src/python.adoc@@ -1,4 +1,4 @@-= Using Python+= Python// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -20,14 +20,16 @@ Solr includes an output format specifically for <<response-writers.adoc#python-r== Simple Python-Making a query is a simple matter. First, tell Python you will need to make HTTP connections.+Making a query is a simple matter.+First, tell Python you will need to make HTTP connections.[source,python]----from urllib2 import *-----Now open a connection to the server and get a response. The `wt` query parameter tells Solr to return results in a format that Python can understand.+Now open a connection to the server and get a response.+The `wt` query parameter tells Solr to return results in a format that Python can understand.[source,python]----@@ -51,7 +53,8 @@ for document in response['response']['docs']:JSON is a more robust response format, and Python has support for it in its standard library since version 2.6.-Making a query is nearly the same as before. However, notice that the `wt` query parameter is now `json` (which is also the default if no `wt` parameter is specified), and the response is now digested by `json.load()`.+Making a query is nearly the same as before.+However, notice that the `wt` query parameter is now `json` (which is also the default if no `wt` parameter is specified), and the response is now digested by `json.load()`.[source,python]----diff --git a/solr/solr-ref-guide/src/the-query-elevation-component.adoc b/solr/solr-ref-guide/src/query-elevation-component.adocsimilarity index 83%rename from solr/solr-ref-guide/src/the-query-elevation-component.adocrename to solr/solr-ref-guide/src/query-elevation-component.adocindex 50e71092b17..6e3946c7483 100644--- a/solr/solr-ref-guide/src/the-query-elevation-component.adoc+++ b/solr/solr-ref-guide/src/query-elevation-component.adoc@@ -1,4 +1,4 @@-= The Query Elevation Component+= Query Elevation Component// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,7 +18,9 @@The Query Elevation Component lets you configure the top results for a given query regardless of the normal Lucene scoring.-This is sometimes called "sponsored search", "editorial boosting", or "best bets." This component matches the user query text to a configured map of top results. The text can be any string or non-string IDs, as long as it's indexed. Although this component will work with any QueryParser, it makes the most sense to use with <<the-dismax-query-parser.adoc#,DisMax>> or <<the-extended-dismax-query-parser.adoc#,eDisMax>>.+This is sometimes called "sponsored search", "editorial boosting", or "best bets." This component matches the user query text to a configured map of top results.+The text can be any string or non-string IDs, as long as it's indexed.+Although this component will work with any QueryParser, it makes the most sense to use with <<dismax-query-parser.adoc#,DisMax>> or <<edismax-query-parser.adoc#,eDisMax>>.The Query Elevation Component also supports distributed searching.@@ -31,7 +33,8 @@ bin/solr -e techproducts== Configuring the Query Elevation Component-You can configure the Query Elevation Component in the `solrconfig.xml` file. Search components like `QueryElevationComponent` may be added to any request handler; a dedicated request handler is used here for brevity.+You can configure the Query Elevation Component in the `solrconfig.xml` file.+Search components like `QueryElevationComponent` may be added to any request handler; a dedicated request handler is used here for brevity.[source,xml]----@@ -61,7 +64,8 @@ Optionally, in the Query Elevation Component configuration you can also specifyThe Query Elevation Search Component takes the following parameters:`queryFieldType`::-Specifies which field type should be used to analyze the incoming text. For example, it may be appropriate to use a field type with a `LowerCaseFilter`.+Specifies which field type should be used to analyze the incoming text.+For example, it may be appropriate to use a field type with a `LowerCaseFilter`.+Another example is if you need to unescape backslash-escaped queries, then you can define the field type to preprocess with a `PatternReplaceCharFilter`.Here is the corresponding example of field type (traditionally in `managed-schema` or `schema.xml`):@@ -88,7 +92,9 @@ However, that doesn't work in SolrCloud, and there has to be an actual index chaIn all cases, you can reload affected cores/collections to use any new configuration in a configset.`forceElevation`::-By default, this component respects the requested `sort` parameter: if the request asks to sort by date, it will order the results by date. If `forceElevation=true`, results will first return the boosted docs, then order by date. This defaults to `false`.+By default, this component respects the requested `sort` parameter: if the request asks to sort by date, it will order the results by date.+If `forceElevation=true`, results will first return the boosted docs, then order by date.+This defaults to `false`.This is also a request parameter, which will override the config.`useConfiguredElevatedOrder`::@@ -103,7 +109,8 @@ If you only want to elevate the docs that are part of the search result, set thi=== The elevate.xml File-Elevated query results can be configured in an external XML file specified in the `config-file` argument. An `elevate.xml` file might look like this:+Elevated query results can be configured in an external XML file specified in the `config-file` argument.+An `elevate.xml` file might look like this:[source,xml]----@@ -125,9 +132,12 @@ Elevated query results can be configured in an external XML file specified in th</elevate>-----In this example, the query "foo bar" would first return documents 1, 2 and 3, then whatever normally appears for the same query. For the query "ipod", it would first return "MA147LL/A", and would make sure that "IW-02" is not in the result set.+In this example, the query "foo bar" would first return documents 1, 2 and 3, then whatever normally appears for the same query.+For the query "ipod", it would first return "MA147LL/A", and would make sure that "IW-02" is not in the result set.-Notice the `match` parameter with the value `"subset"` for the third rule. A query "bill bar foo" would trigger this rule because the rule defines a subset of terms to appear in the query, in any order. This query would elevate document 11 on top.+Notice the `match` parameter with the value `"subset"` for the third rule.+A query "bill bar foo" would trigger this rule because the rule defines a subset of terms to appear in the query, in any order.+This query would elevate document 11 on top.The `match` parameter accepts either `"exact"` (by default) or `"subset"` values.Subset matching is scalable, one can add many rules with the `match="subset"` parameter.@@ -137,7 +147,8 @@ If documents to be elevated are not defined in the `elevate.xml` file, they shou=== The enableElevation Parameter-For debugging it may be useful to see results with and without the elevated docs. To hide results, use `enableElevation=false`:+For debugging it may be useful to see results with and without the elevated docs.+To hide results, use `enableElevation=false`:[source,text]http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&debugQuery=true&enableElevation=true@@ -165,12 +176,13 @@ You can force set `useConfiguredElevatedOrder` during runtime by supplying it as=== Document Transformers and the markExcludes Parameter-The `[elevated]` <<transforming-result-documents.adoc#,Document Transformer>> can be used to annotate each document with information about whether or not it was elevated:+The `[elevated]` <<document-transformers.adoc#,Document Transformer>> can be used to annotate each document with information about whether or not it was elevated:[source,text]http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&fl=id,[elevated]-Likewise, it can be helpful when troubleshooting to see all matching documents – including documents that the elevation configuration would normally exclude. This is possible by using the `markExcludes=true` parameter, and then using the `[excluded]` transformer:+Likewise, it can be helpful when troubleshooting to see all matching documents – including documents that the elevation configuration would normally exclude.+This is possible by using the `markExcludes=true` parameter, and then using the `[excluded]` transformer:[source,text]http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&markExcludes=true&fl=id,[elevated],[excluded]@@ -193,4 +205,5 @@ http://localhost:8983/solr/techproducts/elevate?q=ipod&df=text&elevateIds=IW-02,=== The fq Parameter with Elevation-Query elevation respects the standard filter query (`fq`) parameter. That is, if the query contains the `fq` parameter, all results will be within that filter even if `elevate.xml` adds other documents to the result set.+Query elevation respects the standard filter query (`fq`) parameter.+That is, if the query contains the `fq` parameter, all results will be within that filter even if `elevate.xml` adds other documents to the result set.diff --git a/solr/solr-ref-guide/src/query-guide.adoc b/solr/solr-ref-guide/src/query-guide.adocnew file mode 100644index 00000000000..404c4593924--- /dev/null+++ b/solr/solr-ref-guide/src/query-guide.adoc@@ -0,0 +1,72 @@+= Query Guide+:page-children: query-syntax-and-parsers, \+ enhancing-queries, \+ controlling-results, \+ streaming-expressions+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+This section describes options to control how Solr works with user input and returns results for queries.++It covers the following topics:++****+[discrete]+=== Query Syntax and Parsers++<<query-syntax-and-parsers.adoc#,*Query Syntax and Parsers*>>++// This pulls the sub-section list from the child page to reduce errors+include::query-syntax-and-parsers.adoc[tag=parser-sections]+****++****+[discrete]+=== Enhancing Queries+<<enhancing-queries.adoc#,*Enhancing Queries*>>++// This pulls the sub-section list from the child page to reduce errors+include::enhancing-queries.adoc[tag=queries-sections]+****++****+[discrete]+=== Controlling Results+<<controlling-results.adoc#,*Controlling Results*>>++// This pulls the sub-section list from the child page to reduce errors+include::controlling-results.adoc[tag=results-sections]+****++****+[discrete]+=== Streaming Expressions+<<streaming-expressions.adoc#,*Streaming Expressions*>>++[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<stream-source-reference.adoc#,Stream Source Reference>>: Stream sources.+| <<stream-decorator-reference.adoc#,Stream Decorator Reference>>: Stream decorators.+| <<stream-evaluator-reference.adoc#,Stream Evaluator Reference>>: Stream evaluators.+| <<math-expressions.adoc#,Math Expressions>>: Streaming expressions for math and analytics applications.+| <<graph-traversal.adoc#,Graph Traversal>>: Graph queries using streaming expressions.+| <<stream-api.adoc#,Stream Request Handler API>>: REST API for expression plugins and daemon control.+| <<stream-screen.adoc#,Stream Screen>>: Admin UI screen for streaming expressions.+|+|===+****diff --git a/solr/solr-ref-guide/src/query-re-ranking.adoc b/solr/solr-ref-guide/src/query-re-ranking.adocindex 2e38f0a50f2..afc64e42659 100644--- a/solr/solr-ref-guide/src/query-re-ranking.adoc+++ b/solr/solr-ref-guide/src/query-re-ranking.adoc@@ -1,5 +1,4 @@= Query Re-Ranking-:page-children: learning-to-rank// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -19,17 +18,18 @@Query Re-Ranking allows you to run a simple query (A) for matching documents and then re-rank the top N documents using the scores from a more complex query (B).-Since the more costly ranking from query B is only applied to the top _N_ documents, it will have less impact on performance then just using the complex query B by itself. The trade off is that documents which score very low using the simple query A may not be considered during the re-ranking phase, even if they would score very highly using query B.+Since the more costly ranking from query B is only applied to the top _N_ documents, it will have less impact on performance then just using the complex query B by itself.+The trade off is that documents which score very low using the simple query A may not be considered during the re-ranking phase, even if they would score very highly using query B.== Specifying a Ranking Query-A Ranking query can be specified using the `rq` request parameter. The `rq` parameter must specify a query string that when parsed, produces a {solr-javadocs}/core/org/apache/solr/search/RankQuery.html[RankQuery].+A Ranking query can be specified using the `rq` request parameter.+The `rq` parameter must specify a query string that when parsed, produces a {solr-javadocs}/core/org/apache/solr/search/RankQuery.html[RankQuery].-Three rank queries are currently included in the Solr distribution. You can also configure a custom {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin] you have written, but most users can just use a parser provided with Solr.+Three rank queries are currently included in the Solr distribution.+You can also configure a custom {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin] you have written, but most users can just use a parser provided with Solr.-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="30,70",options="header"]+[%autowidth.stretch,options="header"]|===|Parser |QParserPlugin class|rerank |{solr-javadocs}/core/org/apache/solr/search/ReRankQParserPlugin.html[ReRankQParserPlugin]@@ -42,13 +42,16 @@ Three rank queries are currently included in the Solr distribution. You can alsoThe `rerank` parser wraps a query specified by an local parameter, along with additional parameters indicating how many documents should be re-ranked, and how the final scores should be computed:`reRankQuery`::-The query string for your complex ranking query - in most cases <<local-parameters-in-queries.adoc#,a variable>> will be used to refer to another request parameter. This parameter is required.+The query string for your complex ranking query - in most cases <<local-params.adoc#,a variable>> will be used to refer to another request parameter.+This parameter is required.`reRankDocs`::-The number of top N documents from the original query that should be re-ranked. This number will be treated as a minimum, and may be increased internally automatically in order to rank enough documents to satisfy the query (i.e., start+rows). The default is `200`.+The number of top N documents from the original query that should be re-ranked.+This number will be treated as a minimum, and may be increased internally automatically in order to rank enough documents to satisfy the query (i.e., start+rows). The default is `200`.`reRankWeight`::-A multiplicative factor that will be applied to the score from the reRankQuery for each of the top matching documents, before that score is added to the original score. The default is `2.0`.+A multiplicative factor that will be applied to the score from the reRankQuery for each of the top matching documents, before that score is added to the original score.+The default is `2.0`.In the example below, the top 1000 documents matching the query "greetings" will be re-ranked using the query "(hi hello hey hiya)". The resulting scores for each of those 1000 documents will be 3 times their score from the "(hi hello hey hiya)", plus the score from the original "greetings" query:@@ -65,4 +68,7 @@ The `ltr` stands for Learning To Rank, please see <<learning-to-rank.adoc#,Learn== Combining Ranking Queries with Other Solr Features-The `rq` parameter and the re-ranking feature in general works well with other Solr features. For example, it can be used in conjunction with the <<collapse-and-expand-results.adoc#,collapse parser>> to re-rank the group heads after they've been collapsed. It also preserves the order of documents elevated by the <<the-query-elevation-component.adoc#,elevation component>>. And it even has its own custom explain so you can see how the re-ranking scores were derived when looking at <<common-query-parameters.adoc#debug-parameter,debug information>>.+The `rq` parameter and the re-ranking feature in general works well with other Solr features.+For example, it can be used in conjunction with the <<collapse-and-expand-results.adoc#,collapse parser>> to re-rank the group heads after they've been collapsed.+It also preserves the order of documents elevated by the <<query-elevation-component.adoc#,elevation component>>.+And it even has its own custom explain so you can see how the re-ranking scores were derived when looking at <<common-query-parameters.adoc#debug-parameter,debug information>>.diff --git a/solr/solr-ref-guide/src/query-screen.adoc b/solr/solr-ref-guide/src/query-screen.adocindex 87a36f5e174..510a7e0568c 100644--- a/solr/solr-ref-guide/src/query-screen.adoc+++ b/solr/solr-ref-guide/src/query-screen.adoc@@ -37,7 +37,7 @@ Request-handler (qt)::Specifies the query handler for the request. If a query handler is not specified, Solr processes the response with the standard query handler.q::-The query event. See <<searching.adoc#,Searching>> for an explanation of this parameter.+The query event. See <<query-guide.adoc#,Query Guide>> for an explanation of this parameter.fq::The filter queries. See <<common-query-parameters.adoc#,Common Query Parameters>> for more information on this parameter.@@ -46,10 +46,10 @@ sort::Sorts the response to a query in either ascending or descending order based on the response's score or another specified characteristic.start, rows::-`start` is the offset into the query result starting at which documents should be returned. The default value is 0, meaning that the query should return results starting with the first document that matches. This field accepts the same syntax as the start query parameter, which is described in <<searching.adoc#,Searching>>. `rows` is the number of rows to return.+`start` is the offset into the query result starting at which documents should be returned. The default value is 0, meaning that the query should return results starting with the first document that matches. This field accepts the same syntax as the start query parameter, which is described in <<query-guide.adoc#,Query Guide>>. `rows` is the number of rows to return.fl::-Defines the fields to return for each document. You can explicitly list the stored fields, <<function-queries.adoc#,functions>>, and <<transforming-result-documents.adoc#,doc transformers>> you want to have returned by separating them with either a comma or a space.+Defines the fields to return for each document. You can explicitly list the stored fields, <<function-queries.adoc#,functions>>, and <<document-transformers.adoc#,doc transformers>> you want to have returned by separating them with either a comma or a space.wt::Specifies the Response Writer to be used to format the query response. Defaults to JSON if not specified.@@ -61,10 +61,10 @@ debugQuery::Click this button to augment the query response with debugging information, including "explain info" for each document returned. This debugging information is intended to be intelligible to the administrator or programmer.dismax::-Click this button to enable the Dismax query parser. See <<the-dismax-query-parser.adoc#,The DisMax Query Parser>> for further information.+Click this button to enable the DisMax query parser. See <<dismax-query-parser.adoc#,The DisMax Query Parser>> for further information.edismax::-Click this button to enable the Extended query parser. See <<the-extended-dismax-query-parser.adoc#,The Extended DisMax Query Parser>> for further information.+Click this button to enable the Extended query parser. See <<edismax-query-parser.adoc#,The Extended DisMax Query Parser>> for further information.hl:: Click this button to enable highlighting in the query response. See <<highlighting.adoc#,Highlighting>> for more information.diff --git a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc b/solr/solr-ref-guide/src/query-settings-in-solrconfig.adocdeleted file mode 100644index 546b8afb57e..00000000000--- a/solr/solr-ref-guide/src/query-settings-in-solrconfig.adoc+++ /dev/null@@ -1,274 +0,0 @@-= Query Settings in SolrConfig-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The settings in this section affect the way that Solr will process and respond to queries.--These settings are all configured in child elements of the `<query>` element in `solrconfig.xml`.--[source,xml]------<query>- ...-</query>-------== Caches--Solr caches are associated with a specific instance of an Index Searcher, a specific view of an index that doesn't change during the lifetime of that searcher. As long as that Index Searcher is being used, any items in its cache will be valid and available for reuse. By default cached Solr objects do not expire after a time interval; instead, they remain valid for the lifetime of the Index Searcher. Idle time-based expiration can be enabled by using `maxIdleTime` option.--When a new searcher is opened, the current searcher continues servicing requests while the new one auto-warms its cache. The new searcher uses the current searcher's cache to pre-populate its own. When the new searcher is ready, it is registered as the current searcher and begins handling all new search requests. The old searcher will be closed once it has finished servicing all its requests.--=== Cache Implementations--Solr comes with a default `SolrCache` implementation that is used for different types of caches.--The `CaffeineCache` is an implementation backed by the https://github.com/ben-manes/caffeine[Caffeine caching library]. By default it uses a Window TinyLFU (W-TinyLFU) eviction policy, which allows the eviction based on both frequency and recency of use in O(1) time with a small footprint. Generally this cache implementation is recommended over other legacy caches as it usually offers lower memory footprint, higher hit ratio and better multi-threaded performance over legacy caches.--`CaffeineCache` uses an auto-warm count that supports both integers and percentages which get evaluated relative to the current size of the cache when warming happens.--The Statistics page in the Solr Admin UI will display information about the performance of all the active caches. This information can help you fine-tune the sizes of the various caches appropriately for your particular application. When a Searcher terminates, a summary of its cache usage is also written to the log.--Each cache has settings to define its initial size (`initialSize`), maximum size (`size`) and number of items to use for during warming (`autowarmCount`). For `autowarmCount` this can be also expressed as a percentage instead of an absolute value.--A `maxIdleTime` attribute controls the automatic eviction of entries that haven't been used for a while. This attribute is expressed in seconds, with the default value of `0` meaning no entries are automatically evicted due to exceeded idle time. Smaller values of this attribute will cause older entries to be evicted quickly, which will reduce cache memory usage but may instead cause thrashing due to a repeating eviction-lookup-miss-insertion cycle of the same entries. Larger values will cause entries to stay around longer, waiting to be reused, at the cost of increased memory usage. Reasonable values, depending on the query volume and patterns, may lie somewhere between 60-3600.--The `maxRamMB` attribute limits the maximum amount of memory a cache may consume. When both `size` and `maxRamMB` limits are specified the `maxRamMB` limit will take precedence and the `size` limit will be ignored.--Details of each cache are described below.--=== filterCache--This cache holds parsed queries paired with an unordered set of all documents that match it. Unless such a set is trivially small, the set implementation is a bitset.--The most typical way Solr uses the `filterCache` is to cache results of each `fq` (filter query) search parameter, though there are some other cases as well. Subsequent queries using the same filter query result in cache hits and rapid returns of results. See <<searching.adoc#,Searching>> for a detailed discussion of the `fq` parameter. Another Solr feature using this cache is the `filter(...)` syntax in the default Lucene query parser.-Use of this cache can be disabled for a `fq` using the <<common-query-parameters.adoc#cache-local-parameter,cache local-parameter>>.--Solr also uses this cache for faceting when the configuration parameter `facet.method` is set to `fc`. For a discussion of faceting, see <<searching.adoc#,Searching>>.--[source,xml]------<filterCache class="solr.CaffeineCache"- size="512"- initialSize="512"- autowarmCount="128"/>-------The cache supports a `maxRamMB` parameter which restricts the maximum amount of heap used by this cache. The `CaffeineCache` only supports evictions by either heap usage or size but not both. Therefore, the `size` parameter is ignored if `maxRamMB` is specified.--[source,xml]------<filterCache class="solr.CaffeineCache"- maxRamMB="1000"- autowarmCount="128"/>-------=== queryResultCache--This cache holds the results of previous searches: ordered lists of document IDs (DocList) based on a query, a sort, and the range of documents requested.--The `queryResultCache` has an optional setting to limit the maximum amount of RAM used (`maxRamMB`). This lets you specify the maximum heap size, in megabytes, used by the contents of this cache. When the cache grows beyond this size, oldest accessed queries will be evicted until the heap usage of the cache decreases below the specified limit. If a `size` is specified in addition to `maxRamMB` then only the heap usage limit is respected.--Use of this cache can be disabled on a query-by-query basis in `q` using the <<common-query-parameters.adoc#cache-local-parameter,cache local-parameter>>.--[source,xml]------<queryResultCache class="solr.CaffeineCache"- size="512"- initialSize="512"- autowarmCount="128"/>-------=== documentCache--This cache holds Lucene Document objects (the stored fields for each document). Since Lucene internal document IDs are transient, this cache is not auto-warmed. The size for the `documentCache` should always be greater than `max_results` times the `max_concurrent_queries`, to ensure that Solr does not need to refetch a document during a request. The more fields you store in your documents, the higher the memory usage of this cache will be.--[source,xml]------<documentCache class="solr.CaffeineCache"- size="512"- initialSize="512"- autowarmCount="0"/>-------=== User Defined Caches--You can also define named caches for your own application code to use. You can locate and use your cache object by name by calling the `SolrIndexSearcher` methods `getCache()`, `cacheLookup()` and `cacheInsert()`.--[source,xml]------<cache name="myUserCache" class="solr.CaffeineCache"- size="4096"- initialSize="1024"- autowarmCount="1024"- regenerator="org.mycompany.mypackage.MyRegenerator" />-------If you want auto-warming of your cache, include a `regenerator` attribute with the fully qualified name of a class that implements `solr.search.CacheRegenerator`. You can also use the `NoOpRegenerator`, which simply repopulates the cache with old items. Define it with the `regenerator` parameter as `regenerator="solr.NoOpRegenerator"`.--== Query Sizing and Warming--=== maxBooleanClauses--Sets the maximum number of clauses allowed when parsing a boolean query string.--This limit only impacts boolean queries specified by a user as part of a query string, and provides per-collection controls on how complex user specified boolean queries can be. Query strings that specify more clauses than this will result in an error.--If this per-collection limit is greater than <<format-of-solr-xml#global-maxbooleanclauses,the global `maxBooleanClauses` limit specified in `solr.xml`>> it will have no effect, as that setting also limits the size of user specified boolean queries.--In default configurations this property uses the value of the `solr.max.booleanClauses` system property if specified. This is the same system property used in the <<format-of-solr-xml#global-maxbooleanclauses,global `maxBooleanClauses` setting in the default `solr.xml`>> making it easy for Solr administrators to increase both values (in all collections) without needing to search through and update all of their configs.--[source,xml]------<maxBooleanClauses>${solr.max.booleanClauses:1024}</maxBooleanClauses>-------=== enableLazyFieldLoading--If this parameter is set to true, then fields that are not directly requested will be loaded lazily as needed. This can boost performance if the most common queries only need a small subset of fields, especially if infrequently accessed fields are large in size.--[source,xml]------<enableLazyFieldLoading>true</enableLazyFieldLoading>-------=== useFilterForSortedQuery--This parameter configures Solr to use a filter to satisfy a search. If the requested sort does not include "score", the `filterCache` will be checked for a filter matching the query. For most situations, this is only useful if the same search is requested often with different sort options and none of them ever use "score".--[source,xml]------<useFilterForSortedQuery>true</useFilterForSortedQuery>-------=== queryResultWindowSize--Used with the `queryResultCache`, this will cache a superset of the requested number of document IDs. For example, if the a search in response to a particular query requests documents 10 through 19, and `queryWindowSize` is 50, documents 0 through 49 will be cached.--[source,xml]------<queryResultWindowSize>20</queryResultWindowSize>-------=== queryResultMaxDocsCached--This parameter sets the maximum number of documents to cache for any entry in the `queryResultCache`.--[source,xml]------<queryResultMaxDocsCached>200</queryResultMaxDocsCached>-------=== circuitBreaker--This set of configurations control the behaviour of circuit breakers.--[source,xml]------<circuitBreaker class="solr.CircuitBreakerManager" enabled="true">- <!-- All specific configs in this section -->-</circuitBreaker>-------To control whether Circuit Breakers are globally enabled, use the "enabled" attribute.--=== Memory Circuit Breaker Settings--To turn memory circuit breaker on/off, use the following flag:-[source,xml]------<str name="memEnabled">true</str>-------Memory threshold in percentage for JVM heap usage defined in percentage of maximum heap allocated-to the JVM (-Xmx). Ideally, this value should be in the range of 75-80% of maximum heap allocated-to the JVM. The enabled flag can be used to control the specific toggle for this circuit breaker.--[source,xml]------<str name="memThreshold">75</str>-------=== CPU Circuit Breaker Settings--To control turning on/off this feature, use the following flag:-[source,xml]------<str name="cpuEnabled">true</str>-------Defines the triggering threshold in terms of the average per minute CPU load. The enabled flag can be used to control the specific toggle for this circuit breaker.--[source,xml]------<str name="cpuThreshold">75</str>-------=== useColdSearcher--This setting controls whether search requests for which there is not a currently registered searcher should wait for a new searcher to warm up (false) or proceed immediately (true). When set to "false", requests will block until the searcher has warmed its caches.--[source,xml]------<useColdSearcher>false</useColdSearcher>-------=== maxWarmingSearchers--This parameter sets the maximum number of searchers that may be warming up in the background at any given time. Exceeding this limit will raise an error. For read-only followers, a value of two is reasonable. Leaders should probably be set a little higher.--[source,xml]------<maxWarmingSearchers>2</maxWarmingSearchers>-------== Query-Related Listeners--As described in the section on <<Caches>>, new Index Searchers are cached. It's possible to use the triggers for listeners to perform query-related tasks. The most common use of this is to define queries to further "warm" the Index Searchers while they are starting. One benefit of this approach is that field caches are pre-populated for faster sorting.--Good query selection is key with this type of listener. It's best to choose your most common and/or heaviest queries and include not just the keywords used, but any other parameters such as sorting or filtering requests.--There are two types of events that can trigger a listener. A `firstSearcher` event occurs when a new searcher is being prepared but there is no current registered searcher to handle requests or to gain auto-warming data from (i.e., on Solr startup). A `newSearcher` event is fired whenever a new searcher is being prepared and there is a current searcher handling requests.--The (commented out) examples below can be found in the `solrconfig.xml` file of the `sample_techproducts_configs` <<config-sets.adoc#,configset>> included with Solr, and demonstrate using the `solr.QuerySenderListener` class to warm a set of explicit queries:--[source,xml]------<listener event="newSearcher" class="solr.QuerySenderListener">- <arr name="queries">- <!--- <lst><str name="q">solr</str><str name="sort">price asc</str></lst>- <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst>- -->- </arr>-</listener>--<listener event="firstSearcher" class="solr.QuerySenderListener">- <arr name="queries">- <lst><str name="q">static firstSearcher warming in solrconfig.xml</str></lst>- </arr>-</listener>-------[IMPORTANT]-====-The above code comes from a _sample_ `solrconfig.xml`.--A key best practice is to modify these defaults before taking your application to production, but please note: while the sample queries are commented out in the section for the "newSearcher", the sample query is not commented out for the "firstSearcher" event.--There is no point in auto-warming your Index Searcher with the query string "static firstSearcher warming in solrconfig.xml" if that is not relevant to your search application.-====diff --git a/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc b/solr/solr-ref-guide/src/query-syntax-and-parsers.adocnew file mode 100644index 00000000000..b2cdc1e44bb--- /dev/null+++ b/solr/solr-ref-guide/src/query-syntax-and-parsers.adoc@@ -0,0 +1,65 @@+= Query Syntax and Parsers+:page-children: common-query-parameters, \+ standard-query-parser, \+ dismax-query-parser, \+ edismax-query-parser, \+ function-queries, \+ local-params, \+ json-request-api, \+ searching-nested-documents, \+ block-join-query-parser, \+ join-query-parser, \+ spatial-search, \+ other-parsers, \+ parallel-sql-interface, \+ query-screen+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+A _query parser_ converts a user's search terms into a Lucene query to find appropriately matching documents.++Solr supports several query parsers, offering search application designers great flexibility in controlling how queries are parsed.++This section explains how to specify a query parser and describes the syntax and features supported by several parsers available in Solr.++There are some query parameters common to all Solr parsers; these are discussed in the section <<common-query-parameters.adoc#common-query-parameters,Common Query Parameters>>.++Query parsers are also called `QParserPlugins`. They are all subclasses of {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin]. If you have custom parsing needs, you may want to extend that class to create your own query parser.++****+// This tags the below list so it can be used in the parent page section list+// tag::parser-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<common-query-parameters.adoc#,Common Query Parameters>>: Query parameters that can be used with all query parsers.+| <<standard-query-parser.adoc#,Standard Query Parser>>: The standard Lucene query parser.+| <<dismax-query-parser.adoc#,DisMax Query Parser>>: The DisMax query parser.+| <<edismax-query-parser.adoc#,Extended DisMax Query Parser>>: The Extended DisMax (eDisMax) Query Parser.+| <<function-queries.adoc#,Function Queries>>: Parameters for generating relevancy scores using values from one or more numeric fields.+| <<local-params.adoc#,Local Params>>: How to add local arguments to queries.+| <<json-request-api.adoc#,JSON Request API>>: Solr's JSON Request API.+| <<searching-nested-documents.adoc#,Searching Nested Documents>>: Constructing nested and hierarchical queries.+| <<block-join-query-parser.adoc#,Block Join Query Parser>>: Query parser dedicated to searching nested documents.+| <<join-query-parser.adoc#,Join Query Parser>>: Query parser to facilitate joins.+| <<spatial-search.adoc#,Spatial Search>>: Solr's spatial search capabilities.+| <<other-parsers.adoc#,Other Parsers>>: More parsers designed for use in specific situations.+| <<parallel-sql-interface.adoc#,Parallel SQL Interface>>: SQL interface for Solr.+| <<query-screen.adoc#,Query Screen>>: Form-based query builder.+|===+// end::parser-sections[]+****diff --git a/solr/solr-ref-guide/src/query-syntax-and-parsing.adoc b/solr/solr-ref-guide/src/query-syntax-and-parsing.adocdeleted file mode 100644index 4ef0cc1eb46..00000000000--- a/solr/solr-ref-guide/src/query-syntax-and-parsing.adoc+++ /dev/null@@ -1,31 +0,0 @@-= Query Syntax and Parsing-:page-children: common-query-parameters, the-standard-query-parser, the-dismax-query-parser, the-extended-dismax-query-parser, function-queries, local-parameters-in-queries, other-parsers-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr supports several query parsers, offering search application designers great flexibility in controlling how queries are parsed.--This section explains how to specify the query parser to be used. It also describes the syntax and features supported by the main query parsers included with Solr and describes some other parsers that may be useful for particular situations. There are some query parameters common to all Solr parsers; these are discussed in the section <<common-query-parameters.adoc#,Common Query Parameters>>.--The parsers discussed in this Guide are:--* <<the-standard-query-parser.adoc#,The Standard Query Parser>>-* <<the-dismax-query-parser.adoc#,The DisMax Query Parser>>-* <<the-extended-dismax-query-parser.adoc#,The Extended DisMax Query Parser>>-* <<other-parsers.adoc#,Other Parsers>>--The query parser plugins are all subclasses of {solr-javadocs}/core/org/apache/solr/search/QParserPlugin.html[QParserPlugin]. If you have custom parsing needs, you may want to extend that class to create your own query parser.diff --git a/solr/solr-ref-guide/src/realtime-get.adoc b/solr/solr-ref-guide/src/realtime-get.adocindex b9450e688e5..a74b8681239 100644--- a/solr/solr-ref-guide/src/realtime-get.adoc+++ b/solr/solr-ref-guide/src/realtime-get.adoc@@ -18,7 +18,8 @@For index updates to be visible (searchable), some kind of commit must reopen a searcher to a new point-in-time view of the index.-The *realtime get* feature allows retrieval (by `unique-key`) of the latest version of any documents without the associated cost of reopening a searcher. This is primarily useful when using Solr as a NoSQL data store and not just a search index.+The *realtime get* feature allows retrieval (by `unique-key`) of the latest version of any documents without the associated cost of reopening a searcher.+This is primarily useful when using Solr as a NoSQL data store and not just a search index.Real Time Get relies on the update log feature, which is enabled by default and can be configured in `solrconfig.xml`:@@ -29,7 +30,7 @@ Real Time Get relies on the update log feature, which is enabled by default and</updateLog>-----Real Time Get requests can be performed using the `/get` handler which exists implicitly in Solr - see <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>> - it's equivalent to the following configuration:+Real Time Get requests can be performed using the `/get` handler which exists implicitly in Solr - see <<implicit-requesthandlers.adoc#,Implicit Request Handlers>> - it's equivalent to the following configuration:[source,xml]----@@ -105,7 +106,8 @@ http://localhost:8983/api/collections/techproducts/get?id=mydoc====---You can also specify multiple documents at once via the `ids` parameter and a comma separated list of ids, or by using multiple `id` parameters. If you specify multiple ids, or use the `ids` parameter, the response will mimic a normal query response to make it easier for existing clients to parse.+You can also specify multiple documents at once via the `ids` parameter and a comma separated list of ids, or by using multiple `id` parameters.+If you specify multiple ids, or use the `ids` parameter, the response will mimic a normal query response to make it easier for existing clients to parse.For example:@@ -213,7 +215,8 @@ http://localhost:8983/api/collections/techproducts/get?id=mydoc&id=IW-02&fq=name[WARNING]====-Do *NOT* disable the realtime get handler at `/get` if you are using SolrCloud. Doing so will result in any leader election to cause a full sync in *ALL* replicas for the shard in question.+Do *NOT* disable the realtime get handler at `/get` if you are using SolrCloud.+Doing so will result in any leader election to cause a full sync in *ALL* replicas for the shard in question.Similarly, a replica recovery will also always fetch the complete index from the leader because a partial sync will not be possible in the absence of this handler.====diff --git a/solr/solr-ref-guide/src/reindexing.adoc b/solr/solr-ref-guide/src/reindexing.adocindex 8c5e491c848..ad99c22d986 100644--- a/solr/solr-ref-guide/src/reindexing.adoc+++ b/solr/solr-ref-guide/src/reindexing.adoc@@ -112,7 +112,7 @@ curl -X POST -H 'Content-Type: application/json' --data-binary '{"delete":{"querIt's important to verify that *all* documents have been deleted, as that ensures the Lucene index segments have beendeleted as well.-To verify that there are no segments in your index, look in the data/index directory and confirm it has no segments files. Since the data directory can be customized, see the section <<datadir-and-directoryfactory-in-solrconfig.adoc#specifying-a-location-for-index-data-with-the-datadir-parameter,Specifying a Location for Index Data with the dataDir Parameter>> for the location of your index files.+To verify that there are no segments in your index, look in the data/index directory and confirm it has no segments files. Since the data directory can be customized, see the section <<index-location-format.adoc#specifying-a-location-for-index-data-with-the-datadir-parameter,Specifying a Location for Index Data with the dataDir Parameter>> for the location of your index files.Note you will need to verify the indexes have been removed in every shard and every replica on every node of a cluster.It is not sufficient to only query for the number of documents because you may have no documents but still have index@@ -125,7 +125,7 @@ A variation on this approch is to delete and recreate your collection using the=== Index to Another Collection-Another approach is to use index to a new collection and use Solr's <<collection-aliasing.adoc#createalias,collection alias>> feature to seamlessly point the application to a new collection without downtime.+Another approach is to use index to a new collection and use Solr's <<alias-management.adoc#createalias,collection alias>> feature to seamlessly point the application to a new collection without downtime.This option is only available for Solr installations running in SolrCloud mode.diff --git a/solr/solr-ref-guide/src/replica-management.adoc b/solr/solr-ref-guide/src/replica-management.adocindex 2fb241a5129..6a5f683b065 100644--- a/solr/solr-ref-guide/src/replica-management.adoc+++ b/solr/solr-ref-guide/src/replica-management.adoc@@ -17,7 +17,8 @@// specific language governing permissions and limitations// under the License.-A replica is a physical copy of a shard. Replicas enhance fail over by providing additional copies of the data+A replica is a physical copy of a shard.+Replicas enhance fail over by providing additional copies of the dataand enhance scalability by providing additional capacity for searching.All of the examples in this section assume you are running the "techproducts" Solr example:@@ -30,7 +31,9 @@ bin/solr -c -e techproducts[[addreplica]]== ADDREPLICA: Add Replica-Add one or more replicas to a shard in a collection. The node name can be specified if the replica is to be created in a specific node. Otherwise, a set of nodes can be specified and the most suitable ones among them will be chosen to create the replica(s).+Add one or more replicas to a shard in a collection.+The node name can be specified if the replica is to be created in a specific node.+Otherwise, a set of nodes can be specified and the most suitable ones among them will be chosen to create the replica(s).[.dynamic-tabs]--@@ -101,7 +104,8 @@ curl -X POST http://localhost:8983/api/collections/techproducts/shards -H 'Conte=== ADDREPLICA Parameters`collection`::-The name of the collection where the replica should be created. This parameter is required.+The name of the collection where the replica should be created.+This parameter is required.`shard`::The name of the shard to which replica is to be added.@@ -130,13 +134,16 @@ The instanceDir for the core that will be created.The directory in which the core should be created.`type`::-The type of replica to create. These possible values are allowed:+The type of replica to create.+These possible values are allowed:+-* `nrt`: The NRT type maintains a transaction log and updates its index locally. This is the default and the most commonly used.+* `nrt`: The NRT type maintains a transaction log and updates its index locally.+This is the default and the most commonly used.* `tlog`: The TLOG type maintains a transaction log but only updates its index via replication.-* `pull`: The PULL type does not maintain a transaction log and only updates its index via replication. This type is not eligible to become a leader.+* `pull`: The PULL type does not maintain a transaction log and only updates its index via replication.+This type is not eligible to become a leader.+-See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica type options.+See the section <<solrcloud-shards-indexing.adoc#types-of-replicas,Types of Replicas>> for more information about replica type options.`nrtReplicas`::The number of `nrt` replicas that should be created (optional, defaults to 1 if `type` is `nrt` otherwise 0).@@ -148,15 +155,18 @@ The number of `tlog` replicas that should be created (optional, defaults to 1 ifThe number of `pull` replicas that should be created (optional, defaults to 1 if `type` is `pull` otherwise 0).`property._name_=_value_`::-Set core property _name_ to _value_. See <<defining-core-properties.adoc#,Defining core.properties>> for details about supported properties and values.+Set core property _name_ to _value_. See <<core-discovery.adoc#,Core Discovery>> for details about supported properties and values.[WARNING]====-The entries in each core.properties file are vital for Solr to function correctly. Overriding entries can result in unusable collections. Altering these entries by specifying `property._name_=_value_` is an expert-level option and should only be used if you have a thorough understanding of the consequences.+The entries in each core.properties file are vital for Solr to function correctly.+Overriding entries can result in unusable collections.+Altering these entries by specifying `property._name_=_value_` is an expert-level option and should only be used if you have a thorough understanding of the consequences.====`waitForFinalState`::-If `true`, the request will complete only when all affected replicas become active. The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.+If `true`, the request will complete only when all affected replicas become active.+The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>@@ -203,7 +213,10 @@ http://localhost:8983/solr/admin/collections?action=addreplica&collection=gettin[[movereplica]]== MOVEREPLICA: Move a Replica to a New Node-This command moves a replica from one node to another node by executing ADDREPLICA on the destination and then DELETEREPLICA on the source. If this command is interrupted or times out before the ADDREPLICA operation produces a replica in an active state, the DELETEREPLICA will not occur. Timeouts do not cancel the ADDREPLICA, and will result in extra shards. In case of shared filesystems the `dataDir` will be reused.+This command moves a replica from one node to another node by executing ADDREPLICA on the destination and then DELETEREPLICA on the source.+If this command is interrupted or times out before the ADDREPLICA operation produces a replica in an active state, the DELETEREPLICA will not occur.+Timeouts do not cancel the ADDREPLICA, and will result in extra shards.+In case of shared filesystems the `dataDir` will be reused.If this command is used on a collection where more than one replica from the same shard exists on the same node, and the `shard` and `sourceNode` parameters match more than one replica, the replica selected is not deterministic (currently it's random).@@ -269,25 +282,36 @@ curl -X POST http://localhost:8983/api/collections/techproducts/shards -H 'Conte=== MOVEREPLICA Parameters`collection`::-The name of the collection. This parameter is required.+The name of the collection.+This parameter is required.`targetNode`::-The name of the destination node. This parameter is required.+The name of the destination node.+This parameter is required.`sourceNode`::-The name of the node that contains the replica to move. This parameter is required unless `replica` is specified. If `replica` is specified this parameter is ignored.+The name of the node that contains the replica to move.+This parameter is required unless `replica` is specified.+If `replica` is specified this parameter is ignored.`shard`::-The name of the shard for which a replica should be moved. This parameter is required unless `replica` is specified. If `replica` is specified, this parameter is ignored.+The name of the shard for which a replica should be moved.+This parameter is required unless `replica` is specified.+If `replica` is specified, this parameter is ignored.`replica`::-The name of the replica to move. This parameter is required unless `shard` and `sourceNode` are specified, however this parameter has precedence over those two parameters.+The name of the replica to move.+This parameter is required unless `shard` and `sourceNode` are specified, however this parameter has precedence over those two parameters.`timeout`::-The number of seconds to wait for the replica to be live in the new location before deleting the replica in the old location. Defaults to 600 seconds. Deletion will not occur and creation will not be rolled back in the event of a timeout, potentially leaving an extra replica. Presently, this parameter is ignored if the replica is an hdfs replica.+The number of seconds to wait for the replica to be live in the new location before deleting the replica in the old location.+Defaults to 600 seconds.+Deletion will not occur and creation will not be rolled back in the event of a timeout, potentially leaving an extra replica.+Presently, this parameter is ignored if the replica is an hdfs replica.`inPlaceMove`::-For replicas that use shared filesystems allow 'in-place' move that reuses shared data. Defaults to true, but is ignored if the replica does not have the property `shared_storage` with a value of `true`+For replicas that use shared filesystems allow 'in-place' move that reuses shared data.+Defaults to true, but is ignored if the replica does not have the property `shared_storage` with a value of `true``async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -299,7 +323,8 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronouDeletes a named replica from the specified collection and shard.-If the corresponding core is up and running the core is unloaded, the entry is removed from the clusterstate, and (by default) delete the instanceDir and dataDir. If the node/core is down, the entry is taken off the clusterstate and if the core comes up later it is automatically unregistered.+If the corresponding core is up and running the core is unloaded, the entry is removed from the clusterstate, and (by default) delete the instanceDir and dataDir.+If the node/core is down, the entry is taken off the clusterstate and if the core comes up later it is automatically unregistered.[.dynamic-tabs]--@@ -335,32 +360,42 @@ curl -X DELETE http://localhost:8983/api/collections/techproducts/shards/shard1/=== DELETEREPLICA Parameters`collection`::-The name of the collection. This parameter is required.+The name of the collection.+This parameter is required.`shard`::-The name of the shard that includes the replica to be removed. This parameter is required.+The name of the shard that includes the replica to be removed.+This parameter is required.`replica`::The name of the replica to remove.+-If `count` is used instead, this parameter is not required. Otherwise, this parameter must be supplied.+If `count` is used instead, this parameter is not required.+Otherwise, this parameter must be supplied.`count`::-The number of replicas to remove. If the requested number exceeds the number of replicas, no replicas will be deleted. If there is only one replica, it will not be removed.+The number of replicas to remove.+If the requested number exceeds the number of replicas, no replicas will be deleted.+If there is only one replica, it will not be removed.+-If `replica` is used instead, this parameter is not required. Otherwise, this parameter must be supplied.+If `replica` is used instead, this parameter is not required.+Otherwise, this parameter must be supplied.`deleteInstanceDir`::-By default Solr will delete the entire instanceDir of the replica that is deleted. Set this to `false` to prevent the instance directory from being deleted.+By default Solr will delete the entire instanceDir of the replica that is deleted.+Set this to `false` to prevent the instance directory from being deleted.`deleteDataDir`::-By default Solr will delete the dataDir of the replica that is deleted. Set this to `false` to prevent the data directory from being deleted.+By default Solr will delete the dataDir of the replica that is deleted.+Set this to `false` to prevent the data directory from being deleted.`deleteIndex`::-By default Solr will delete the index of the replica that is deleted. Set this to `false` to prevent the index directory from being deleted.+By default Solr will delete the index of the replica that is deleted.+Set this to `false` to prevent the index directory from being deleted.`onlyIfDown`::-When set to `true`, no action will be taken if the replica is active. Default `false`.+When set to `true`, no action will be taken if the replica is active.+Default `false`.`async`::Request ID to track this action which will be <<collections-api.adoc#asynchronous-calls,processed asynchronously>>.@@ -368,7 +403,8 @@ Request ID to track this action which will be <<collections-api.adoc#asynchronou[[addreplicaprop]]== ADDREPLICAPROP: Add Replica Property-Assign an arbitrary property to a particular replica and give it the value specified. If the property already exists, it will be overwritten with the new value.+Assign an arbitrary property to a particular replica and give it the value specified.+If the property already exists, it will be overwritten with the new value.[.dynamic-tabs]--@@ -410,18 +446,23 @@ curl -X POST http://localhost:8983/api/collections/techproducts -H 'Content-Type=== ADDREPLICAPROP Parameters`collection`::-The name of the collection the replica belongs to. This parameter is required.+The name of the collection the replica belongs to.+This parameter is required.`shard`::-The name of the shard the replica belongs to. This parameter is required.+The name of the shard the replica belongs to.+This parameter is required.`replica`::-The replica, e.g., `core_node1`. This parameter is required.+The replica, e.g., `core_node1`.+This parameter is required.`property`::-The name of the property to add. This property is required.+The name of the property to add.+This property is required.+-This will have the literal `property.` prepended to distinguish it from system-maintained properties. So these two forms are equivalent:+This will have the literal `property.` prepended to distinguish it from system-maintained properties.+So these two forms are equivalent:+`property=special`+@@ -430,18 +471,22 @@ and`property=property.special``property.value`::-The value to assign to the property. This parameter is required.+The value to assign to the property.+This parameter is required.`shardUnique`::-If `true`, then setting this property in one replica will remove the property from all other replicas in that shard. The default is `false`.+If `true`, then setting this property in one replica will remove the property from all other replicas in that shard.+The default is `false`.+There is one pre-defined property `preferredLeader` for which `shardUnique` is forced to `true` and an error returned if `shardUnique` is explicitly set to `false`.+-`preferredLeader` is a boolean property. Any value assigned that is not equal (case insensitive) to `true` will be interpreted as `false` for `preferredLeader`.+`preferredLeader` is a boolean property.+Any value assigned that is not equal (case insensitive) to `true` will be interpreted as `false` for `preferredLeader`.=== ADDREPLICAPROP Response-The response will include the status of the request. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request.+If the status is anything other than "0", an error message will explain why the request failed.=== Additional Examples using ADDREPLICAPROP@@ -511,16 +556,21 @@ curl -X POST http://localhost:8983/api/collections/techproducts -H 'Content-Type=== DELETEREPLICAPROP Parameters`collection`::-The name of the collection the replica belongs to. This parameter is required.+The name of the collection the replica belongs to.+This parameter is required.`shard`::-The name of the shard the replica belongs to. This parameter is required.+The name of the shard the replica belongs to.+This parameter is required.`replica`::-The replica, e.g., `core_node1`. This parameter is required.+The replica, e.g., `core_node1`.+This parameter is required.`property`::-The property to add. This will have the literal `property.` prepended to distinguish it from system-maintained properties. So these two forms are equivalent:+The property to add.+This will have the literal `property.` prepended to distinguish it from system-maintained properties.+So these two forms are equivalent:+`property=special`+@@ -530,4 +580,5 @@ and=== DELETEREPLICAPROP Response-The response will include the status of the request. If the status is anything other than "0", an error message will explain why the request failed.+The response will include the status of the request.+If the status is anything other than "0", an error message will explain why the request failed.diff --git a/solr/solr-ref-guide/src/replication-screen.adoc b/solr/solr-ref-guide/src/replication-screen.adocdeleted file mode 100644index a5602662b56..00000000000--- a/solr/solr-ref-guide/src/replication-screen.adoc+++ /dev/null@@ -1,33 +0,0 @@-= Replication Screen-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Replication screen shows you the current replication state for the core you have specified. <<solrcloud.adoc#,SolrCloud>> has supplanted much of this functionality, but if you are still using Leader-Follower index replication, you can use this screen to:--. View the replicatable index state. (on a leader node)-. View the current replication status (on a follower node)-. Disable replication. (on a leader node)--.Caution When Using SolrCloud-[IMPORTANT]-====-When using <<getting-started-with-solrcloud.adoc#,SolrCloud>>, do not attempt to disable replication via this screen.--image::images/replication-screen/replication.png[image,width=412,height=250]-====--More details on how to configure replication is available in the section called <<index-replication.adoc#,Index Replication>>.diff --git a/solr/solr-ref-guide/src/request-parameters-api.adoc b/solr/solr-ref-guide/src/request-parameters-api.adocindex 5e338c63379..4600de35ce2 100644--- a/solr/solr-ref-guide/src/request-parameters-api.adoc+++ b/solr/solr-ref-guide/src/request-parameters-api.adoc@@ -20,7 +20,11 @@ The Request Parameters API allows creating parameter sets, a.k.a. paramsets, thaThe parameter sets defined with this API can be used in requests to Solr, or referenced directly in `solrconfig.xml` request handler definitions.-It is really another endpoint of the <<config-api.adoc#,Config API>> instead of a separate API, and has distinct commands. It does not replace or modify any sections of `solrconfig.xml`, but instead provides another approach to handling parameters used in requests. It behaves in the same way as the Config API, by storing parameters in another file that will be used at runtime. In this case, the parameters are stored in a file named `params.json`. This file is kept in ZooKeeper or in the `conf` directory of a standalone Solr instance.+It is really another endpoint of the <<config-api.adoc#,Config API>> instead of a separate API, and has distinct commands.+It does not replace or modify any sections of `solrconfig.xml`, but instead provides another approach to handling parameters used in requests.+It behaves in the same way as the Config API, by storing parameters in another file that will be used at runtime.+In this case, the parameters are stored in a file named `params.json`.+This file is kept in ZooKeeper when running SolrCloud or in the `conf` directory of a user-managed cluster or single-node installation.The settings stored in `params.json` are used at query time to override settings defined in `solrconfig.xml` in some cases as described below.@@ -37,13 +41,17 @@ All requests are sent to the `/config/params` endpoint of the Config API.== Configuring Request Parameters-The request to set, or update request parameters is sent as a set of Maps with names. These objects can be directly used in a request or a request handler definition. The request to delete parameter set is sent as list of names or a name.+The request to set, or update request parameters is sent as a set of maps with names.+These objects can be directly used in a request or a request handler definition.+The request to delete parameter set is sent as list of names or a name.The available commands are:* `set`: Create or overwrite a parameter set map.* `delete`: Remove a request parameter set map.-* `update`: update a parameter set map. This is equivalent to a `map.putAll(newMap)`. Both the maps are merged and if the new map has same keys as old they are overwritten.+* `update`: update a parameter set map.+This is equivalent to a `map.putAll(newMap)`.+Both maps are merged and if the new map has same keys as the old they are overwritten.You can mix these commands into a single request if necessary.@@ -66,7 +74,8 @@ curl http://localhost:8983/solr/techproducts/config/params -H 'Content-type:appl}'-----In the above example all the parameters are equivalent to the "defaults" in `solrconfig.xml`. It is possible to add invariants and appends as follows:+In the above example all the parameters are equivalent to the "defaults" in `solrconfig.xml`.+It is possible to add invariants and appends as follows:[source,bash]----@@ -113,7 +122,7 @@ After creating the `my_handler_params` paramset in the above section, it is poss<requestHandler name="/my_handler" class="solr.SearchHandler" useParams="my_handler_params"/>-----It will be equivalent to a standard request handler definition such as this one:+It will be equivalent to a standard request handler definition like:[source,xml]----@@ -135,7 +144,8 @@ It will be equivalent to a standard request handler definition such as this one:=== Implicit RequestHandlers with the Request Parameters API-Solr ships with many out-of-the-box request handlers that may only be configured via the Request Parameters API, because their configuration is not present in `solrconfig.xml`. See <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>> for the paramset to use when configuring an implicit request handler.+Solr ships with many out-of-the-box request handlers that may only be configured via the Request Parameters API, because their configuration is not present in `solrconfig.xml`.+See <<implicit-requesthandlers.adoc#,Implicit Request Handlers>> for the paramset to use when configuring an implicit request handler.=== Viewing Expanded Paramsets and Effective Parameters with RequestHandlers@@ -143,7 +153,7 @@ To see the expanded paramset and the resulting effective parameters for a Reques[source,bash]-----curl http://localhost:8983/solr/techproducts/config/requestHandler?componentName=/export&expandParams=true+$ curl http://localhost:8983/solr/techproducts/config/requestHandler?componentName=/export&expandParams=true----== Viewing Request Parameters@@ -152,15 +162,16 @@ To see the paramsets that have been created, you can use the `/config/params` en[source,bash]-----curl http://localhost:8983/solr/techproducts/config/params+$ curl http://localhost:8983/solr/techproducts/config/params#Or use the paramset name-curl http://localhost:8983/solr/techproducts/config/params/myQueries+$ curl http://localhost:8983/solr/techproducts/config/params/myQueries----== The useParams Parameter-When making a request, the `useParams` parameter applies the request parameters sent to the request. This is translated at request time to the actual parameters.+When making a request, the `useParams` parameter applies the request parameters sent to the request.+This is translated at request time to the actual parameters.For example (using the names we set up in the earlier examples, please replace with your own name):@@ -169,16 +180,21 @@ For example (using the names we set up in the earlier examples, please replace whttp://localhost/solr/techproducts/select?useParams=myQueries-----It is possible to pass more than one parameter set in the same request. For example:+It is possible to pass more than one parameter set in the same request.+For example:[source,text]----http://localhost/solr/techproducts/select?useParams=myFacets,myQueries-----In the above example the parameter set 'myQueries' is applied on top of 'myFacets'. So, values in 'myQueries' take precedence over values in 'myFacets'. Additionally, any values passed in the request take precedence over `useParams` parameters. This acts like the "defaults" specified in the `<requestHandler>` definition in `solrconfig.xml`.+In the above example the parameter set 'myQueries' is applied on top of 'myFacets'.+So, values in 'myQueries' take precedence over values in 'myFacets'.+Additionally, any values passed in the request take precedence over `useParams` parameters.+This acts like the "defaults" specified in the `<requestHandler>` definition in `solrconfig.xml`.-The parameter sets can be used directly in a request handler definition as follows. Please note that the `useParams` specified is always applied even if the request contains `useParams`.+The parameter sets can be used directly in a request handler definition as follows.+Please note that the `useParams` specified is always applied even if the request contains `useParams`.[source,xml]----@@ -204,8 +220,10 @@ To summarize, parameters are applied in this order:== Public APIs-The RequestParams Object can be accessed using the method `SolrConfig#getRequestParams()`. Each paramset can be accessed by their name using the method `RequestParams#getRequestParams(String name)`.+The RequestParams Object can be accessed using the method `SolrConfig#getRequestParams()`.+Each paramset can be accessed by their name using the method `RequestParams#getRequestParams(String name)`.== Examples Using the Request Parameters API-The Solr "films" example demonstrates the use of the parameters API. You can use this example in your Solr installation (in the `example/films` directory) or view the files in the Apache GitHub mirror at https://github.com/apache/lucene-solr/tree/leader/solr/example/films.+The Solr "films" example demonstrates the use of the parameters API.+You can use this example in your Solr installation (in the `example/films` directory) or view the files in the Solr GitHub mirror at https://github.com/apache/solr/tree/main/solr/example/films.diff --git a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc b/solr/solr-ref-guide/src/requestdispatcher.adocsimilarity index 54%rename from solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adocrename to solr/solr-ref-guide/src/requestdispatcher.adocindex 77a1ddf65d2..290eadf9b4e 100644--- a/solr/solr-ref-guide/src/requestdispatcher-in-solrconfig.adoc+++ b/solr/solr-ref-guide/src/requestdispatcher.adoc@@ -1,4 +1,4 @@-= RequestDispatcher in SolrConfig+= RequestDispatcher// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -33,7 +33,8 @@ It governs how Solr responds to requests such as `/select?qt=XXX`.The default value "false" will ignore requests to `/select` if a request handler is not explicitly registered with the name `/select`.A value of "true" will route query requests to the parser defined with the `qt` value if a request handler is not explicitly registered with the name `/select`.-In recent versions of Solr, a `/select` request handler is defined by default, so a value of "false" will work fine. See the section <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,RequestHandlers and SearchComponents in SolrConfig>> for more information.+In recent versions of Solr, a `/select` request handler is defined by default, so a value of "false" will work fine.+See the section <<requesthandlers-searchcomponents.adoc#,Request Handlers and Search Components>> for more information.[source,xml]----@@ -44,24 +45,65 @@ In recent versions of Solr, a `/select` request handler is defined by default, s== requestParsers Element-The `<requestParsers>` sub-element controls values related to parsing requests. This is an empty XML element that doesn't have any content, only attributes.+The `<requestParsers>` sub-element controls values related to parsing requests.+This is an empty XML element that doesn't have any content, only attributes.`enableRemoteStreaming`::-This attribute controls whether remote streaming of content is allowed. If omitted or set to `false` (the default), streaming will not be allowed. Setting it to `true` lets you specify the location of content to be streamed using `stream.file` or `stream.url` parameters.+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++Controls whether remote streaming of content is allowed.+When set to `false`, streaming will not be allowed.+Setting it to `true` lets you specify the location of content to be streamed using `stream.file` or `stream.url` parameters.`enableStreamBody`::-This attribute controls whether streaming content from the HTTP parameter `stream.body` is allowed. If omitted or set to `false` (the default), streaming will not be allowed. Setting it to `true` lets you pass data in the `stream.body` parameter.+-If you enable remote streaming, be sure that you have authentication enabled. Otherwise, someone could potentially gain access to your content by accessing arbitrary URLs. It's also a good idea to place Solr behind a firewall to prevent it from being accessed from untrusted clients.+[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++This attribute controls whether streaming content from the HTTP parameter `stream.body` is allowed.+When set to `false`, streaming will not be allowed.+Setting it to `true` lets you pass data in the `stream.body` parameter.+++If you enable remote streaming, be sure that you have authentication enabled.+Otherwise, someone could potentially gain access to your content by accessing arbitrary URLs.+It's also a good idea to place Solr behind a firewall to prevent it from being accessed from untrusted clients.`multipartUploadLimitInKB`::-This attribute sets an upper limit in kilobytes on the size of a document that may be submitted in a multi-part HTTP POST request. The value specified is multiplied by 1024 to determine the size in bytes. A value of `-1` means MAX_INT, which is also the system default if omitted.+++[%autowidth,frame=none]+|===+|Optional |Default: `-1`+|===+++This attribute sets an upper limit in kilobytes on the size of a document that may be submitted in a multi-part HTTP POST request.+The value specified is multiplied by 1024 to determine the size in bytes.+A value of `-1` means MAX_INT, which is also the system default if omitted.`formdataUploadLimitInKB`::-This attribute sets a limit in kilobytes on the size of form data (`application/x-www-form-urlencoded`) submitted in a HTTP POST request, which can be used to pass request parameters that will not fit in a URL. A value of `-1` means MAX_INT, which is also the system default if omitted.+++[%autowidth,frame=none]+|===+|Optional |Default: `-1`+|===+++This attribute sets a limit in kilobytes on the size of form data (`application/x-www-form-urlencoded`) submitted in a HTTP POST request, which can be used to pass request parameters that will not fit in a URL.+A value of `-1` means MAX_INT, which is also the system default if omitted.`addHttpRequestToContext`::-This attribute can be used to indicate that the original `HttpServletRequest` object should be included in the context map of the `SolrQueryRequest` using the key `httpRequest`. This `HttpServletRequest` is not used by any Solr component, but may be useful when developing custom plugins.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++This attribute can be used to indicate that the original `HttpServletRequest` object should be included in the context map of the `SolrQueryRequest` using the key `httpRequest`.+This `HttpServletRequest` is not used by any Solr component, but may be useful when developing custom plugins.[source,xml]----@@ -87,7 +129,7 @@ curl -H 'Content-type:application/json' -d '{"set-property": {"requestDispatcher[example.tab-pane#v2enablestreaming]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node*[source,bash]----curl -H 'Content-type:application/json' -d '{"set-property": {"requestDispatcher.requestParsers.enableRemoteStreaming": true}, "set-property":{"requestDispatcher.requestParsers.enableStreamBody": true}}' http://localhost:8983/api/cores/gettingstarted/config@@ -106,18 +148,47 @@ curl -H 'Content-type:application/json' -d '{"set-property": {"requestDispatcher== httpCaching Element-The `<httpCaching>` element controls HTTP cache control headers. Do not confuse these settings with Solr's internal cache configuration. This element controls caching of HTTP responses as defined by the W3C HTTP specifications.+The `<httpCaching>` element controls HTTP cache control headers.+Do not confuse these settings with Solr's internal cache configuration.+This element controls caching of HTTP responses as defined by the W3C HTTP specifications.-This element allows for three attributes and one sub-element. The attributes of the `<httpCaching>` element control whether a 304 response to a GET request is allowed, and if so, what sort of response it should be. When an HTTP client application issues a GET, it may optionally specify that a 304 response is acceptable if the resource has not been modified since the last time it was fetched.+This element allows for three attributes and one sub-element.+The attributes of the `<httpCaching>` element control whether a 304 response to a GET request is allowed, and if so, what sort of response it should be.+When an HTTP client application issues a GET, it may optionally specify that a 304 response is acceptable if the resource has not been modified since the last time it was fetched.`never304`::-If present with the value `true`, then a GET request will never respond with a 304 code, even if the requested resource has not been modified. When this attribute is set to true, the next two attributes are ignored. Setting this to true is handy for development, as the 304 response can be confusing when tinkering with Solr responses through a web browser or other client that supports cache headers.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++If present with the value `true`, then a GET request will never respond with a 304 code, even if the requested resource has not been modified.+When this attribute is set to `true`, the next two attributes are ignored.+Setting this to `true` is handy for development, as the 304 response can be confusing when tinkering with Solr responses through a web browser or other client that supports cache headers.`lastModFrom`::-This attribute may be set to either `openTime` (the default) or `dirLastMod`. The value `openTime` indicates that last modification times, as compared to the If-Modified-Since header sent by the client, should be calculated relative to the time the Searcher started. Use `dirLastMod` if you want times to exactly correspond to when the index was last updated on disk.+++[%autowidth,frame=none]+|===+|Optional |Default: `openTime`+|===+++This attribute may be set to either `openTime` or `dirLastMod`.+++The value `openTime` indicates that last modification times, as compared to the If-Modified-Since header sent by the client, should be calculated relative to the time the Searcher started.+++Use `dirLastMod` if you want times to exactly correspond to when the index was last updated on disk.`etagSeed`::-This value of this attribute is sent as the value of the `ETag` header. Changing this value can be helpful to force clients to re-fetch content even when the indexes have not changed---for example, when you've made some changes to the configuration.+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++This value of this attribute is sent as the value of the `ETag` header.+Changing this value can be helpful to force clients to re-fetch content even when the indexes have not changed; for example, when you've made some changes to the configuration.[source,xml]----@@ -130,6 +201,12 @@ This value of this attribute is sent as the value of the `ETag` header. Changing=== cacheControl Element-In addition to these attributes, `<httpCaching>` accepts one child element: `<cacheControl>`. The content of this element will be sent as the value of the Cache-Control header on HTTP responses. This header is used to modify the default caching behavior of the requesting client. The possible values for the Cache-Control header are defined by the HTTP 1.1 specification in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9[Section 14.9].+In addition to these attributes, `<httpCaching>` accepts one child element: `<cacheControl>`.+The content of this element will be sent as the value of the Cache-Control header on HTTP responses.+This header is used to modify the default caching behavior of the requesting client.+The possible values for the Cache-Control header are defined by the HTTP 1.1 specification in http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9[Section 14.9].-Setting the max-age field controls how long a client may re-use a cached response before requesting it again from the server. This time interval should be set according to how often you update your index and whether or not it is acceptable for your application to use content that is somewhat out of date. Setting `must-revalidate` will tell the client to validate with the server that its cached copy is still good before re-using it. This will ensure that the most timely result is used, while avoiding a second fetch of the content if it isn't needed, at the cost of a request to the server to do the check.+Setting the max-age field controls how long a client may re-use a cached response before requesting it again from the server.+This time interval should be set according to how often you update your index and whether or not it is acceptable for your application to use content that is somewhat out of date.+Setting `must-revalidate` will tell the client to validate with the server that its cached copy is still good before re-using it.+This will ensure that the most timely result is used, while avoiding a second fetch of the content if it isn't needed, at the cost of a request to the server to do the check.diff --git a/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc b/solr/solr-ref-guide/src/requesthandlers-searchcomponents.adocsimilarity index 86%rename from solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adocrename to solr/solr-ref-guide/src/requesthandlers-searchcomponents.adocindex c362d34cb3c..677432292a7 100644--- a/solr/solr-ref-guide/src/requesthandlers-and-searchcomponents-in-solrconfig.adoc+++ b/solr/solr-ref-guide/src/requesthandlers-searchcomponents.adoc@@ -1,4 +1,4 @@-= Request Handlers and Search Components in SolrConfig+= Request Handlers and Search Components// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,11 +18,13 @@After the `<query>` section of `solrconfig.xml`, request handlers and search components are configured.-A _request handler_ processes requests coming to Solr. These might be query requests, index update requests or specialized interactions such as <<ping.adoc#,ping>>.+A _request handler_ processes requests coming to Solr.+These might be query requests, index update requests or specialized interactions such as <<ping.adoc#,ping>>.-Not all handlers are defined explicitly in `solrconfig.xml`, many essential ones are actually defined <<implicit-requesthandlers.adoc#,implicitly>>.+Not all handlers are defined explicitly in `solrconfig.xml`, many are defined implicitly.+See <<implicit-requesthandlers.adoc#,Implicit Request Handlers>> for details.-Additionally, handlers can be defined - or even overridden - in `configoverlay.json` by using <<config-api.adoc#,Config API>>.+Additionally, handlers can be defined or overridden in `configoverlay.json` by using <<config-api.adoc#,Config API>>.Finally, independent parameter sets can be also defined by <<request-parameters-api.adoc#,Request Parameters API>>.They will be stored in `params.json` file and referenced with <<#paramsets-and-useparams,useParams>>.@@ -33,7 +35,8 @@ At the same time, using every single option explained in this guide, will most c== Defining and Calling Request Handlers-Every request handler is defined with a name and a class. The name of the request handler is referenced with the request to Solr, typically as a path.+Every request handler is defined with a name and a class.+The name of the request handler is referenced with the request to Solr, typically as a path.For example, if Solr is installed at `\http://localhost:8983/solr/`, and you have a collection named "gettingstarted", you can make a request that looks like this:[source,text]@@ -41,7 +44,8 @@ For example, if Solr is installed at `\http://localhost:8983/solr/`, and you havhttp://localhost:8983/solr/gettingstarted/select?q=solr-----This query will be processed by the request handler with the name `/select`. We've only used the "q" parameter here, which includes our query term, a simple keyword of "solr".+This query will be processed by the request handler with the name `/select`.+We've only used the "q" parameter here, which includes our query term, a simple keyword of "solr".If the request handler has more default parameters defined, those will be used with any query we send to that request handler unless they are overridden by the client (or user) in the query itself.If you have another request handler defined, you could send your request with that name.@@ -163,7 +167,7 @@ these are the only facets they will be able to see counts for; regardless of whaIt is also possible to configure defaults for request handlers with a section called `initParams`.These defaults can be used when you want to have common properties that will be used by each separate handler.For example, if you intend to create several request handlers that will all request the same list of fields in the response, you can configure an `initParams` section with your list of fields.-For more information about `initParams`, see the section <<initparams-in-solrconfig.adoc#,InitParams in SolrConfig>>.+For more information about `initParams`, see the section <<initparams.adoc#,InitParams>>.=== Paramsets and UseParamsIf you are expecting to change the parameters often, or if you want define sets of parameters that you can apply on the fly,@@ -206,9 +210,10 @@ The following sections are allowed within a Search Handler:All the blocks are optional, especially since parameters can also be provided with `initParams` and `useParams`.-The defaults/appends/invariants blocks were described <<#defaults-appends-and-invariants,higher>> on the page. All the parameters described in the section <<searching.adoc#,Searching>> can be defined as parameters for any of the Search Handlers.+The defaults/appends/invariants blocks were described <<#defaults-appends-and-invariants,higher>> on the page.+All the parameters described in the section <<query-guide.adoc#,Query Guide>> can be defined as parameters for any of the Search Handlers.-The Search Components blocks are described next, and <<distributed-requests.adoc#configuring-the-shardhandlerfactory,shardHandlerFactory>> is for fine-tuning of the SolrCloud distributed requests.+The Search Components blocks are described next, and <<solrcloud-distributed-requests.adoc#configuring-the-shardhandlerfactory,shardHandlerFactory>> is for fine-tuning of the SolrCloud distributed requests.=== Defining Search ComponentsThe search components themselves are defined outside of the Request Handlers and then are referenced from various Search Handlers that want to use them.@@ -217,19 +222,20 @@ It is quite rare - and somewhat brittle - to completely override the component s==== Default Components-As you can see below, what we see as a search experience is mostly a sequence of components defined below. They are called in the order listed.+As you can see below, what we see as a search experience is mostly a sequence of components defined below.+They are called in the order listed.[cols="20,40,40",options="header"]|===|Component Name |Class Name |More Information-|query |`solr.QueryComponent` |Described in the section <<query-syntax-and-parsing.adoc#,Query Syntax and Parsing>>.+|query |`solr.QueryComponent` |Described in the section <<query-syntax-and-parsers.adoc#,Query Syntax and Parsing>>.|facet |`solr.FacetComponent` |Original parameter-based facet component, described in the section <<faceting.adoc#,Faceting>>.|facet_module |`solr.facet.FacetModule` | JSON Faceting and Analytics module, described in the section <<json-facet-api.adoc#, JSON Facet API>>.|mlt |`solr.MoreLikeThisComponent` |Described in the section <<morelikethis.adoc#,MoreLikeThis>>.|highlight |`solr.HighlightComponent` |Described in the section <<highlighting.adoc#,Highlighting>>.-|stats |`solr.StatsComponent` |Described in the section <<the-stats-component.adoc#,The Stats Component>>.+|stats |`solr.StatsComponent` |Described in the section <<stats-component.adoc#,The Stats Component>>.|expand |`solr.ExpandComponent` |Described in the section <<collapse-and-expand-results.adoc#,Collapse and Expand Results>>.-|terms |`solr.TermsComponent` |Described in the section <<the-terms-component.adoc#,The Terms Component>>.+|terms |`solr.TermsComponent` |Described in the section <<terms-component.adoc#,The Terms Component>>.|debug |`solr.DebugComponent` |Described in the section on <<common-query-parameters.adoc#debug-parameter,Common Query Parameters>>.|===@@ -240,12 +246,12 @@ They do need to defined and referenced in `solrconfig.xml` to be actually used.* `AnalyticsComponent`, described in the section <<analytics.adoc#,Analytics>>.* `ClusteringComponent`, described in the section <<result-clustering.adoc#,Result Clustering>>.* `PhrasesIdentificationComponent`, used to identify & score "phrases" found in the input string, based on shingles in indexed fields, described in the {solr-javadocs}/core/org/apache/solr/handler/component/PhrasesIdentificationComponent.html[PhrasesIdentificationComponent] javadocs.-* `QueryElevationComponent`, described in the section <<the-query-elevation-component.adoc#,The Query Elevation Component>>.+* `QueryElevationComponent`, described in the section <<query-elevation-component.adoc#,The Query Elevation Component>>.* `RealTimeGetComponent`, described in the section <<realtime-get.adoc#,RealTime Get>>.* `ResponseLogComponent`, used to record which documents are returned to the user via the Solr log, described in the {solr-javadocs}/core/org/apache/solr/handler/component/ResponseLogComponent.html[ResponseLogComponent] javadocs.* `SpellCheckComponent`, described in the section <<spell-checking.adoc#,Spell Checking>>.* `SuggestComponent`, described in the section <<suggester.adoc#,Suggester>>.-* `TermVectorComponent`, described in the section <<the-term-vector-component.adoc#,The Term Vector Component>>.+* `TermVectorComponent`, described in the section <<term-vector-component.adoc#,The Term Vector Component>>.Some third party components are also linked from https://solr.cool/ website.@@ -262,7 +268,8 @@ To define custom component, the syntax is:</searchComponent>-----Custom components often have configuration elements not described here. Check specific component's documentation/examples for details.+Custom components often have configuration elements not described here.+Check specific component's documentation/examples for details.Notice: If you register a new search component with one of the default names, the newly defined component will be used instead of the default.This allows to override a specific component, while not having to worry so much about upgrading Solr.@@ -302,7 +309,8 @@ This should be considered as a last-resort option as the default list may change== Update Request Handlers-The Update Request Handlers are request handlers which process updates to the index. Most of the request handlers are <<implicit-requesthandlers.adoc#update-handlers,implicit>>+The Update Request Handlers are request handlers which process updates to the index.+Most of the request handlers are <<implicit-requesthandlers.adoc#update-handlers,implicit>>and can be customized by defining properly named Paramsets.If you need to define additional Update Request Handler, the syntax is:@@ -315,10 +323,10 @@ If you need to define additional Update Request Handler, the syntax is:-----The full details are covered in the section <<uploading-data-with-index-handlers.adoc#,Uploading Data with Index Handlers>>.+The full details are covered in the section <<indexing-with-update-handlers.adoc#,Indexing with Update Handlers>>.Similar to Search Components for Search Handlers, Solr has document-preprocessing plugins for Update Request Handlers,called <<update-request-processors.adoc#,Update Request Processors>>,which also allow for default and custom configuration chains.-Note: Do not confuse Update Request Handlers with <<updatehandlers-in-solrconfig.adoc#,`updateHandler`>> section also defined in `solrconfig.xml`.+Note: Do not confuse Update Request Handlers with <<commits-transaction-logs.adoc#,`updateHandler`>> section also defined in `solrconfig.xml`.diff --git a/solr/solr-ref-guide/src/resource-loading.adoc b/solr/solr-ref-guide/src/resource-loading.adocindex 0886d45302e..d489a35ea0e 100644--- a/solr/solr-ref-guide/src/resource-loading.adoc+++ b/solr/solr-ref-guide/src/resource-loading.adoc@@ -19,24 +19,26 @@Solr components can be configured using *resources*: data stored in external files that may be referred to in a location-independent fashion.-Examples of resources include: files needed by schema components, e.g., a stopword list for <<filter-descriptions.adoc#stop-filter,Stop Filter>>; and machine-learned models for <<learning-to-rank.adoc#,Learning to Rank>>.+Examples of resources include: files needed by schema components, e.g., a stopword list for <<filters.adoc#stop-filter,Stop Filter>>; and machine-learned models for <<learning-to-rank.adoc#,Learning to Rank>>._Resources are typically resolved from the configSet_ but there are other options too.Solr's resources are generally only loaded initially when the Solr collection or Solr core is loaded.-After you update a resource, you'll typically need to _reload_ the affected collections (SolrCloud) or the cores (standalone Solr).+After you update a resource, you'll typically need to _reload_ the affected collections when running SolrCloud, or the cores when running a user-managed cluster or single-node installation.Restarting all affected Solr nodes also works.<<managed-resources.adoc#,Managed resources>> can be manipulated via APIs and do not need an explicit reload.== Resources in Configsets<<config-sets.adoc#,Configsets>> are the directories containing solrconfig.xml, the schema, and resources referenced by them.-In SolrCloud they are in ZooKeeper whereas in standalone they are on the file system.-In either mode, configsets might be shared or might be dedicated to a configset.+In SolrCloud they are stored in ZooKeeper.+In a user-managed cluster and a single-node installation they are stored on the file system.+In any mode, resources may be shared or may be dedicated to a configSet.Prefer to put resources here.== Resources in Other Places-Resources can also be placed in an arbitrary directory and <<libs.adoc#lib-directives-in-solrconfig,referenced>> from a `<lib />` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file. Example: `<lib path="/volume/models/" />`+Resources can also be placed in an arbitrary directory and <<libs.adoc#lib-directives-in-solrconfig,referenced>> from a `<lib />` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file.+Example: `<lib path="/volume/models/" />`This choice may make sense if the resource is too large for a configset in ZooKeeper.However it's up to you to somehow ensure that all nodes in your cluster have access to these resources.diff --git a/solr/solr-ref-guide/src/response-writers.adoc b/solr/solr-ref-guide/src/response-writers.adocindex b1321657c1f..c40bc267f37 100644--- a/solr/solr-ref-guide/src/response-writers.adoc+++ b/solr/solr-ref-guide/src/response-writers.adoc@@ -21,7 +21,8 @@ A Response Writer generates the formatted response of a search.Solr supports a variety of Response Writers to ensure that query responses can be parsed by the appropriate language or application.-The `wt` parameter selects the Response Writer to be used. The list below describe shows the most common settings for the `wt` parameter, with links to further sections that discuss them in more detail.+The `wt` parameter selects the Response Writer to be used.+The list below describe shows the most common settings for the `wt` parameter, with links to further sections that discuss them in more detail.* <<CSV Response Writer,csv>>* <<GeoJSON Response Writer,geojson>>@@ -94,17 +95,21 @@ browser expects `application/json` will trigger the browser to block the request==== json.nl-This parameter controls the output format of NamedLists, where order is more important than access by name. NamedList is currently used for field faceting data.+This parameter controls the output format of NamedLists, where order is more important than access by name.+NamedList is currently used for field faceting data.The `json.nl` parameter takes the following values:`flat`::-The default. NamedList is represented as a flat array, alternating names and values.+The default.+NamedList is represented as a flat array, alternating names and values.+With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `["a",1, "bar","foo", null,3, null,null]`.`map`::-NamedList is represented as a JSON object. Although this is the simplest mapping, a NamedList can have optional keys, repeated keys, and preserves order. Using a JSON object (essentially a map or hash) for a NamedList results in the loss of some information.+NamedList is represented as a JSON object.+Although this is the simplest mapping, a NamedList can have optional keys, repeated keys, and preserves order.+Using a JSON object (essentially a map or hash) for a NamedList results in the loss of some information.+With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output would be `{"a":1, "bar":"foo", "":3, "":null}`.@@ -133,14 +138,16 @@ With input of `NamedList("a"=1, "bar"="foo", null=3, null=null)`, the output wou== Standard XML Response Writer-The XML Response Writer is the most general purpose and reusable Response Writer currently included with Solr. It is the format used in most discussions and documentation about the response of Solr queries.+The XML Response Writer is the most general purpose and reusable Response Writer currently included with Solr.+It is the format used in most discussions and documentation about the response of Solr queries.Note that the XSLT Response Writer can be used to convert the XML produced by this writer to other vocabularies or text-based formats.The behavior of the XML Response Writer can be driven by the following query parameters.`version`::-The `version` parameter determines the XML protocol used in the response. Clients are strongly encouraged to _always_ specify the protocol version, so as to ensure that the format of the response they receive does not change unexpectedly if the Solr server is upgraded and a new default format is introduced.+The `version` parameter determines the XML protocol used in the response.+Clients are strongly encouraged to _always_ specify the protocol version, so as to ensure that the format of the response they receive does not change unexpectedly if the Solr server is upgraded and a new default format is introduced.+The only currently supported version value is `2.2`. The format of the `responseHeader` changed to use the same `<lst>` structure as the rest of the response.+@@ -153,7 +160,8 @@ The default behavior is not to return any stylesheet declaration at all.+[IMPORTANT]====-Use of the `stylesheet` parameter is discouraged, as there is currently no way to specify external stylesheets, and no stylesheets are provided in the Solr distributions. This is a legacy parameter, which may be developed further in a future release.+Use of the `stylesheet` parameter is discouraged, as there is currently no way to specify external stylesheets, and no stylesheets are provided in the Solr distributions.+This is a legacy parameter, which may be developed further in a future release.====`indent`::@@ -163,7 +171,8 @@ The default behavior is not to indent.== XSLT Response Writer-The XSLT Response Writer applies an XML stylesheet to output. It can be used for tasks such as formatting results for an RSS feed.+The XSLT Response Writer applies an XML stylesheet to output.+It can be used for tasks such as formatting results for an RSS feed.This response writer has been added as part of the Scripting contrib module.@@ -171,7 +180,8 @@ Learn more about adding the `dist/solr-scripting-*.jar` file into Solr's <<libs.=== tr Parameter-The XSLT Response Writer accepts one parameter: the `tr` parameter, which identifies the XML transformation to use. The transformation must be found in the Solr `conf/xslt` directory.+The XSLT Response Writer accepts one parameter: the `tr` parameter, which identifies the XML transformation to use.+The transformation must be found in the Solr `conf/xslt` directory.The Content-Type of the response is set according to the `<xsl:output>` statement in the XSLT transform, for example: `<xsl:output media-type="text/html"/>`@@ -191,7 +201,8 @@ The example below, from the `sample_techproducts_configs` <<config-sets.adoc#,co</queryResponseWriter>-----A value of 5 for `xsltCacheLifetimeSeconds` is good for development, to see XSLT changes quickly. For production you probably want a much higher value.+A value of 5 for `xsltCacheLifetimeSeconds` is good for development, to see XSLT changes quickly.+For production you probably want a much higher value.=== XSLT Writer Example@@ -241,11 +252,15 @@ Lastly, the `luke.xsl` transformation demonstrates that you can apply very sophi== Binary Response Writer-This is a custom binary format used by Solr for inter-node communication as well as client-server communication. SolrJ uses this as the default for indexing as well as querying. See <<client-apis.adoc#,Client APIs>> for more details.+This is a custom binary format used by Solr for inter-node communication as well as client-server communication.+SolrJ uses this as the default for indexing as well as querying.+See <<client-apis.adoc#,Client APIs>> for more details.== GeoJSON Response Writer-Returns Solr results in http://geojson.org[GeoJSON] augmented with Solr-specific JSON. To use this, set `wt=geojson` and `geojson.field` to the name of a spatial Solr field. Not all spatial fields types are supported, and you'll get an error if you use an unsupported one.+Returns Solr results in http://geojson.org[GeoJSON] augmented with Solr-specific JSON.+To use this, set `wt=geojson` and `geojson.field` to the name of a spatial Solr field.+Not all spatial fields types are supported, and you'll get an error if you use an unsupported one.== Python Response Writer@@ -260,7 +275,8 @@ Solr has an optional Python response format that extends its JSON output in the[[php-writer]]== PHP Response Writer and PHP Serialized Response Writer-Solr has a PHP response format that outputs an array (as PHP code) which can be evaluated. Setting the `wt` parameter to `php` invokes the PHP Response Writer.+Solr has a PHP response format that outputs an array (as PHP code) which can be evaluated.+Setting the `wt` parameter to `php` invokes the PHP Response Writer.Example usage:@@ -271,7 +287,8 @@ eval("$result = " . $code . ";");print_r($result);-----Solr also includes a PHP Serialized Response Writer that formats output in a serialized array. Setting the `wt` parameter to `phps` invokes the PHP Serialized Response Writer.+Solr also includes a PHP Serialized Response Writer that formats output in a serialized array.+Setting the `wt` parameter to `phps` invokes the PHP Serialized Response Writer.Example usage:@@ -288,7 +305,8 @@ Solr has an optional Ruby response format that extends its JSON output in the fo* Ruby's single quoted strings are used to prevent possible string exploits.* \ and ' are the only two characters escaped.-* Unicode escapes are not used. Data is written as raw UTF-8.+* Unicode escapes are not used.+Data is written as raw UTF-8.* nil used for null.* \=> is used as the key/value separator in maps.@@ -307,13 +325,15 @@ rsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'\] }== CSV Response Writer-The CSV response writer returns a list of documents in comma-separated values (CSV) format. Other information that would normally be included in a response, such as facet information, is excluded.+The CSV response writer returns a list of documents in comma-separated values (CSV) format.+Other information that would normally be included in a response, such as facet information, is excluded.-The CSV response writer supports multi-valued fields, as well as <<transforming-result-documents.adoc#,pseudo-fields>>, and the output of this CSV format is compatible with Solr's <<uploading-data-with-index-handlers.adoc#csv-formatted-index-updates,CSV update format>>.+The CSV response writer supports multi-valued fields, as well as <<document-transformers.adoc#,pseudo-fields>>, and the output of this CSV format is compatible with Solr's <<indexing-with-update-handlers.adoc#csv-formatted-index-updates,CSV update format>>.=== CSV Parameters-These parameters specify the CSV format that will be returned. You can accept the default values or specify your own.+These parameters specify the CSV format that will be returned.+You can accept the default values or specify your own.[width="50%",options="header",]|===@@ -323,12 +343,14 @@ These parameters specify the CSV format that will be returned. You can accept th|csv.separator |`,`|csv.header |Defaults to `true`. If `false`, Solr does not print the column headers.|csv.newline |`\n`-|csv.null |Defaults to a zero length string. Use this parameter when a document has no value for a particular field.+|csv.null |Defaults to a zero length string.+Use this parameter when a document has no value for a particular field.|====== Multi-Valued Field CSV Parameters-These parameters specify how multi-valued fields are encoded. Per-field overrides for these values can be done using `f.<fieldname>.csv.separator=|`.+These parameters specify how multi-valued fields are encoded.+Per-field overrides for these values can be done using `f.<fieldname>.csv.separator=|`.[width="50%",options="header",]|===@@ -356,9 +378,13 @@ The Smile format is a JSON-compatible binary format, described in detail here: h== XLSX Response Writer-Use this to get the response as a spreadsheet in the .xlsx (Microsoft Excel) format. It accepts parameters in the form `colwidth.<field-name>` and `colname.<field-name>` which helps you customize the column widths and column names.+Use this to get the response as a spreadsheet in the .xlsx (Microsoft Excel) format.+It accepts parameters in the form `colwidth.<field-name>` and `colname.<field-name>` which helps you customize the column widths and column names.-This response writer has been added as part of the extraction library, and will only work if the extraction contrib is present in the server classpath. Defining the classpath with the `lib` directive is not sufficient. Instead, you will need to copy the necessary .jars to the Solr webapp's `lib` directory manually. You can run these commands from your `$SOLR_INSTALL` directory:+This response writer has been added as part of the extraction library, and will only work if the extraction contrib is present in the server classpath.+Defining the classpath with the `lib` directive is not sufficient.+Instead, you will need to copy the necessary .jars to the Solr webapp's `lib` directory manually.+You can run these commands from your `$SOLR_INSTALL` directory:[source,bash]----diff --git a/solr/solr-ref-guide/src/result-clustering.adoc b/solr/solr-ref-guide/src/result-clustering.adocindex 1563a0968a3..54bdd67ccaa 100644--- a/solr/solr-ref-guide/src/result-clustering.adoc+++ b/solr/solr-ref-guide/src/result-clustering.adoc@@ -18,24 +18,27 @@[IMPORTANT]====-The clustering component implementation and API (parameters) have changed significantly-in version 9.0. Please refer to the Solr Guide version matching your Solr release-exactly.+The clustering component implementation and API (parameters) have changed significantly in version 9.0.+Please refer to the Solr Guide version matching your Solr release exactly.====The *clustering* (or *cluster analysis*) plugin attempts to automatically discover groups of related search hits (documents) and assign human-readable labels to these groups.The clustering algorithm in Solr is applied to documents included in search result of each single query -— this is called an _on-line_ clustering.-Clusters discovered for a given query can be perceived as _dynamic facets_. This is beneficial when regular faceting is difficult (field values are not known in advance) or when the queries are exploratory in nature. Take a look at the https://search.carrot2.org/#/search/web/apache%20solr/treemap[Carrot^2^] project's demo page to see an example of search results clustering in action (the groups in the visualization have been discovered automatically in search results to the right, there is no external information involved).+Clusters discovered for a given query can be perceived as _dynamic facets_. This is beneficial when regular faceting is difficult (field values are not known in advance) or when the queries are exploratory in nature.+Take a look at the https://search.carrot2.org/#/search/web/apache%20solr/treemap[Carrot^2^] project's demo page to see an example of search results clustering in action (the groups in the visualization have been discovered automatically in search results to the right, there is no external information involved).image::images/result-clustering/carrot2.png[image,width=900]-The query issued to the system was _Apache Solr_. It seems clear that faceting could not yield a similar set of groups, although the goals of both techniques are similar—to let the user explore the set of search results and either rephrase the query or narrow the focus to a subset of current documents. Clustering is also similar to <<result-grouping.adoc#,Result Grouping>> in that it can help to look deeper into search results, beyond the top few hits.+The query issued to the system was _Apache Solr_.+It seems clear that faceting could not yield a similar set of groups, although the goals of both techniques are similar -- to let the user explore the set of search results and either rephrase the query or narrow the focus to a subset of current documents.+Clustering is also similar to <<result-grouping.adoc#,Result Grouping>> in that it can help to look deeper into search results, beyond the top few hits.== Configuration Quick Starter-The clustering extension works as a search component. It needs to be declared and configured in `solrconfig.xml`, for example:+The clustering extension works as a search component.+It needs to be declared and configured in `solrconfig.xml`, for example:[source,xml]----@@ -48,12 +51,11 @@ The clustering extension works as a search component. It needs to be declared an</searchComponent>-----The above declares the clustering component with a single *engine* -- there may be-multiple engines declared and switched at runtime. We will return to the details of-how to configure engines later.+The above declares the clustering component with a single *engine* -- there may be multiple engines declared and switched at runtime.+We will return to the details of how to configure engines later.-The clustering component must be attached to a `SearchHandler` and explicitly enabled-via property `clustering`. It is important to attach it as the *last* component in the handler's pipeline, as shown below:+The clustering component must be attached to a `SearchHandler` and explicitly enabled via property `clustering`.+It is important to attach it as the *last* component in the handler's pipeline, as shown below:[source,xml]----@@ -70,7 +72,8 @@ via property `clustering`. It is important to attach it as the *last* component----Once attached, as in the example above, the clustering will be performed automatically-on all documents matching the search handler's query. The clustering extension will take into+on all documents matching the search handler's query.+The clustering extension will take intoaccount all text fields listed in `clustering.fields` parameter of the engine and willproduct a section of the response called `clusters` containing the discovered structure ofgroups, for example (JSON response for brevity):@@ -196,7 +199,11 @@ The output XML should include search hits and an array of automatically discover</response>-----A few clusters discovered for this query (`\*:*`), separate all search hits into various categories: DDR, iPod, Hard Drive, etc. Each cluster has a label and score that indicates the "goodness" of the cluster. The score is algorithm-specific and is meaningful only in relation to the scores of other clusters in the same set. In other words, if cluster _A_ has a higher score than cluster _B_, cluster _A_ should be of better quality (have a better label and/or more coherent document set). Each cluster has an array of identifiers of documents belonging to it. These identifiers correspond to the `uniqueKey` field declared in the schema.+A few clusters discovered for this query (`\*:*`), separate all search hits into various categories: DDR, iPod, Hard Drive, etc.+Each cluster has a label and score that indicates the "goodness" of the cluster.+The score is algorithm-specific and is meaningful only in relation to the scores of other clusters in the same set.+In other words, if cluster _A_ has a higher score than cluster _B_, cluster _A_ should be of better quality (have a better label and/or more coherent document set). Each cluster has an array of identifiers of documents belonging to it.+These identifiers correspond to the `uniqueKey` field declared in the schema.Sometimes cluster labels may not make much sense (this depends on many factors -- text in clustered fields, number of documents, algorithm paramerters). Also, some documents may be left out and not be clustered at all; these will be assigned to the synthetic _Other Topics_ group, marked with the `other-topics` property set to `true` (see the XML dump above for an example). The score of the other topics group is zero.@@ -219,11 +226,13 @@ You can include the required contrib JARs in `solrconfig.xml` as shown below (byThe following properties control `ClusteringComponent` state.`clustering`::-The component is by default disabled, even if properly declared and attached to a search handler. The `clustering` property must be set to `true` to enable it (this can be done by setting+The component is by default disabled, even if properly declared and attached to a search handler.+The `clustering` property must be set to `true` to enable it (this can be done by settingup default parameters in the search handler -- see below).`clustering.engine`::-Declares which engine to use. If not present, the first declared active engine is used.+Declares which engine to use.+If not present, the first declared active engine is used.=== Clustering Engines@@ -248,7 +257,8 @@ The declaration of clustering component in `solrconfig.xml` must include one or----This declares two separate engines (`lingo` and `stc`): these configurations have a-different clustering algorithm, and a different set of clustered document fields. The+different clustering algorithm, and a different set of clustered document fields.+Theactive engine can be selected by passing `clustering.engine=_name_` parameterat runtime (via URL) or as the default within the search handler's configuration,as shown below:@@ -278,21 +288,29 @@ Each declared engine can be configured using a number of parameters described be`clustering.fields` (_required_)::A comma (or space) separated list of text fields which should contain the text-content for clustering. At least one field must be provided. The fields are separate from search handler's `fl` parameter so that clustered fields don't have to be included in the response.+content for clustering.+At least one field must be provided.+The fields are separate from search handler's `fl` parameter so that clustered fields don't have to be included in the response.`clustering.algorithm` (_required_)::-The clustering algorithm is the actual logic (implementation) that discovers relationships among the documents and forms human-readable cluster labels. This parameter sets the name of the clustering algorithm this engine is going to use. Algorithms are supplied to Solr via Carrot^2^-defined service extension. By default, the following open-source algorithms should be available: `Lingo`, `STC`, `Bisecting K-Means`. A commercial clustering algorithm `Lingo3G` plugs into the same extension point and can be used, if it is available on classpath.+The clustering algorithm is the actual logic (implementation) that discovers relationships among the documents and forms human-readable cluster labels.+This parameter sets the name of the clustering algorithm this engine is going to use.+Algorithms are supplied to Solr via Carrot^2^-defined service extension.+By default, the following open-source algorithms should be available: `Lingo`, `STC`, `Bisecting K-Means`.+A commercial clustering algorithm `Lingo3G` plugs into the same extension point and can be used, if it is available on classpath..How to choose the Clustering Algorithm?****-The question of which algorithm to choose depends on the amount of traffic, the expected result, and the input data (each algorithm will cluster the input slightly differently). There is no one answer which algorithm is "the best": Lingo3G provides hierarchical clusters, Lingo and STC provide flat clusters. STC is faster than Lingo, but arguably produces less intuitive clusters, Lingo3G is the fastest algorithm but is not free or open source... Experiment and pick one that suits your needs.+The question of which algorithm to choose depends on the amount of traffic, the expected result, and the input data (each algorithm will cluster the input slightly differently). There is no one answer which algorithm is "the best": Lingo3G provides hierarchical clusters, Lingo and STC provide flat clusters.+STC is faster than Lingo, but arguably produces less intuitive clusters, Lingo3G is the fastest algorithm but is not free or open source... Experiment and pick one that suits your needs.For a comparison of characteristics of these algorithms see the following links:* https://carrot2.github.io/release/4.0.4/doc/algorithms/* https://carrotsearch.com/lingo3g-comparison.html-The clustering component lists all available algorithms, languages and algorithm-language compatibility at startup. You can peek at startup logs to see what's available+The clustering component lists all available algorithms, languages and algorithm-language compatibility at startup.+You can peek at startup logs to see what's availablein your Solr installation.****@@ -302,25 +320,30 @@ be truncated). By default all labels are returned.`clustering.includeSubclusters`::If `true`, sub-clusters are included in the response for algorithms that support hierarchical-clustering. `false` causes only top-level clusters to be returned.+clustering.+`false` causes only top-level clusters to be returned.`clustering.includeOtherTopics`::-If `true`, a synthetic cluster called _Other Topics_, consisting of all documents not assigned to any other cluster is formed and returned. The default value of this parameter is `true` but if+If `true`, a synthetic cluster called _Other Topics_, consisting of all documents not assigned to any other cluster is formed and returned.+The default value of this parameter is `true` but ifthere is no need for this synthetic cluster, it can be set to `false`.`clustering.resources`::-Location of algorithm-specific resources and configuration files (stop words, other lexical resources, default settings). This property is `null` by default and all resources are read-from their respective algorithm default resource pool (JARs). If this property is not empty,-it resolves relative to Solr core's configuration directory. This parameter can be applied during Solr startup _only_, it can't be overriden per-request.+Location of algorithm-specific resources and configuration files (stop words, other lexical resources, default settings).+This property is `null` by default and all resources are read from their respective algorithm default resource pool (JARs).+If this property is not empty, it resolves relative to Solr core's configuration directory.+This parameter can be applied during Solr startup _only_, it can't be overriden per-request.[.text-center]🙪-There are more properties applying to engine configuration. We describe these in functional sections that follow.+There are more properties applying to engine configuration.+We describe these in functional sections that follow.=== Full Field and Query-Context (Snippet) Clustering-The clustering algorithm can consume full content of fields or just the left and right context around query-matching regions (so-called _snippets_). Contrary to the intuition, using query contexts can increase the quality of clustering even if it feeds less data to the algorithm. This is typically caused by the fact that snippets are more focused around the phrases and terms surrounding the query and the algorithm has a better signal-to-noise ratio of data to work with.+The clustering algorithm can consume full content of fields or just the left and right context around query-matching regions (so-called _snippets_). Contrary to the intuition, using query contexts can increase the quality of clustering even if it feeds less data to the algorithm.+This is typically caused by the fact that snippets are more focused around the phrases and terms surrounding the query and the algorithm has a better signal-to-noise ratio of data to work with.We recommend using query contexts when fields contain a lot of content (this would affect clustering performance).@@ -337,17 +360,22 @@ The maximum number of different, non-contiguous snippets from a single field.=== Default Clustering Language-The default implementations of clustering algorithms in Carrot^2^ (shipped with Solr)-have built-in support (stemming, stop words) for preprocessing a number of languages. It is important to provide the clustering algorithm with a hint of what language should be used for clustering. This can be done in two ways -- by passing the name of the default language or by providing the language as a field with each document. The following two engine configuration parameters control this:+The default implementations of clustering algorithms in Carrot^2^ (shipped with Solr) have built-in support (stemming, stop words) for preprocessing a number of languages.+It is important to provide the clustering algorithm with a hint of what language should be used for clustering.+This can be done in two ways -- by passing the name of the default language or by providing the language as a field with each document.+The following two engine configuration parameters control this:`clustering.language`::-Name of the default language to use for clustering. The default value of this field is `English`. The provided language must be available and the clustering algorithm must support it.+Name of the default language to use for clustering.+The default value of this field is `English`.+The provided language must be available and the clustering algorithm must support it.`clustering.languageField`::-Name of the document field that stores the document's language. If the field does not exist-for a document or the value is blank, the default language is used.+Name of the document field that stores the document's language.+If the field does not exist for a document or the value is blank, the default language is used.-The list of supported languages can change dynamically (languages are loaded via external service provider extension) and may depend on the selected algorithm (algorithms can support a subset of languages for which resources are available). The clustering component will log all supported algorithm-language pairs at Solr startup, so you can inspect what's supported on your particular Solr instance. For example:+The list of supported languages can change dynamically (languages are loaded via external service provider extension) and may depend on the selected algorithm (algorithms can support a subset of languages for which resources are available). The clustering component will log all supported algorithm-language pairs at Solr startup, so you can inspect what's supported on your particular Solr instance.+For example:[source,text]----@@ -358,17 +386,24 @@ The list of supported languages can change dynamically (languages are loaded via=== Handling Multilingual Content-It is often the case that the index (and query result) contains documents in _multiple_ languages. Clustering such search results is problematic. Ideally, the engine should translate (or understand) the content of documents and then group relevant information together, regardless of the language it is written in.+It is often the case that the index (and query result) contains documents in _multiple_ languages.+Clustering such search results is problematic.+Ideally, the engine should translate (or understand) the content of documents and then group relevant information together, regardless of the language it is written in.-In reality, clustering algorithms are typically much simpler -- they infer similarity between documents from statistical properties of terms and phrases that occur in those documents. So texts written in different languages will not cluster too well.+In reality, clustering algorithms are typically much simpler -- they infer similarity between documents from statistical properties of terms and phrases that occur in those documents.+So texts written in different languages will not cluster too well.-To deal witht his situation the default clustering component implementation in Solr will try to first group all documents by their language and then apply clustering to each sub-group in that language. It is recommended to store the language of each document as its separate field and point at it using the `clustering.languageField` configuration property described above.+To deal witht his situation the default clustering component implementation in Solr will try to first group all documents by their language and then apply clustering to each sub-group in that language.+It is recommended to store the language of each document as its separate field and point at it using the `clustering.languageField` configuration property described above.== Tweaking Algorithm Settings-The clustering algorithms that come with Solr use their default parameter values and language resources. We highly recommend tuning both for production uses. Improving the default language resources to include words and phrases common to a particular document domain will improve clustering quality significantly.+The clustering algorithms that come with Solr use their default parameter values and language resources.+We highly recommend tuning both for production uses.+Improving the default language resources to include words and phrases common to a particular document domain will improve clustering quality significantly.-Carrot^2^ algorithms have an extensive set of parameters and language resource tuning options. Please refer to https://carrot2.github.io/release/latest/[up-to-date project documentation]. In particular, the language resources section and each algorithm's attributes section.+Carrot^2^ algorithms have an extensive set of parameters and language resource tuning options.+Please refer to https://carrot2.github.io/release/latest/[up-to-date project documentation]. In particular, the language resources section and each algorithm's attributes section.=== Changing Clustering Algorithm Parameters@@ -417,7 +452,9 @@ The following rules apply.* The type of the parameter must be consistent with the type listed in Carrot^2^ specification.-* If the parameter is added to the engine's configuration in `solrconfig.xml`, the core must be reloaded for the changes to be picked up. Alternatively, pass the parameter via the request URL to change things dynamically on a per-request basis. For example, if you have the `techproducts` example running, this will cut the clusters to only those containing at least three documents:+* If the parameter is added to the engine's configuration in `solrconfig.xml`, the core must be reloaded for the changes to be picked up.+Alternatively, pass the parameter via the request URL to change things dynamically on a per-request basis.+For example, if you have the `techproducts` example running, this will cut the clusters to only those containing at least three documents:`http://localhost:8983/solr/techproducts/clustering?q=\*:*&rows=100&wt=json&preprocessing.documentAssigner.minClusterSize=3`* For complex types, the parameter key with the name of the instantiated type must precede any of its own parameters.@@ -427,7 +464,10 @@ The following rules apply.Clustering algorithms rely on language and domain-specific language resources toimprove the quality of clusters (by discarding domain-specific noise and boilerplate language).-By default, language resources are read from the engine-declared algorithm default JAR. You can pass a custom location for these resources by specifying the `clustering.resources` parameter. The value of this parameter resolves to a location relative to Solr core's configuration directory. For example, the following definition:+By default, language resources are read from the engine-declared algorithm default JAR.+You can pass a custom location for these resources by specifying the `clustering.resources` parameter.+The value of this parameter resolves to a location relative to Solr core's configuration directory.+For example, the following definition:[source,xml]----@@ -459,7 +499,8 @@ Clustering of search results comes with some performance considerations:* Additional computational cost of the clustering itself.* In distributed mode the content of document fields for clustering is collected from shards and adds some additional network overhead.-For simple queries, the clustering time will usually dominate everything else. If document fields are very long, the retrieval of stored content can become a bottleneck.+For simple queries, the clustering time will usually dominate everything else.+If document fields are very long, the retrieval of stored content can become a bottleneck.The performance impact of clustering can be lowered in several ways.diff --git a/solr/solr-ref-guide/src/result-grouping.adoc b/solr/solr-ref-guide/src/result-grouping.adocindex fb811486443..b7924e1b180 100644--- a/solr/solr-ref-guide/src/result-grouping.adoc+++ b/solr/solr-ref-guide/src/result-grouping.adoc@@ -18,17 +18,23 @@Result Grouping groups documents with a common field value into groups and returns the top documents for each group.-For example, if you searched for "DVD" on an electronic retailer's e-commerce site, you might be returned three categories such as "TV and Video", "Movies", and "Computers" with three results per category. In this case, the query term "DVD" appeared in all three categories, so Solr groups them together in order to increase relevancy for the user.+For example, if you searched for "DVD" on an electronic retailer's e-commerce site, you might be returned three categories such as "TV and Video", "Movies", and "Computers" with three results per category.+In this case, the query term "DVD" appeared in all three categories, so Solr groups them together in order to increase relevancy for the user..Prefer Collapse & Expand instead[NOTE]====-Solr's <<collapse-and-expand-results.adoc#,Collapse and Expand>> feature is newer and mostly overlaps with Result Grouping. There are features unique to both, and they have different performance characteristics. That said, in most cases Collapse and Expand is preferable to Result Grouping.+Solr's <<collapse-and-expand-results.adoc#,Collapse and Expand>> feature is newer and mostly overlaps with Result Grouping.+There are features unique to both, and they have different performance characteristics.+That said, in most cases Collapse and Expand is preferable to Result Grouping.====-Result Grouping is separate from <<faceting.adoc#,Faceting>>. Though it is conceptually similar, faceting returns all relevant results and allows the user to refine the results based on the facet category. For example, if you search for "shoes" on a footwear retailer's e-commerce site, Solr would return all results for that query term, along with selectable facets such as "size," "color," "brand," and so on.+Result Grouping is separate from <<faceting.adoc#,Faceting>>. Though it is conceptually similar, faceting returns all relevant results and allows the user to refine the results based on the facet category.+For example, if you search for "shoes" on a footwear retailer's e-commerce site, Solr would return all results for that query term, along with selectable facets such as "size," "color," "brand," and so on.-You can however combine grouping with faceting. Grouped faceting supports `facet.field` and `facet.range` but currently doesn't support date and pivot faceting. The facet counts are computed based on the first `group.field` parameter, and other `group.field` parameters are ignored.+You can however combine grouping with faceting.+Grouped faceting supports `facet.field` and `facet.range` but currently doesn't support date and pivot faceting.+The facet counts are computed based on the first `group.field` parameter, and other `group.field` parameters are ignored.Grouped faceting differs from non grouped facets `(sum of all facets) == (total of products with that property)` as shown in the following example:@@ -54,13 +60,16 @@ If you ask Solr to group these documents by "product_range", then the total amou== Grouping Parameters-Result Grouping takes the following request parameters. Any number of these request parameters can be included in a single request:+Result Grouping takes the following request parameters.+Any number of these request parameters can be included in a single request:`group`::If `true`, query results will be grouped.`group.field`::-The name of the field by which to group results. The field must be single-valued, and either be indexed or a field type that has a value source and works in a function query, such as `ExternalFileField`. It must also be a string-based field, such as `StrField` or `TextField`+The name of the field by which to group results.+The field must be single-valued, and either be indexed or a field type that has a value source and works in a function query, such as `ExternalFileField`.+It must also be a string-based field, such as `StrField` or `TextField``group.func`::Group based on the unique values of a function query.@@ -71,48 +80,63 @@ NOTE: This option does not work with <<Distributed Result Grouping Caveats,distrReturn a single group of documents that match the given query.`rows`::-The number of groups to return. The default value is `10`.+The number of groups to return.+The default value is `10`.`start`::Specifies an initial offset for the list of groups.`group.limit`::-Specifies the number of results to return for each group. The default value is `1`.+Specifies the number of results to return for each group.+The default value is `1`.`group.offset`::Specifies an initial offset for the document list of each group.`sort`::-Specifies how Solr sorts the groups relative to each other. For example, `sort=popularity desc` will cause the groups to be sorted according to the highest popularity document in each group. The default value is `score desc`.+Specifies how Solr sorts the groups relative to each other.+For example, `sort=popularity desc` will cause the groups to be sorted according to the highest popularity document in each group.+The default value is `score desc`.`group.sort`::-Specifies how Solr sorts documents within each group. The default behavior if `group.sort` is not specified is to use the same effective value as the `sort` parameter.+Specifies how Solr sorts documents within each group.+The default behavior if `group.sort` is not specified is to use the same effective value as the `sort` parameter.`group.format`::-If this parameter is set to `simple`, the grouped documents are presented in a single flat list, and the `start` and `rows` parameters affect the numbers of documents instead of groups. An alternate value for this parameter is `grouped`.+If this parameter is set to `simple`, the grouped documents are presented in a single flat list, and the `start` and `rows` parameters affect the numbers of documents instead of groups.+An alternate value for this parameter is `grouped`.`group.main`::If `true`, the result of the first field grouping command is used as the main result list in the response, using `group.format=simple`.`group.ngroups`::-If `true`, Solr includes the number of groups that have matched the query in the results. The default value is false.+If `true`, Solr includes the number of groups that have matched the query in the results.+The default value is false.+See below for <<Distributed Result Grouping Caveats>> when using sharded indexes.`group.truncate`::-If `true`, facet counts are based on the most relevant document of each group matching the query. The default value is `false`.+If `true`, facet counts are based on the most relevant document of each group matching the query.+The default value is `false`.`group.facet`::-Determines whether to compute grouped facets for the field facets specified in facet.field parameters. Grouped facets are computed based on the first specified group. As with normal field faceting, fields shouldn't be tokenized (otherwise counts are computed for each token). Grouped faceting supports single and multivalued fields. Default is `false`.+Determines whether to compute grouped facets for the field facets specified in facet.field parameters.+Grouped facets are computed based on the first specified group.+As with normal field faceting, fields shouldn't be tokenized (otherwise counts are computed for each token). Grouped faceting supports single and multivalued fields.+Default is `false`.+WARNING: There can be a heavy performance cost to this option.+See below for <<Distributed Result Grouping Caveats>> when using sharded indexes.`group.cache.percent`::-Setting this parameter to a number greater than 0 enables caching for result grouping. Result Grouping executes two searches; this option caches the second search. The default value is `0`. The maximum value is `100`.+Setting this parameter to a number greater than 0 enables caching for result grouping.+Result Grouping executes two searches; this option caches the second search.+The default value is `0`.+The maximum value is `100`.+-Testing has shown that group caching only improves search time with Boolean, wildcard, and fuzzy queries. For simple queries like term or "match all" queries, group caching degrades performance.+Testing has shown that group caching only improves search time with Boolean, wildcard, and fuzzy queries.+For simple queries like term or "match all" queries, group caching degrades performance.Any number of group commands (e.g., `group.field`, `group.func`, `group.query`, etc.) may be specified in a single request.@@ -172,9 +196,13 @@ http://localhost:8983/solr/techproducts/select?fl=id,name&q=solr+memory&group=tr}]}}}-----The response indicates that there are six total matches for our query. For each of the five unique values of `group.field`, Solr returns a `docList` for that `groupValue` such that the `numFound` indicates the total number of documents in that group, and the top documents are returned according to the implicit default `group.limit=1` and `group.sort=score desc` parameters. The resulting groups are then sorted by the score of the top document within each group based on the implicit `sort=score desc`, and the number of groups returned is limited to the implicit `rows=10`.+The response indicates that there are six total matches for our query.+For each of the five unique values of `group.field`, Solr returns a `docList` for that `groupValue` such that the `numFound` indicates the total number of documents in that group, and the top documents are returned according to the implicit default `group.limit=1` and `group.sort=score desc` parameters.+The resulting groups are then sorted by the score of the top document within each group based on the implicit `sort=score desc`, and the number of groups returned is limited to the implicit `rows=10`.-We can run the same query with the request parameter `group.main=true`. This will format the results as a single flat document list. This flat format does not include as much information as the normal result grouping query results – notably the `numFound` in each group – but it may be easier for existing Solr clients to parse.+We can run the same query with the request parameter `group.main=true`.+This will format the results as a single flat document list.+This flat format does not include as much information as the normal result grouping query results – notably the `numFound` in each group – but it may be easier for existing Solr clients to parse.[source,text]http://localhost:8983/solr/techproducts/select?fl=id,name,manufacturer&q=solr+memory&group=true&group.field=manu_exact&group.main=true@@ -260,11 +288,13 @@ http://localhost:8983/solr/techproducts/select?indent=true&fl=name,price&q=memor}-----In this case, Solr found five matches for "memory," but only returns four results grouped by price. This is because one result for "memory" did not have a price assigned to it.+In this case, Solr found five matches for "memory," but only returns four results grouped by price.+This is because one result for "memory" did not have a price assigned to it.== Distributed Result Grouping Caveats-Grouping is supported for <<solrcloud.adoc#,distributed searches>>, with some caveats:+Grouping is supported for <<cluster-types.adoc#solrcloud-mode,distributed searches>>, with some caveats:* Currently `group.func` is is not supported in any distributed searches-* `group.ngroups` and `group.facet` require that all documents in each group must be co-located on the same shard in order for accurate counts to be returned. <<shards-and-indexing-data-in-solrcloud.adoc#,Document routing via composite keys>> can be a useful solution in many situations.+* `group.ngroups` and `group.facet` require that all documents in each group must be co-located on the same shard in order for accurate counts to be returned.+<<solrcloud-shards-indexing.adoc#,Document routing via composite keys>> can be a useful solution in many situations.diff --git a/solr/solr-ref-guide/src/using-solr-from-ruby.adoc b/solr/solr-ref-guide/src/ruby.adocsimilarity index 93%rename from solr/solr-ref-guide/src/using-solr-from-ruby.adocrename to solr/solr-ref-guide/src/ruby.adocindex b34e56fc7d3..84f0eeef2dd 100644--- a/solr/solr-ref-guide/src/using-solr-from-ruby.adoc+++ b/solr/solr-ref-guide/src/ruby.adoc@@ -1,4 +1,4 @@-= Using Solr From Ruby+= Ruby// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -111,4 +111,5 @@ puts 'number of matches = ' + rsp['response']['numFound'].to_srsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'] }-----For simple interactions with Solr, this may be all you need! If you are building complex interactions with Solr, then consider the libraries mentioned in the https://cwiki.apache.org/confluence/display/solr/Ruby+Response+Format#RubyResponseFormat-Libraries[Solr Wiki].+For simple interactions with Solr, this may be all you need!+If you are building complex interactions with Solr, then consider the libraries mentioned in the https://cwiki.apache.org/confluence/display/solr/Ruby+Response+Format#RubyResponseFormat-Libraries[Solr Wiki].diff --git a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adocindex 3c82cfc79ed..6b5a4f26d02 100644--- a/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc+++ b/solr/solr-ref-guide/src/rule-based-authorization-plugin.adoc@@ -16,76 +16,141 @@// specific language governing permissions and limitations// under the License.+Solr provides authorization plugins that offer fine-grained user access control to critical Solr APIs and features.+Solr's authentication plugins control whether users can access Solr in a binary fashion.A user is either authenticated, or they aren't.-For more fine-grained access control, Solr's Rule-Based Authorization Plugins (hereafter, "RBAP") can be used.+For more fine-grained access control, Solr's Rule-Based Authorization Plugins (RBAP) can be used.[CAUTION]====-Solr's Admin UI interacts with Solr using its regular APIs. When rule-based authorization is in use, logged-in users not authorized to access the full range of these APIs may see some sections of the UI that appear blank or "broken". For best results, the Admin UI should only be accessed by users with full API access.+Solr's Admin UI interacts with Solr using its regular APIs.+When rule-based authorization is in use, logged-in users not authorized to access the full range of these APIs may see some sections of the UI that appear blank or "broken".++For best results, the Admin UI should only be accessed by users with full API access.====== Rule-Based Auth Concepts-"Users", "roles" and "permissions" play a central role in configuring authorization correctly.+"Users", "roles", and "permissions" play a central role in configuring authorization correctly.-In Rule-Based Authorization, administrators define a series of roles based on the permissions they want those roles to confer. Users are then assigned one or more roles.+In Rule-Based Authorization, administrators define a series of roles based on the permissions they want those roles to confer.+Users are then assigned one or more roles.=== Users-The users that RBAP sees come from whatever authentication plugin has been configured. RBAP is compatible with all of the authentication plugins that Solr ships with out of the box. It is also compatible with any custom authentication plugins users might write, provided that the plugin sets a user principal on the HttpServletRequest it receives. The user value seen by RBAP in each case depends on the authentication plugin being used: the Kerberos principal if the <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> is being used, the "sub" JWT claim if the <<jwt-authentication-plugin.adoc#,JWT Authentication Plugin>> is being used, etc.+The users that RBAP sees come from whatever authentication plugin has been configured.+RBAP is compatible with all of the authentication plugins that Solr ships with out of the box.+It is also compatible with any custom authentication plugins users might write, provided that the plugin sets a user principal on the HttpServletRequest it receives.++The user value seen by RBAP in each case depends on the authentication plugin being used: the Kerberos principal if the <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>> is being used, the "sub" JWT claim if the <<jwt-authentication-plugin.adoc#,JWT Authentication Plugin>> is being used, etc.=== Roles-Roles help bridge the gap between users and permissions. The roles can be used with any of the authentication plugins or with a custom authentication plugin if you have created one. You will only need to ensure that logged-in users are mapped to the roles defined by the plugin. There are two implementations of the plugin, which only differs in how the user's roles are obtained:+Roles bridge the gap between users and permissions.+The roles can be used with any of the authentication plugins or with a custom authentication plugin if you have created one.+You will only need to ensure that logged-in users are mapped to the roles defined by the plugin.++There are two implementations of the plugin, which only differ in how the user's roles are obtained:* `RuleBasedAuthorizationPlugin`: The role-to-user mappings must be defined explicitly in `security.json` for every possible authenticated user.-* `ExternalRoleRuleBasedAuthorizationPlugin`: The role-to-user mappings are managed externally. This plugin expects the AuthenticationPlugin to provide a Principal that has the roles information as well, implementing the `VerifiedUserRoles` interface.++* `ExternalRoleRuleBasedAuthorizationPlugin`: The role-to-user mappings are managed externally.+This plugin expects the AuthenticationPlugin to provide a Principal that has the roles information as well, implementing the `VerifiedUserRoles` interface.=== Permissions-Permissions control which roles (and consequently, which users) have access to particular chunks of Solr's API. Each permission has two main components: a description of the APIs this permission applies to, and a list of the roles that should be allowed to access to this set of APIs.+Permissions control which roles (and consequently which users) have access to which Solr APIs.++Each permission has two main components: a description of the APIs the permission applies to, and a list of the roles that should be allowed to access to this set of APIs.Administrators can use permissions from a list of predefined options or define their own custom permissions, are are free to mix and match both.== Configuring the Rule-Based Authorization Plugins-Like all of Solr's security plugins, configuration for RBAP lives in a file or ZooKeeper node with the name `security.json`. See <<authentication-and-authorization-plugins.adoc#enable-plugins-with-security-json,here>> for more information on how to setup `security.json` in your cluster.+Like all of Solr's security plugins, configuration for RBAP lives in a file or ZooKeeper node with the name `security.json`.+See <<authentication-and-authorization-plugins.adoc#configuring-security-json,Configuring security.json>> for more information on how to setup `security.json` in your cluster.-Solr offers an <<Authorization API>> for making changes to RBAP configuration. Authorized administrators should use this to make changes under most circumstances. Users may also make edits to `security.json` directly if it is stored in ZooKeeper, but this is an expert-level feature and is discouraged in most circumstances. The API simplifies some aspects of configuration, and provides error feedback that isn't provided when editing ZooKeeper directly.+Solr offers an <<Authorization API>> for making changes to RBAP configuration.+Authorized administrators should use this to make changes under most circumstances.+Users may also make edits to `security.json` directly if it is stored in ZooKeeper, but this is an expert-level feature and is discouraged in most circumstances.+The API simplifies some aspects of configuration, and provides error feedback that isn't provided when editing ZooKeeper directly.=== Configuration Syntax-RBAP configuration consists of a small number of required configuration properties. Each of these lives under the `authorization` top level property in `security.json`+RBAP configuration consists of a small number of required configuration properties.+Each of these lives under the `authorization` top level property in `security.json`++`class`::+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++The authorization plugin to use.+There are two options: `solr.RuleBasedAuthorizationPlugin` or `solr.ExternalRoleRuleBasedAuthorizationPlugin`.-class:: The authorization plugin to use. For RBAP, this value will always be `solr.RuleBasedAuthorizationPlugin`-permissions:: A JSON array of permission rules used to restrict access to sections of Solr's API. For example:+`permissions`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++A JSON array of permission rules used to restrict access to sections of Solr's API.+For example:+[source,json]-----"permissions": [+{+ "permissions": [{ "name": "read", "collection": "techproducts", "role": ["admin", "dev"] },{ "name": "all", "role": "admin"}-]+ ]+}----+The syntax for individual permissions is more involved and is treated in greater detail <<Permissions,below>>.-User's roles may either come from the request itself, then you will use the `ExternalRoleRuleBasedAuthorizationPlugin` variant of RBAC. If you need to hardcode user-role mappings, then you need to use the `RuleBasedAuthorizationPlugin` and define the user-role mappings in `security.json` like this:+User roles may come from the request itself when you use the `ExternalRoleRuleBasedAuthorizationPlugin` for the `class`.+In this case, skip defining `permissions`.++If you need to hardcode user-role mappings, then define the `RuleBasedAuthorizationPlugin` for the `class` and define the user-role mappings in `security.json` like this:-user-role:: A mapping of individual users to the roles they belong to. The value of this property is a JSON map, where each property name is a user, and each property value is either the name of a single role or a JSON array of multiple roles that the specified user belongs to. For example:+`user-role`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++A mapping of individual users to their assigned roles.+The value of this parameter is a JSON map, where each property name is a user, and each property value is either the name of a single role or a JSON array of multiple roles that the specified user belongs to.+++For example:++[source,json]-----"user-role": {+{+ "user-role": {"user1": "role1","user2": ["role1", "role2"]+ }}-----useShortName:: Determines if user-role mappings will resolve using the full principal or a shortened name provided by the authentication plugin. For example, the `KerberosAuthPlugin` may provide a full principal as `[email protected]`, while the corresponding short name would be `user`.++`useShortName`::+-For some plugins the principal name and short name may be the same.+[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++Determines if user-role mappings will resolve using the full principal or a shortened name provided by the authentication plugin.+For example, the `KerberosAuthPlugin` may provide a full principal as `[email protected]`, while the corresponding short name would be `user`.+-This setting is optional and defaults to `false` if unset.+For some plugins the principal name and short name may be the same.=== Example for RuleBasedAuthorizationPlugin and BasicAuth@@ -117,12 +182,20 @@ This example `security.json` shows how the <<basic-authentication-plugin.adoc#,B}-----<1> Solr is using the Basic Authentication plugin for authentication. This configuration establishes two users: `admin-user` and `dev-user`.-<2> The `authorization` property begins the authorization configuration. Solr will use RBAP for authorization.-<3> Two roles are defined: `admin` and `dev`. Each user belongs to one role: `admin-user` is an `admin`, and `dev-user` is a `dev`.-<4> Three permissions restrict access to Solr. The first permission (a "custom" permission) indicates that only the `dev` role can read from a special collection with the name `dev-private`. The last two permissions ("predefined" permissions) indicate that only the `admin` role is permitted to use Solr's security APIs. See below for more information on permission syntax.--Altogether, this example carves out two restricted areas. Only `admin-user` can access Solr's Authentication and Authorization APIs, and only `dev-user` can access their `dev-private` collection. All other APIs are left open, and can be accessed by both users.+<1> Solr is using the Basic Authentication plugin for authentication.+This configuration establishes two users: `admin-user` and `dev-user`.+<2> The `authorization` property begins the authorization configuration.+Solr will use RBAP for authorization.+<3> Two roles are defined: `admin` and `dev`.+Each user belongs to one role: `admin-user` is an `admin`, and `dev-user` is a `dev`.+<4> Three permissions restrict access to Solr.+The first permission (a "custom" permission) indicates that only the `dev` role can read from a special collection with the name `dev-private`.+The last two permissions ("predefined" permissions) indicate that only the `admin` role is permitted to use Solr's security APIs.+See below for more information on permission syntax.++Altogether, this example carves out two restricted areas.+Only `admin-user` can access Solr's Authentication and Authorization APIs, and only `dev-user` can access their `dev-private` collection.+All other APIs are left open, and can be accessed by both users.=== Example for External Role RuleBasedAuthorizationPlugin with JWT auth@@ -155,102 +228,211 @@ Only requests from users having a JWT token with role "admin" will be granted th== Permissions-Solr's Rule-Based Authorization plugin supports a flexible and powerful permission syntax. RBAP supports two types of permissions, each with a slightly different syntax.+Solr's Rule-Based Authorization plugin supports a flexible and powerful permission syntax.+RBAP supports two types of permissions, each with a slightly different syntax.=== Custom PermissionsAdministrators can write their own custom permissions that can match requests based on the collection, request handler, HTTP method, particular request parameters, etc.-Each custom permission is a JSON object under the `permissions` property, with one or more of the properties below:+Each custom permission is a JSON object under the `permissions` parameter, with one or more of the properties below:++name::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++An identifier for the permission.+++For custom permissions, this is used only as a clue to administrators about what this permission does.+++Care must be taken when setting this parameter to avoid colliding with one of Solr's <<Permissions,predefined permissions>>, whose names are reserved.+If this name matches a predefined permission, Solr ignores any other properties set and uses the semantics of the predefined permission instead.-name:: An optional identifier for the permission. For custom permissions, this is used only as a clue to administrators about what this permission does. Even so, care must be taken when setting this property to avoid colliding with one of Solr's predefined permissions, whose names are semantically meaningful. If this name matches a predefined permission, Solr ignores any other properties set and uses the semantics of the predefined permission instead.-collection:: An optional property identifying which collection(s) this permission applies to. The value can either be a single collection name, or a JSON array containing multiple collections. The wildcard `\*` can be used to indicate that this rule applies to all collections. Similarly the special value `null` can be used to indicate that this permission governs Solr's collection-agnostic ("admin") APIs. If not specified, this property defaults to `"*"`.+collection::+++[%autowidth,frame=none]+|===+|Optional |Default: `*` (all)+|===+++Defines the collection(s) the permission applies to.+The value can either be a single collection name, or a JSON array containing multiple collections.+++The wildcard `*` is used to indicate that this rule applies to all collections.+Similarly the special value `null` can be used to indicate that this permission governs Solr's collection-agnostic ("admin") APIs.+[NOTE]====-The collection property can only be used to match _collections_. It currently cannot be used to match aliases. Aliases are resolved before Solr's security plugins are invoked; a `collection` property given an alias will never match because RBAP will be comparing an alias name to already-resolved collection names. Instead, set a `collection` property that contains all collections in the alias concerned (or the `*` wildcard).+The `collection` parameter can only contain values that are real _collection_ names.+It currently cannot be used to match aliases.+++Aliases are resolved before Solr's security plugins are invoked.+A `collection` parameter given an alias as a value will never match because RBAP will be comparing an alias name to already-resolved collection names.+++Instead, set a `collection` parameter that contains all collections in the alias concerned (or the `*` wildcard).====-path:: An optional property identifying which paths this permission applies to. The value can either be a single path string, or a JSON array containing multiple strings. For APIs accessing collections, path values should start after the collection name, and often just look like the request handler (e.g., `"/select"`). For collection-agnostic ("admin") APIs, path values should start at the `"/admin` path segment. The wildcard `\*` can be used to indicate that this permission applies to all paths. If not specified, this property defaults to `null`.-method:: An optional property identifying which HTTP methods this permission applies to. Options include `HEAD`, `POST`, `PUT`, `GET`, `DELETE`, and the wildcard `\*`. Multiple values can also be specified using a JSON array. If not specified, this property defaults to `*`.-params:: An optional property identifying which query parameters this permission applies to. The value is a JSON object containing the names and values of request parameters that must be matched for this permission to apply.++path::+++[%autowidth,frame=none]+|===+|Optional |Default: _null_+|===+++Defines the paths the permission applies to.+The value can either be a single path string, or a JSON array containing multiple strings.+++For APIs that access collections, path values should start after the collection name, and often just look like the request handler (e.g., `"/select"`).+++For collection-agnostic (aka, "admin") APIs, path values should start at the `"/admin` path segment.+The wildcard `\*` can be used to indicate that this permission applies to all paths.++method::+++[%autowidth,frame=none]+|===+|Optional |Default: `*`+|===+-For example, this property could be used to limit the actions a role is allowed to perform with the Collections API. If the role should only be allowed to perform the LIST or CLUSTERSTATUS requests, you would define this as follows:+Defines the HTTP methods this permission applies to.+Options include `HEAD`, `POST`, `PUT`, `GET`, `DELETE`, and the wildcard `\*`.+Multiple values can also be specified using a JSON array.++params::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Defines the query parameters the permission applies to.+The value is a JSON object containing the names and values of request parameters that must be matched for this permission to apply.+++For example, this parameter could be used to limit the actions a role is allowed to perform with the Collections API.+If the role should only be allowed to perform the LIST or CLUSTERSTATUS requests, you would define this as follows:+[source,json]-----"params": {+{"params": {"action": ["LIST", "CLUSTERSTATUS"]-}+ }+ }----+-The request parameter value can be a simple string or a regular expression. Use the prefix `REGEX:` to use a regular expression match instead of simpler string matching+The request parameter value can be a simple string or a regular expression.+Use the prefix `REGEX:` to use a regular expression match instead of simpler string matching.+If the commands LIST and CLUSTERSTATUS are case insensitive, the example above can be written as follows:+[source,json]-----"params": {+{"params": {"action": ["REGEX:(?i)LIST", "REGEX:(?i)CLUSTERSTATUS"]+ }}-----+-If not specified, the permission is independent of any parameters.-role:: A required property identifying which role (or roles) are allowed access to the APIs controlled by this permission. Multiple values can be specified using a JSON array. The wildcard `*` can be used to indicate that all roles can access the described functionality.+role::+++[%autowidth,frame=none]+|===+s|Required |Default: none+|===+++Defines which role (or roles) are allowed access to the APIs controlled by this permission.+Multiple values can be specified using a JSON array.+The wildcard `*` can be used to indicate that all roles can access the described functionality.=== Predefined Permissions-Custom permissions give administrators flexibility in configuring fine-grained access control. But in an effort to make configuration as simple as possible, RBAP also offers a handful of predefined permissions, which cover many common use-cases.+Custom permissions give administrators flexibility in configuring fine-grained access control.+But in an effort to make configuration as simple as possible, RBAP also offers a handful of predefined permissions, which cover many common use-cases.-Administrators invoke a predefined permission by choosing a `name` property that matches one of Solr's predefined permission options (listed below). Solr has its own definition for each of these permissions, and uses this information when checking whether a predefined permission matches an incoming request. This trades flexibility for simplicity: predefined permissions do not support the `path`, `params`, or `method` properties which custom permissions allow.+Administrators invoke a predefined permission by choosing a `name` that matches one of Solr's predefined permission options (listed below).+Solr has its own definition for each of these permissions, and uses this information when checking whether a predefined permission matches an incoming request.+This trades flexibility for simplicity: predefined permissions do not support the `path`, `params`, or `method` properties which custom permissions allow.The predefined permission names (and their effects) are:-* *security-edit:* this permission is allowed to edit the security configuration, meaning any update action that modifies `security.json` through the APIs will be allowed.+* *security-edit*: this permission is allowed to edit the security configuration, meaning any update action that modifies `security.json` through the APIs will be allowed.* *security-read*: this permission is allowed to read the security configuration, meaning any action that reads `security.json` settings through the APIs will be allowed.-* *schema-edit*: this permission is allowed to edit a collection's schema using the <<schema-api.adoc#,Schema API>>. Note that this allows schema edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created.-* *schema-read*: this permission is allowed to read a collection's schema using the <<schema-api.adoc#,Schema API>>. Note that this allows schema read permissions for _all_ collections. If read permissions should only be applied to specific collections, a custom permission would need to be created.-* *config-edit*: this permission is allowed to edit a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`. Note that this allows configuration edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created.-* *config-read*: this permission is allowed to read a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`. Note that this allows configuration read permissions for _all_ collections. If read permissions should only be applied to specific collections, a custom permission would need to be created.-* *metrics-read*: this permission allows access to Solr's <<metrics-reporting.adoc#metrics-api,Metrics API>>+* *schema-edit*: this permission is allowed to edit a collection's schema using the <<schema-api.adoc#,Schema API>>.+Note that this allows schema edit permissions for _all_ collections.+If edit permissions should only be applied to specific collections, a custom permission would need to be created.+* *schema-read*: this permission is allowed to read a collection's schema using the <<schema-api.adoc#,Schema API>>.+Note that this allows schema read permissions for _all_ collections.+If read permissions should only be applied to specific collections, a custom permission would need to be created.+* *config-edit*: this permission is allowed to edit a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`.+Note that this allows configuration edit permissions for _all_ collections.+If edit permissions should only be applied to specific collections, a custom permission would need to be created.+* *config-read*: this permission is allowed to read a collection's configuration using the <<config-api.adoc#,Config API>>, the <<request-parameters-api.adoc#,Request Parameters API>>, and other APIs which modify `configoverlay.json`.+Note that this allows configuration read permissions for _all_ collections.+If read permissions should only be applied to specific collections, a custom permission would need to be created.+* *metrics-read*: this permission allows access to Solr's <<metrics-reporting.adoc#metrics-api,Metrics API>>.* *core-admin-edit*: Core admin commands that can mutate the system state.* *core-admin-read*: Read operations on the core admin API-* *collection-admin-edit*: this permission is allowed to edit a collection's configuration using the <<collections-api.adoc#,Collections API>>. Note that this allows configuration edit permissions for _all_ collections. If edit permissions should only be applied to specific collections, a custom permission would need to be created. Specifically, the following actions of the Collections API would be allowed:-** CREATE-** RELOAD-** SPLITSHARD-** CREATESHARD-** DELETESHARD-** CREATEALIAS-** DELETEALIAS-** DELETE-** DELETEREPLICA-** ADDREPLICA-** CLUSTERPROP-** MIGRATE-** ADDROLE-** REMOVEROLE-** ADDREPLICAPROP-** DELETEREPLICAPROP-** BALANCESHARDUNIQUE-** REBALANCELEADERS-* *collection-admin-read*: this permission is allowed to read a collection's configuration using the <<collections-api.adoc#,Collections API>>. Note that this allows configuration read permissions for _all_ collections. If read permissions should only be applied to specific collections, a custom permission would need to be created. Specifically, the following actions of the Collections API would be allowed:-** LIST-** OVERSEERSTATUS-** CLUSTERSTATUS-** REQUESTSTATUS-* *update*: this permission is allowed to perform any update action on any collection. This includes sending documents for indexing (using an <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#update-request-handlers,update request handler>>). This applies to all collections by default (`collection:"*"`).-* *read*: this permission is allowed to perform any read action on any collection. This includes querying using search handlers (using <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#search-handlers,request handlers>>) such as `/select`, `/get`, `/tvrh`, `/terms`, `/clustering`, `/elevate`, `/export`, `/spell`, `/clustering`, and `/sql`. This applies to all collections by default ( `collection:"*"` ).+* *collection-admin-edit*: this permission is allowed to edit a collection's configuration using the <<collections-api.adoc#,Collections API>>.+Note that this allows configuration edit permissions for _all_ collections.+If edit permissions should only be applied to specific collections, a custom permission would need to be created.+++Specifically, the following actions of the Collections API would be allowed:++++[.lowpadding,cols="1,1,1",frame=none,grid=none,stripes=none]+|===+| CREATE+| RELOAD+| SPLITSHARD+| CREATESHARD+| DELETESHARD+| CREATEALIAS+| DELETEALIAS+| DELETE+| DELETEREPLICA+| ADDREPLICA+| CLUSTERPROP+| MIGRATE+| ADDROLE+| REMOVEROLE+| ADDREPLICAPROP+| DELETEREPLICAPROP+| BALANCESHARDUNIQUE+| REBALANCELEADERS+|===++* *collection-admin-read*: this permission is allowed to read a collection's configuration using the <<collections-api.adoc#,Collections API>>.+Note that this allows configuration read permissions for _all_ collections.+If read permissions should only be applied to specific collections, a custom permission would need to be created.+++Specifically, the following actions of the Collections API would be allowed:+++LIST ++OVERSEERSTATUS ++CLUSTERSTATUS ++REQUESTSTATUS++* *update*: this permission is allowed to perform any update action on any collection.+This includes sending documents for indexing (using an <<requesthandlers-searchcomponents.adoc#update-request-handlers,update request handler>>).+This applies to all collections by default (`collection:"*"`).+* *read*: this permission is allowed to perform any read action on any collection.+This includes querying using search handlers (using <<requesthandlers-searchcomponents.adoc#search-handlers,request handlers>>) such as `/select`, `/get`, `/tvrh`, `/terms`, `/clustering`, `/elevate`, `/export`, `/spell`, `/clustering`, and `/sql`.+This applies to all collections by default ( `collection:"*"` ).* *zk-read* : Permission to read content from ZK (`/api/cluster/zk/data/*` , `/api/cluster/zk/ls/*` )* *all*: Any requests coming to Solr.=== Permission Ordering and Resolution-The permission syntax discussed above doesn't do anything to prevent multiple permissions from overlapping and applying to the same Solr APIs. In cases where multiple permissions match an incoming request, Solr chooses the first matching permission and ignores all others - even if those other permissions would match the incoming request!+The permission syntax discussed above doesn't do anything to prevent multiple permissions from overlapping and applying to the same Solr APIs.+In cases where multiple permissions match an incoming request, Solr chooses the first matching permission and ignores all others - even if those other permissions would match the incoming request!Since Solr only uses the first matching permission it finds, it's important for administrators to understand what ordering Solr uses when processing the permission list.-The ordering Solr uses is complex. Solr tries to check first any permissions which are specific or relevant to the incoming request, only moving on to more general permissions if none of the more-specific ones match. In effect, this means that different requests may check the same permissions in very different orders.+The ordering Solr uses is complex.+Solr tries to check first any permissions which are specific or relevant to the incoming request, only moving on to more general permissions if none of the more-specific ones match.+In effect, this means that different requests may check the same permissions in very different orders.If the incoming request is collection-agnostic (doesn't apply to a particular collection), Solr checks permissions in the following order:@@ -277,9 +459,11 @@ As an example, consider the permissions below:{"name": "all", "role": "admin"} <4>-----All of the permissions in this list match `/select` queries. But different permissions will be used depending on the collection being queried.+All of the permissions in this list match `/select` queries.+But different permissions will be used depending on the collection being queried.-For a query to the `techproducts` collection, permission 3 will be used because it specifically targets `techproducts`. Only users with the `other` role will be authorized.+For a query to the `techproducts` collection, permission 3 will be used because it specifically targets `techproducts`.+Only users with the `other` role will be authorized.For a query to a collection called `collection1` on the other hand, the most specific permission present is permission 2, so _all_ roles are given access.@@ -297,9 +481,13 @@ Three commands control managing permissions:* `update-permission`: update some attributes of an existing permission definition.* `delete-permission`: remove a permission definition.-Created properties can either be custom or predefined. In addition to the permission syntax discussed above, these commands also allow permissions to have a `before` property, whose value matches the index of the permission that this new permission should be placed before in `security.json`.+Created properties can either be custom or predefined.++In addition to the permission syntax discussed above, these commands also allow permissions to have a `before` property, whose value matches the index of the permission that this new permission should be placed before in `security.json`.-The following creates a new permission named "collection-mgr" that is allowed to create and list collections. The permission will be placed before the "read" permission. Note also that we have defined "collection as `null`, this is because requests to the Collections API are never collection-specific.+The following creates a new permission named "collection-mgr" that is allowed to create and list collections.+The permission will be placed before the "read" permission.+Note also that we have defined `collection` as `null` because requests to the Collections API are never collection-specific.[source,bash]curl --user solr:SolrRocks -H 'Content-type:application/json' -d '{@@ -320,7 +508,8 @@ curl --user solr:SolrRocks -H 'Content-type:application/json' -d '{=== Update or Delete Permissions-Permissions can be accessed using their index in the list. Use the `/admin/authorization` API to see the existing permissions and their indices.+Permissions can be accessed using their index in the list.+Use the `/admin/authorization` API to see the existing permissions and their indices.The following example updates the `'role'` attribute of permission at index `3`:@@ -344,7 +533,8 @@ A single command allows roles to be mapped to users:* `set-user-role`: map a user to a permission.-To remove a user's permission, you should set the role to `null`. There is no command to delete a user role.+To remove a user's permission, you should set the role to `null`.+There is no command to delete a user role.The values supplied to the command are simply a user ID and one or more roles the user should have.diff --git a/solr/solr-ref-guide/src/running-your-analyzer.adoc b/solr/solr-ref-guide/src/running-your-analyzer.adocdeleted file mode 100644index 092b4b96776..00000000000--- a/solr/solr-ref-guide/src/running-your-analyzer.adoc+++ /dev/null@@ -1,56 +0,0 @@-= Running Your Analyzer-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Once you've <<field-type-definitions-and-properties.adoc#,defined a field type in your Schema>>, and specified the analysis steps that you want applied to it, you should test it out to make sure that it behaves the way you expect it to.--Luckily, there is a very handy page in the Solr <<using-the-solr-administration-user-interface.adoc#,admin interface>> that lets you do just that. You can invoke the analyzer for any text field, provide sample input, and display the resulting token stream.--For example, let's look at some of the "Text" field types available in the `bin/solr -e techproducts` example configuration, and use the <<analysis-screen.adoc#,Analysis Screen>> (`\http://localhost:8983/solr/#/techproducts/analysis`) to compare how the tokens produced at index time for the sentence "Running an Analyzer" match up with a slightly different query text of "run my analyzer"--We can begin with "```text_ws```" - one of the most simplified Text field types available:--image::images/running-your-analyzer/analysis_compare_0.png[image]--By looking at the start and end positions for each term, we can see that the only thing this field type does is tokenize text on whitespace. Notice in this image that the term "Running" has a start position of 0 and an end position of 7, while "an" has a start position of 8 and an end position of 10, and "Analyzer" starts at 11 and ends at 19. If the whitespace between the terms was also included, the count would be 21; since it is 19, we know that whitespace has been removed from this query.--Note also that the indexed terms and the query terms are still very different. "Running" doesn't match "run", "Analyzer" doesn't match "analyzer" (to a computer), and obviously "an" and "my" are totally different words. If our objective is to allow queries like "run my analyzer" to match indexed text like "Running an Analyzer" then we will evidently need to pick a different field type with index & query time text analysis that does more processing of the inputs.--In particular we will want:--* Case insensitivity, so "Analyzer" and "analyzer" match.-* Stemming, so words like "Run" and "Running" are considered equivalent terms.-* Stop Word Pruning, so small words like "an" and "my" don't affect the query.--For our next attempt, let's try the "```text_general```" field type:--image::images/running-your-analyzer/analysis_compare_1.png[image]--With the verbose output enabled, we can see how each stage of our new analyzers modify the tokens they receive before passing them on to the next stage. As we scroll down to the final output, we can see that we do start to get a match on "analyzer" from each input string, thanks to the "LCF" stage -- which if you hover over with your mouse, you'll see is the "```LowerCaseFilter```":--image::images/running-your-analyzer/analysis_compare_2.png[image]--The "```text_general```" field type is designed to be generally useful for any language, and it has definitely gotten us closer to our objective than "```text_ws```" from our first example by solving the problem of case sensitivity. It's still not quite what we are looking for because we don't see stemming or stopword rules being applied. So now let us try the "```text_en```" field type:--image::images/running-your-analyzer/analysis_compare_3.png[image]--Now we can see the "SF" (`StopFilter`) stage of the analyzers solving the problem of removing Stop Words ("an"), and as we scroll down, we also see the "PSF" (`PorterStemFilter`) stage apply stemming rules suitable for our English language input, such that the terms produced by our "index analyzer" and the terms produced by our "query analyzer" match the way we expect.--image::images/running-your-analyzer/analysis_compare_4.png[image]---At this point, we can continue to experiment with additional inputs, verifying that our analyzers produce matching tokens when we expect them to match, and disparate tokens when we do not expect them to match, as we iterate and tweak our field type configuration.diff --git a/solr/solr-ref-guide/src/scaling-solr.adoc b/solr/solr-ref-guide/src/scaling-solr.adocnew file mode 100644index 00000000000..491d8c528e2--- /dev/null+++ b/solr/solr-ref-guide/src/scaling-solr.adoc@@ -0,0 +1,49 @@+= Scaling Solr+:page-children: cluster-types, \+ user-managed-clusters, \+ solrcloud-clusters+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Solr supports large implementations by providing two distinct options for multi-node clusters.++The section *<<cluster-types.adoc#,Solr Cluster Types>>* describes the two main approaches available for scaling.++Both approaches share the ability to distribute parts of an index across multiple servers, a process called _sharding_.+This allows for an index that is larger than any one server can host and maintain reasonable performance.++A second process called _replication_ is also shared by both approaches.+This allows for copying the index (in whole or as shards) across multiple servers, and provides query volume distribution.++The differences between the approaches are then how each cluster is managed and what features are available to you.++****+// This tags the below list so it can be used in the parent page section list+// tag::scaling-sections[]+[discrete]+==== User-Managed Clusters++include::user-managed-clusters.adoc[tag=user-managed-sections]++[discrete]+==== SolrCloud Clusters++include::solrcloud-clusters.adoc[tag=solrcloud-sections]++// end::scaling-sections[]+****diff --git a/solr/solr-ref-guide/src/schema-api.adoc b/solr/solr-ref-guide/src/schema-api.adocindex 21946b766e5..782d96ffaee 100644--- a/solr/solr-ref-guide/src/schema-api.adoc+++ b/solr/solr-ref-guide/src/schema-api.adoc@@ -18,28 +18,40 @@The Schema API allows you to use an HTTP API to manage many of the elements of your schema.-The Schema API utilizes the ManagedIndexSchemaFactory class, which is the default schema factory in modern Solr versions. See the section <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for more information about choosing a schema factory for your index.+This API provides read and write access to the Solr schema for each collection (or core, when not running SolrCloud).+Read access to all schema elements is supported.+Fields, dynamic fields, field types and copyField rules may be added, removed or replaced.+Future Solr releases will extend write access to allow more schema elements to be modified.-This API provides read and write access to the Solr schema for each collection (or core, when using standalone Solr). Read access to all schema elements is supported. Fields, dynamic fields, field types and copyField rules may be added, removed or replaced. Future Solr releases will extend write access to allow more schema elements to be modified.+The Schema API utilizes the `ManagedIndexSchemaFactory` class, which is the default schema factory in modern Solr versions.+See the section <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for more information about choosing a schema factory for your index.-.Why is hand editing of the managed schema discouraged?+.Hand editing of the managed schema is discouraged[NOTE]====-The file named "managed-schema" in the example configurations may include a note that recommends never hand-editing the file. Before the Schema API existed, such edits were the only way to make changes to the schema, and users may have a strong desire to continue making changes this way.+The file named "managed-schema" in the example configurations may include a note that recommends never hand-editing the file.+Before the Schema API existed, such edits were the only way to make changes to the schema, and users may have a strong desire to continue making changes this way.-The reason that this is discouraged is because hand-edits of the schema may be lost if the Schema API described here is later used to make a change, unless the core or collection is reloaded or Solr is restarted before using the Schema API. If care is taken to always reload or restart after a manual edit, then there is no problem at all with doing those edits.+The reason that this is discouraged is because hand-edits of the schema may be lost if the Schema API described here is later used to make a change, unless the core or collection is reloaded or Solr is restarted before using the Schema API.+If care is taken to always reload or restart after a manual edit, then there is no problem at all with doing those edits.====-The API allows two output modes for all calls: JSON or XML. When requesting the complete schema, there is another output mode which is XML modeled after the managed-schema file itself, which is in XML format.+The API allows two output modes for all calls: JSON or XML.+When requesting the complete schema, there is another output mode which is XML modeled after the managed-schema file itself, which is in XML format.-When modifying the schema with the API, a core reload will automatically occur in order for the changes to be available immediately for documents indexed thereafter. Previously indexed documents will *not* be automatically updated - they *must* be reindexed if existing index data uses schema elements that you changed.+When modifying the schema with the API, a core reload will automatically occur in order for the changes to be available immediately for documents indexed thereafter.+Previously indexed documents will *not* be automatically updated - they *must* be reindexed if existing index data uses schema elements that you changed..Reindex after schema modifications![IMPORTANT]====-If you modify your schema, you will likely need to reindex all documents. If you do not, you may lose access to documents, or not be able to interpret them properly, e.g., after replacing a field type.+If you modify your schema, you will likely need to reindex all documents.+If you do not, you may lose access to documents, or not be able to interpret them properly, e.g., after replacing a field type.-Modifying your schema will never modify any documents that are already indexed. You must reindex documents in order to apply schema changes to them. Queries and updates made after the change may encounter errors that were not present before the change. Completely deleting the index and rebuilding it is usually the only option to fix such errors.+Modifying your schema will never modify any documents that are already indexed.+You must reindex documents in order to apply schema changes to them.+Queries and updates made after the change may encounter errors that were not present before the change.+Completely deleting the index and rebuilding it is usually the only option to fix such errors.See the section <<reindexing.adoc#,Reindexing>> for more information about reindexing.====@@ -53,7 +65,8 @@ bin/solr -e techproducts== Modify the Schema-To add, remove or replace fields, dynamic field rules, copy field rules, or new field types, you can send a POST request to the `/schema/` endpoint with a sequence of commands in JSON format to perform the requested actions. The following commands are supported:+To add, remove, or replace fields, dynamic field rules, copy field rules, or new field types, you can send a POST request to the `/schema/` endpoint with a sequence of commands in JSON format to perform the requested actions.+The following commands are supported:* `add-field`: add a new field with parameters you provide.* `delete-field`: delete a field.@@ -67,17 +80,21 @@ To add, remove or replace fields, dynamic field rules, copy field rules, or new* `add-copy-field`: add a new copy field rule.* `delete-copy-field`: delete a copy field rule.-These commands can be issued in separate POST requests or in the same POST request. Commands are executed in the order in which they are specified.+These commands can be issued in separate POST requests or in the same POST request.+Commands are executed in the order in which they are specified.In each case, the response will include the status and the time to process the request, but will not include the entire schema.-When modifying the schema with the API, a core reload will automatically occur in order for the changes to be available immediately for documents indexed thereafter. Previously indexed documents will *not* be automatically handled - they *must* be reindexed if they used schema elements that you changed.+When modifying the schema with the API, a core reload will automatically occur in order for the changes to be available immediately for documents indexed thereafter.+Previously indexed documents will *not* be automatically handled - they *must* be reindexed if they used schema elements that you changed.=== Add a New Field-The `add-field` command adds a new field definition to your schema. If a field with the same name exists an error is thrown.+The `add-field` command adds a new field definition to your schema.+If a field with the same name exists an error is thrown.-All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API. These request attributes are described in detail in the section <<defining-fields.adoc#,Defining Fields>>.+All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API.+These request attributes are described in detail in the section <<fields.adoc#,Fields>>.For example, to define a new stored field named "sell_by", of type "pdate", you would POST the following request:@@ -114,7 +131,8 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Delete a Field-The `delete-field` command removes a field definition from your schema. If the field does not exist in the schema, or if the field is the source or destination of a copy field rule, an error is thrown.+The `delete-field` command removes a field definition from your schema.+If the field does not exist in the schema, or if the field is the source or destination of a copy field rule, an error is thrown.For example, to delete a field named "sell_by", you would POST the following request:@@ -145,9 +163,12 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Replace a Field-The `replace-field` command replaces a field's definition. Note that you must supply the full definition for a field - this command will *not* partially modify a field's definition. If the field does not exist in the schema an error is thrown.+The `replace-field` command replaces a field's definition.+Note that you must supply the full definition for a field - this command will *not* partially modify a field's definition.+If the field does not exist in the schema an error is thrown.-All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API. These request attributes are described in detail in the section <<defining-fields.adoc#,Defining Fields>>.+All of the properties available when defining a field with manual `schema.xml` edits can be passed via the API.+These request attributes are described in detail in the section <<fields.adoc#,Fields>>.For example, to replace the definition of an existing field "sell_by", to make it be of type "date" and to not be stored, you would POST the following request:@@ -186,7 +207,8 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{The `add-dynamic-field` command adds a new dynamic field rule to your schema.-All of the properties available when editing `schema.xml` can be passed with the POST request. The section <<dynamic-fields.adoc#,Dynamic Fields>> has details on all of the attributes that can be defined for a dynamic field rule.+All of the properties available when editing `schema.xml` can be passed with the POST request.+The section <<dynamic-fields.adoc#,Dynamic Fields>> has details on all of the attributes that can be defined for a dynamic field rule.For example, to create a new dynamic field rule where all incoming fields ending with "_s" would be stored and have field type "string", you can POST a request like this:@@ -223,7 +245,8 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Delete a Dynamic Field Rule-The `delete-dynamic-field` command deletes a dynamic field rule from your schema. If the dynamic field rule does not exist in the schema, or if the schema contains a copy field rule with a target or destination that matches only this dynamic field rule, an error is thrown.+The `delete-dynamic-field` command deletes a dynamic field rule from your schema.+If the dynamic field rule does not exist in the schema, or if the schema contains a copy field rule with a target or destination that matches only this dynamic field rule, an error is thrown.For example, to delete a dynamic field rule matching "*_s", you can POST a request like this:@@ -254,9 +277,12 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Replace a Dynamic Field Rule-The `replace-dynamic-field` command replaces a dynamic field rule in your schema. Note that you must supply the full definition for a dynamic field rule - this command will *not* partially modify a dynamic field rule's definition. If the dynamic field rule does not exist in the schema an error is thrown.+The `replace-dynamic-field` command replaces a dynamic field rule in your schema.+Note that you must supply the full definition for a dynamic field rule - this command will *not* partially modify a dynamic field rule's definition.+If the dynamic field rule does not exist in the schema an error is thrown.-All of the properties available when editing `schema.xml` can be passed with the POST request. The section <<dynamic-fields.adoc#,Dynamic Fields>> has details on all of the attributes that can be defined for a dynamic field rule.+All of the properties available when editing `schema.xml` can be passed with the POST request.+The section <<dynamic-fields.adoc#,Dynamic Fields>> has details on all of the attributes that can be defined for a dynamic field rule.For example, to replace the definition of the "*_s" dynamic field rule with one where the field type is "text_general" and it's not stored, you can POST a request like this:@@ -295,7 +321,9 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{The `add-field-type` command adds a new field type to your schema.-All of the field type properties available when editing `schema.xml` by hand are available for use in a POST request. The structure of the command is a JSON mapping of the standard field type definition, including the name, class, index and query analyzer definitions, etc. Details of all of the available options are described in the section <<solr-field-types.adoc#,Solr Field Types>>.+All of the field type properties available when editing `schema.xml` by hand are available for use in a POST request.+The structure of the command is a JSON mapping of the standard field type definition, including the name, class, index and query analyzer definitions, etc.+Details of all of the available options are described in the section <<field-types.adoc#,Field Types>>.For example, to create a new field type named "myNewTxtField", you can POST a request as follows:@@ -331,7 +359,8 @@ Note in this example that we have only defined a single analyzer section that wi====[tab-label]*V1 API with Two Analyzers*-If we wanted to define separate analysis, we would replace the `analyzer` section in the above example with separate sections for `indexAnalyzer` and `queryAnalyzer`. As in this example:+If we wanted to define separate analysis, we would replace the `analyzer` section in the above example with separate sections for `indexAnalyzer` and `queryAnalyzer`.+As in this example:[source,bash]----@@ -375,7 +404,8 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Delete a Field Type-The `delete-field-type` command removes a field type from your schema. If the field type does not exist in the schema, or if any field or dynamic field rule in the schema uses the field type, an error is thrown.+The `delete-field-type` command removes a field type from your schema.+If the field type does not exist in the schema, or if any field or dynamic field rule in the schema uses the field type, an error is thrown.For example, to delete the field type named "myNewTxtField", you can make a POST request as follows:@@ -406,9 +436,13 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Replace a Field Type-The `replace-field-type` command replaces a field type in your schema. Note that you must supply the full definition for a field type - this command will *not* partially modify a field type's definition. If the field type does not exist in the schema an error is thrown.+The `replace-field-type` command replaces a field type in your schema.+Note that you must supply the full definition for a field type - this command will *not* partially modify a field type's definition.+If the field type does not exist in the schema an error is thrown.-All of the field type properties available when editing `schema.xml` by hand are available for use in a POST request. The structure of the command is a JSON mapping of the standard field type definition, including the name, class, index and query analyzer definitions, etc. Details of all of the available options are described in the section <<solr-field-types.adoc#,Solr Field Types>>.+All of the field type properties available when editing `schema.xml` by hand are available for use in a POST request.+The structure of the command is a JSON mapping of the standard field type definition, including the name, class, index and query analyzer definitions, etc.+Details of all of the available options are described in the section <<field-types.adoc#,Field Types>>.For example, to replace the definition of a field type named "myNewTxtField", you can make a POST request as follows:@@ -456,13 +490,16 @@ The `add-copy-field` command adds a new copy field rule to your schema.The attributes supported by the command are the same as when creating copy field rules by manually editing the `schema.xml`, as below:`source`::-The source field. This parameter is required.+The source field.+This parameter is required.`dest`::-A field or an array of fields to which the source field will be copied. This parameter is required.+A field or an array of fields to which the source field will be copied.+This parameter is required.`maxChars`::-The upper limit for the number of characters to be copied. The section <<copying-fields.adoc#,Copying Fields>> has more details.+The upper limit for the number of characters to be copied.+The section <<copy-fields.adoc#,Copy Fields>> has more details.For example, to define a rule to copy the field "shelf" to the "location" and "catchall" fields, you would POST the following request:@@ -497,7 +534,8 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Delete a Copy Field Rule-The `delete-copy-field` command deletes a copy field rule from your schema. If the copy field rule does not exist in the schema an error is thrown.+The `delete-copy-field` command deletes a copy field rule from your schema.+If the copy field rule does not exist in the schema an error is thrown.The `source` and `dest` attributes are required by this command.@@ -530,11 +568,15 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{=== Multiple Commands in a Single POST-It is possible to perform one or more add requests in a single command. The API is transactional and all commands in a single call either succeed or fail together.+It is possible to perform one or more add requests in a single command.+The API is transactional and all commands in a single call either succeed or fail together.-The commands are executed in the order in which they are specified. This means that if you want to create a new field type and in the same request use the field type on a new field, the section of the request that creates the field type must come before the section that creates the new field. Similarly, since a field must exist for it to be used in a copy field rule, a request to add a field must come before a request for the field to be used as either the source or the destination for a copy field rule.+The commands are executed in the order in which they are specified.+This means that if you want to create a new field type and in the same request use the field type on a new field, the section of the request that creates the field type must come before the section that creates the new field.+Similarly, since a field must exist for it to be used in a copy field rule, a request to add a field must come before a request for the field to be used as either the source or the destination for a copy field rule.-The syntax for making multiple requests supports several approaches. First, the commands can simply be made serially, as in this request to create a new field type and then a field that uses that type:+The syntax for making multiple requests supports several approaches.+First, the commands can simply be made serially, as in this request to create a new field type and then a field that uses that type:[source,bash]----@@ -598,15 +640,21 @@ curl -X POST -H 'Content-type:application/json' --data-binary '{When running in SolrCloud mode, changes made to the schema on one node will propagate to all replicas in the collection.-You can pass the `updateTimeoutSecs` parameter with your request to set the number of seconds to wait until all replicas confirm they applied the schema updates. This helps your client application be more robust in that you can be sure that all replicas have a given schema change within a defined amount of time.+You can pass the `updateTimeoutSecs` parameter with your request to set the number of seconds to wait until all replicas confirm they applied the schema updates.+This helps your client application be more robust in that you can be sure that all replicas have a given schema change within a defined amount of time.-If agreement is not reached by all replicas in the specified time, then the request fails and the error message will include information about which replicas had trouble. In most cases, the only option is to re-try the change after waiting a brief amount of time. If the problem persists, then you'll likely need to investigate the server logs on the replicas that had trouble applying the changes.+If agreement is not reached by all replicas in the specified time, then the request fails and the error message will include information about which replicas had trouble.+In most cases, the only option is to re-try the change after waiting a brief amount of time.+If the problem persists, then you'll likely need to investigate the server logs on the replicas that had trouble applying the changes.-If you do not supply an `updateTimeoutSecs` parameter, the default behavior is for the receiving node to return immediately after persisting the updates to ZooKeeper. All other replicas will apply the updates asynchronously. Consequently, without supplying a timeout, your client application cannot be sure that all replicas have applied the changes.+If you do not supply an `updateTimeoutSecs` parameter, the default behavior is for the receiving node to return immediately after persisting the updates to ZooKeeper.+All other replicas will apply the updates asynchronously.+Consequently, without supplying a timeout, your client application cannot be sure that all replicas have applied the changes.== Retrieve Schema Information-The following endpoints allow you to read how your schema has been defined. You can GET the entire schema, or only portions of it as needed.+The following endpoints allow you to read how your schema has been defined.+You can GET the entire schema, or only portions of it as needed.To modify the schema, see the previous section <<Modify the Schema>>.@@ -626,13 +674,16 @@ The collection (or core) name.The query parameters should be added to the API request after '?'.`wt`::-Defines the format of the response. The options are *json*, *xml* or *schema.xml*. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are *json*, *xml* or *schema.xml*.+If not specified, JSON will be returned by default.==== Retrieve Schema Response*Output Content*-The output will include all fields, field types, dynamic rules and copy field rules, in the format requested (JSON or XML). The schema name and version are also included.+The output will include all fields, field types, dynamic rules and copy field rules, in the format requested (JSON or XML).+The schema name and version are also included.==== Retrieve Schema Examples@@ -804,10 +855,13 @@ The specific fieldname (if limiting the request to a single field).The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.`fl`::-Comma- or space-separated list of one or more fields to return. If not specified, all fields will be returned by default.+Comma- or space-separated list of one or more fields to return.+If not specified, all fields will be returned by default.`includeDynamic`::If `true`, and if the `fl` query parameter is specified or the `fieldname` path parameter is used, matching dynamic fields are included in the response and identified with the `dynamicBase` property.@@ -817,13 +871,16 @@ If neither the `fl` query parameter nor the `fieldname` path parameter is specifIf `false`, the default, matching dynamic fields will not be returned.`showDefaults`::-If `true`, all default field properties from each field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`). If `false`, the default, only explicitly specified field properties will be included.+If `true`, all default field properties from each field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`).+If `false`, the default, only explicitly specified field properties will be included.==== List Fields Response-The output will include each field and any defined configuration for each field. The defined configuration can vary for each field, but will minimally include the field `name`, the `type`, if it is `indexed` and if it is `stored`.+The output will include each field and any defined configuration for each field.+The defined configuration can vary for each field, but will minimally include the field `name`, the `type`, if it is `indexed` and if it is `stored`.-If `multiValued` is defined as either true or false (most likely true), that will also be shown. See the section <<defining-fields.adoc#,Defining Fields>> for more information about each parameter.+If `multiValued` is defined as either true or false (most likely true), that will also be shown.+See the section <<fields.adoc#,Fields>> for more information about each parameter.==== List Fields Examples@@ -904,14 +961,19 @@ The name of the dynamic field rule (if limiting request to a single dynamic fielThe query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.`showDefaults`::-If `true`, all default field properties from each dynamic field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`). If `false`, the default, only explicitly specified field properties will be included.+If `true`, all default field properties from each dynamic field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`).+If `false`, the default, only explicitly specified field properties will be included.==== List Dynamic Field Response-The output will include each dynamic field rule and the defined configuration for each rule. The defined configuration can vary for each rule, but will minimally include the dynamic field `name`, the `type`, if it is `indexed` and if it is `stored`. See the section <<dynamic-fields.adoc#,Dynamic Fields>> for more information about each parameter.+The output will include each dynamic field rule and the defined configuration for each rule.+The defined configuration can vary for each rule, but will minimally include the dynamic field `name`, the `type`, if it is `indexed` and if it is `stored`.+See the section <<dynamic-fields.adoc#,Dynamic Fields>> for more information about each parameter.==== List Dynamic Field Examples@@ -1004,14 +1066,20 @@ The name of the field type (if limiting request to a single field type).The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.`showDefaults`::-If `true`, all default field properties from each dynamic field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`). If `false`, the default, only explicitly specified field properties will be included.+If `true`, all default field properties from each dynamic field's field type will be included in the response (e.g., `tokenized` for `solr.TextField`).+If `false`, the default, only explicitly specified field properties will be included.==== List Field Type Response-The output will include each field type and any defined configuration for the type. The defined configuration can vary for each type, but will minimally include the field type `name` and the `class`. If query or index analyzers, tokenizers, or filters are defined, those will also be shown with other defined parameters. See the section <<solr-field-types.adoc#,Solr Field Types>> for more information about how to configure various types of fields.+The output will include each field type and any defined configuration for the type.+The defined configuration can vary for each type, but will minimally include the field type `name` and the `class`.+If query or index analyzers, tokenizers, or filters are defined, those will also be shown with other defined parameters.+See the section <<field-types.adoc#,Field Types>> for more information about how to configure various types of fields.==== List Field Type Examples@@ -1104,17 +1172,24 @@ The collection (or core) name.The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.`source.fl`::-Comma- or space-separated list of one or more copyField source fields to include in the response - copyField directives with all other source fields will be excluded from the response. If not specified, all copyField-s will be included in the response.+Comma- or space-separated list of one or more copyField source fields to include in the response.+CopyField directives with all other source fields will be excluded from the response.+If not specified, all copyField-s will be included in the response.`dest.fl`::-Comma- or space-separated list of one or more copyField destination fields to include in the response. copyField directives with all other `dest` fields will be excluded. If not specified, all copyField-s will be included in the response.+Comma- or space-separated list of one or more copyField destination fields to include in the response.+CopyField directives with all other `dest` fields will be excluded.+If not specified, all copyField-s will be included in the response.==== List Copy Field Response-The output will include the `source` and `dest` (destination) of each copy field rule defined in `schema.xml`. For more information about copying fields, see the section <<copying-fields.adoc#,Copying Fields>>.+The output will include the `source` and `dest` (destination) of each copy field rule defined in `schema.xml`.+For more information about copy fields, see the section <<copy-fields.adoc#,Copy Fields>>.==== List Copy Field Examples@@ -1187,7 +1262,9 @@ The collection (or core) name.The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.==== Show Schema Response@@ -1244,7 +1321,9 @@ The collection (or core) name.The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.==== Show Schema Version Response@@ -1300,8 +1379,10 @@ The collection (or core) name.The query parameters can be added to the API request after a '?'.-|`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+`wt`::+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.==== List UniqueKey Response@@ -1357,7 +1438,9 @@ The collection (or core) name.The query parameters can be added to the API request after a '?'.`wt`::-Defines the format of the response. The options are `json` or `xml`. If not specified, JSON will be returned by default.+Defines the format of the response.+The options are `json` or `xml`.+If not specified, JSON will be returned by default.==== Show Global Similary Response@@ -1400,4 +1483,6 @@ curl -X GET "http://localhost:8983/api/collections/techproducts/schema/similarit== Manage Resource Data-The <<managed-resources.adoc#,Managed Resources>> REST API provides a mechanism for any Solr plugin to expose resources that should support CRUD (Create, Read, Update, Delete) operations. Depending on what Field Types and Analyzers are configured in your Schema, additional `/schema/` REST API paths may exist. See the <<managed-resources.adoc#,Managed Resources>> section for more information and examples.+The <<managed-resources.adoc#,Managed Resources>> REST API provides a mechanism for any Solr plugin to expose resources that should support CRUD (Create, Read, Update, Delete) operations.+Depending on which field types and analyzers are configured in your Schema, additional `/schema/` REST API paths may exist.+See the <<managed-resources.adoc#,Managed Resources>> section for more information and examples.diff --git a/solr/solr-ref-guide/src/schema-elements.adoc b/solr/solr-ref-guide/src/schema-elements.adocnew file mode 100644index 00000000000..31b1358dc36--- /dev/null+++ b/solr/solr-ref-guide/src/schema-elements.adoc@@ -0,0 +1,158 @@+= Schema Elements+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr stores details about the field types and fields it is expected to understand in a schema file.++== Solr's Schema File+The name and location of Solr's schema file may vary depending on how you initially configured Solr or if you modified it later.++* `managed-schema` is the name for the schema file Solr uses by default to support making schema changes at runtime via the <<schema-api.adoc#,Schema API>>, or <<schemaless-mode.adoc#,Schemaless Mode>> features.+++You may <<schema-factory.adoc#,explicitly configure the managed schema features>> to use an alternative filename if you choose, but the contents of the files are still updated automatically by Solr.+* `schema.xml` is the traditional name for a schema file which can be edited manually by users who use the <<schema-factory.adoc#,`ClassicIndexSchemaFactory`>>.+* If you are using SolrCloud you may not be able to find any file by these names on the local filesystem.+You will only be able to see the schema through the Schema API (if enabled) or through the Solr Admin UI's <<cloud-screens.adoc#,Cloud Screens>>.++Whichever name of the file in use in your installation, the structure of the file is not changed.+However, the way you interact with the file will change.+If you are using the managed schema, it is expected that you only interact with the file with the Schema API, and never make manual edits.+If you do not use the managed schema, you will only be able to make manual edits to the file, the Schema API will not support any modifications.++Note that if you are not using the Schema API yet you do use SolrCloud, you will need to interact with `schema.xml` through ZooKeeper using `upconfig` and `downconfig` commands to make a local copy and upload your changes.+The options for doing this are described in <<solr-control-script-reference.adoc#,Solr Control Script Reference>> and <<zookeeper-file-management.adoc#,ZooKeeper File Management>>.++== Structure of the Schema File++This example is not real XML, but shows the primary elements that make up a schema file.++[source,xml]+----+<schema>+ <types>+ <fieldType>+ <fields>+ <field>+ <copyField>+ <dynamicField>+ <similarity>+ <uniqueKey>+</schema>+----++The most commonly defined elements are `types` and `fields`, where the field types and the actual fields are configured.+The sections <<field-types.adoc#,Field Types>>, and <<fields.adoc#,Fields>> describe how to configure these for your schema.++These are supplemented by `copyFields`, described in <<copy-fields.adoc#,Copy Fields>>, and `dynamicFields`, described in <<dynamic-fields.adoc#,Dynamic Fields>>.++The `uniqueKey` described in <<Unique Key>> below must always be defined.++A default `similarity` will be used, but can be modifed as described in the section <<Similarity>> below.++.Types and fields are optional tags+[NOTE]+====+Note that the `types` and `fields` sections are optional, meaning you are free to mix `field`, `dynamicField`, `copyField` and `fieldType` definitions on the top level.+This allows for a more logical grouping of related elements (such as fields grouped with their field type definition) in your schema.+====++== Unique Key++The `uniqueKey` element specifies which field is a unique identifier for documents.+Although `uniqueKey` is not required, it is nearly always warranted by your application design.+For example, `uniqueKey` should be used if you will ever update a document in the index.++You can define the unique key field by naming it:++[source,xml]+----+<uniqueKey>id</uniqueKey>+----++Schema defaults and `copyFields` cannot be used to populate the `uniqueKey` field.+The `fieldType` of `uniqueKey` must not be analyzed and must not be any of the `*PointField` types.+You can use `UUIDUpdateProcessorFactory` to have `uniqueKey` values generated automatically.++Further, the operation will fail if the `uniqueKey` field is used, but is multivalued (or inherits the multivalued-ness from the `fieldtype`).+However, `uniqueKey` will continue to work, as long as the field is properly used.++== Similarity++Similarity is a Lucene class used to score a document in searching.++Each collection has one "global" Similarity.+By default Solr uses an implicit {solr-javadocs}/core/org/apache/solr/search/similarities/SchemaSimilarityFactory.html[`SchemaSimilarityFactory`] which allows individual field types to be configured with a "per-type" specific Similarity and implicitly uses `BM25Similarity` for any field type which does not have an explicit Similarity.++This default behavior can be overridden by declaring a top level `<similarity/>` element in your `schema.xml`, outside of any single field type.+This similarity declaration can either refer directly to the name of a class with a no-argument constructor, such as in this example showing `BM25Similarity`:++[source,xml]+----+<similarity class="solr.BM25SimilarityFactory"/>+----++or by referencing a `SimilarityFactory` implementation, which may take optional initialization parameters:++[source,xml]+----+<similarity class="solr.DFRSimilarityFactory">+ <str name="basicModel">P</str>+ <str name="afterEffect">L</str>+ <str name="normalization">H2</str>+ <float name="c">7</float>+</similarity>+----++In most cases, specifying global level similarity like this will cause an error if your `schema.xml` also includes field type specific `<similarity/>` declarations.+One key exception to this is that you may explicitly declare a {solr-javadocs}/core/org/apache/solr/search/similarities/SchemaSimilarityFactory.html[`SchemaSimilarityFactory`] and specify what that default behavior will be for all field types that do not declare an explicit Similarity using the name of field type (specified by `defaultSimFromFieldType`) that _is_ configured with a specific similarity:++[source,xml]+----+<similarity class="solr.SchemaSimilarityFactory">+ <str name="defaultSimFromFieldType">text_dfr</str>+</similarity>++<fieldType name="text_dfr" class="solr.TextField">+ <analyzer ... />+ <similarity class="solr.DFRSimilarityFactory">+ <str name="basicModel">I(F)</str>+ <str name="afterEffect">B</str>+ <str name="normalization">H3</str>+ <float name="mu">900</float>+ </similarity>+</fieldType>++<fieldType name="text_ib" class="solr.TextField">+ <analyzer ... />+ <similarity class="solr.IBSimilarityFactory">+ <str name="distribution">SPL</str>+ <str name="lambda">DF</str>+ <str name="normalization">H2</str>+ </similarity>+</fieldType>++<fieldType name="text_other" class="solr.TextField">+ <analyzer ... />+</fieldType>+----++In the example above `IBSimilarityFactory` (using the Information-Based model) will be used for any fields of type `text_ib`, while `DFRSimilarityFactory` (divergence from random) will be used for any fields of type `text_dfr`, as well as any fields using a type that does not explicitly specify a `<similarity/>`.++If `SchemaSimilarityFactory` is explicitly declared without configuring a `defaultSimFromFieldType`, then `BM25Similarity` is implicitly used as the default for `luceneMatchVersion >= 8.0.0` and otherwise `LegacyBM25Similarity` is used to mimic the same BM25 formula that was the default in those versions.++In addition to the various factories mentioned on this page, there are several other similarity implementations that can be used such as the `SweetSpotSimilarityFactory`, `ClassicSimilarityFactory`, `LegacyBM25SimilarityFactory` etc.+For details, see the Solr Javadocs for the {solr-javadocs}/core/org/apache/solr/schema/SimilarityFactory.html[similarity factories].diff --git a/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc b/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adocdeleted file mode 100644index 4a9d242be7e..00000000000--- a/solr/solr-ref-guide/src/schema-factory-definition-in-solrconfig.adoc+++ /dev/null@@ -1,91 +0,0 @@-= Schema Factory Definition in SolrConfig-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr's <<schema-api.adoc#,Schema API>> enables remote clients to access <<documents-fields-and-schema-design.adoc#,schema>> information, and make schema modifications, through a REST interface.--Other features such as Solr's <<schemaless-mode.adoc#,Schemaless Mode>> also work via schema modifications made programatically at run time.--[IMPORTANT]-====-Using the Managed Schema is required to be able to use the Schema API to modify your schema. However, using Managed Schema does not by itself mean you are also using Solr in Schemaless Mode (or "schema guessing" mode).--Schemaless mode requires enabling the Managed Schema if it is not already, but full schema guessing requires additional configuration as described in the section <<schemaless-mode.adoc#,Schemaless Mode>>.-====--While the "read" features of the Schema API are supported for all schema types, support for making schema modifications programatically depends on the `<schemaFactory/>` in use.--== Solr Uses Managed Schema by Default--When a `<schemaFactory/>` is not explicitly declared in a `solrconfig.xml` file, Solr implicitly uses a `ManagedIndexSchemaFactory`, which is by default `"mutable"` and keeps schema information in a `managed-schema` file.--[source,xml]------ <!-- An example of Solr's implicit default behavior if no- no schemaFactory is explicitly defined.- -->- <schemaFactory class="ManagedIndexSchemaFactory">- <bool name="mutable">true</bool>- <str name="managedSchemaResourceName">managed-schema</str>- </schemaFactory>-------If you wish to explicitly configure `ManagedIndexSchemaFactory` the following options are available:--* `mutable` - controls whether changes may be made to the Schema data. This must be set to *true* to allow edits to be made with the Schema API.-* `managedSchemaResourceName` is an optional parameter that defaults to "managed-schema", and defines a new name for the schema file that can be anything other than "`schema.xml`".--With the default configuration shown above, you can use the <<schema-api.adoc#,Schema API>> to modify the schema as much as you want, and then later change the value of `mutable` to *false* if you wish to "lock" the schema in place and prevent future changes.--== Classic schema.xml--An alternative to using a managed schema is to explicitly configure a `ClassicIndexSchemaFactory`. `ClassicIndexSchemaFactory` requires the use of a `schema.xml` configuration file, and disallows any programatic changes to the Schema at run time. The `schema.xml` file must be edited manually and is only loaded only when the collection is loaded.--[source,xml]------ <schemaFactory class="ClassicIndexSchemaFactory"/>-------=== Switching from schema.xml to Managed Schema--If you have an existing Solr collection that uses `ClassicIndexSchemaFactory`, and you wish to convert to use a managed schema, you can simply modify the `solrconfig.xml` to specify the use of the `ManagedIndexSchemaFactory`.--Once Solr is restarted and it detects that a `schema.xml` file exists, but the `managedSchemaResourceName` file (i.e., "`managed-schema`") does not exist, the existing `schema.xml` file will be renamed to `schema.xml.bak` and the contents are re-written to the managed schema file. If you look at the resulting file, you'll see this at the top of the page:--[source,xml]------<!-- Solr managed schema - automatically generated - DO NOT EDIT -->-------You are now free to use the <<schema-api.adoc#,Schema API>> as much as you want to make changes, and remove the `schema.xml.bak`.--=== Switching from Managed Schema to Manually Edited schema.xml--If you have started Solr with managed schema enabled and you would like to switch to manually editing a `schema.xml` file, you should take the following steps:--. Rename the `managed-schema` file to `schema.xml`.-. Modify `solrconfig.xml` to replace the `schemaFactory` class.-.. Remove any `ManagedIndexSchemaFactory` definition if it exists.-.. Add a `ClassicIndexSchemaFactory` definition as shown above-. Reload the core(s).--If you are using SolrCloud, you may need to modify the files via ZooKeeper. The `bin/solr` script provides an easy way to download the files from ZooKeeper and upload them back after edits. See the section <<solr-control-script-reference.adoc#zookeeper-operations,ZooKeeper Operations>> for more information.--[TIP]-====-To have full control over your `schema.xml` file, you may also want to disable schema guessing, which allows unknown fields to be added to the schema during indexing. The properties that enable this feature are discussed in the section <<schemaless-mode.adoc#,Schemaless Mode>>.-====diff --git a/solr/solr-ref-guide/src/schema-factory.adoc b/solr/solr-ref-guide/src/schema-factory.adocnew file mode 100644index 00000000000..125054e7fa9--- /dev/null+++ b/solr/solr-ref-guide/src/schema-factory.adoc@@ -0,0 +1,121 @@+= Schema Factory Configuration+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++Solr supports two styles of schema: a managed schema and a manually maintained `schema.xml` file.+When using a managed schema, features such as the <<schema-api.adoc#,Schema API>> and <<schemaless-mode.adoc#,Schemaless Mode>> are available.+When using `schema.xml`, the only way changes can be made to Solr's schema is by manually editing the file.++== <schemaFactory> in solrconfig.xml++The `<schemaFactory>` in `solrconfig.xml` defines whether your system will use a managed schema or `schema.xml`.++[source,xml]+----+<config>+ <schemaFactory>+ ...+ </schemaFactory>+</config>+----++++While the "read" features of the Schema API are supported for all schema types, support for making schema modifications programmatically depends on the `<schemaFactory/>` in use.++== Schema Factories++=== ManagedIndexSchemaFactory++When a `<schemaFactory/>` is not explicitly declared in a `solrconfig.xml` file, Solr implicitly uses a `ManagedIndexSchemaFactory`, which is by default `"mutable"` (editable) and keeps schema information in a `managed-schema` file.++Using the Managed Schema is required to be able to use the Schema API to modify your schema.+However, using Managed Schema does not mean you are also using Solr in Schemaless Mode (or "schema guessing" mode).++Schemaless mode requires enabling the Managed Schema if it is not already, but full schema guessing requires additional configuration as described in the section <<schemaless-mode.adoc#,Schemaless Mode>>.++Below is an example of a `schemaFactory` that reflects Solr's defaults:++[source,xml]+----+ <schemaFactory class="ManagedIndexSchemaFactory">+ <bool name="mutable">true</bool>+ <str name="managedSchemaResourceName">managed-schema</str>+ </schemaFactory>+----++The defaults can be overridden by explicitly configuring `ManagedIndexSchemaFactory` and changing one of the following options:++`mutable`::+Controls whether changes may be made to the schema data.+This must be set to `true` to allow edits to be made with the Schema API.+++With the default configuration shown above, you could use the <<schema-api.adoc#,Schema API>> to modify the schema as much as you want, and then later change the value of `mutable` to `false` to "lock" the schema in place and prevent future changes.++`managedSchemaResourceName`::+The name of the schema file.+The name can be anything other than `schema.xml`, as that name is reserved for the `ClassicIndexSchemaFactory`.++=== ClassicIndexSchemaFactory++An alternative to using a managed schema is to explicitly configure a `ClassicIndexSchemaFactory`.+This requires the use of a `schema.xml` file, and disallows any programatic changes to the Schema at run time.+The `schema.xml` file must be edited manually and is loaded only when the collection is loaded.++[source,xml]+----+ <schemaFactory class="ClassicIndexSchemaFactory"/>+----++This option takes no parameters.++== Changing Schema Factories++=== Switching from schema.xml to Managed Schema++If you have an existing Solr collection that uses `ClassicIndexSchemaFactory`, and you wish to convert to use a managed schema, you can simply modify the `solrconfig.xml` to specify the use of the `ManagedIndexSchemaFactory`.++Once Solr is restarted and it detects that a `schema.xml` file exists, but the `managedSchemaResourceName` file (i.e., "`managed-schema`") does not exist, the existing `schema.xml` file will be renamed to `schema.xml.bak` and the contents re-written to the managed schema file.++If you look at the resulting file, you'll see this at the top of the page:++[source,xml]+----+<!-- Solr managed schema - automatically generated - DO NOT EDIT -->+----++You are now free to use the <<schema-api.adoc#,Schema API>> to make changes, and remove the `schema.xml.bak`.++=== Switching from Managed Schema to schema.xml++If you have started Solr with managed schema enabled and you would like to switch to manually editing a `schema.xml` file, you should take the following steps:++. Rename the `managed-schema` file to `schema.xml`.+. Modify `solrconfig.xml` to replace the `schemaFactory` class.+.. Remove any `ManagedIndexSchemaFactory` definition if it exists.+.. Add a `ClassicIndexSchemaFactory` definition as shown above+. Reload the core(s).++If you are using SolrCloud, you may need to modify the files via ZooKeeper.+The `bin/solr` script provides an easy way to download the files from ZooKeeper and upload them back after edits.+See the section <<solr-control-script-reference.adoc#zookeeper-operations,ZooKeeper Operations>> for more information.++[TIP]+====+To have full control over your `schema.xml` file, you may also want to disable schema guessing, which allows unknown fields to be added to the schema during indexing.+The properties that enable this feature are discussed in the section <<schemaless-mode.adoc#,Schemaless Mode>>.+====diff --git a/solr/solr-ref-guide/src/schema-indexing-guide.adoc b/solr/solr-ref-guide/src/schema-indexing-guide.adocnew file mode 100644index 00000000000..0e6450444d8--- /dev/null+++ b/solr/solr-ref-guide/src/schema-indexing-guide.adoc@@ -0,0 +1,63 @@+= Schema and Indexing Guide+:page-children: solr-schema, \+ fields-and-schema-design, \+ document-analysis, \+ indexing-data-operations+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+This Guide covers indexing documents in Solr.++****+[discrete]+=== Solr's Schema++<<solr-schema.adoc#,*Solr Schema*>>++// This pulls the sub-section list from the child page to reduce errors+include::solr-schema.adoc[tag=schema-sections]+****++****+[discrete]+=== Fields and Schema Design++<<fields-and-schema-design.adoc#,*Fields and Schema Design*>>++// This pulls the sub-section list from the child page to reduce errors+include::fields-and-schema-design.adoc[tag=fields-sections]+****++****+[discrete]+=== Document Analysis++<<document-analysis.adoc#,*Document Analysis in Solr*>>++// This pulls the sub-section list from the child page to reduce errors+include::document-analysis.adoc[tag=analysis-sections]+****++****+[discrete]+=== Indexing and Data Operations+<<indexing-data-operations.adoc#,*Indexing and Data Operations*>>++// This pulls the sub-section list from the child page to reduce errors+include::indexing-data-operations.adoc[tag=indexing-sections]+****diff --git a/solr/solr-ref-guide/src/schemaless-mode.adoc b/solr/solr-ref-guide/src/schemaless-mode.adocindex 1c60249612e..d62e9825ff0 100644--- a/solr/solr-ref-guide/src/schemaless-mode.adoc+++ b/solr/solr-ref-guide/src/schemaless-mode.adoc@@ -20,9 +20,9 @@ Schemaless Mode is a set of Solr features that, when used together, allow usersThese Solr features, all controlled via `solrconfig.xml`, are:-. Managed schema: Schema modifications are made at runtime through Solr APIs, which requires the use of a `schemaFactory` that supports these changes. See the section <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for more details.+. Managed schema: Schema modifications are made at runtime through Solr APIs, which requires the use of a `schemaFactory` that supports these changes. See the section <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for more details.. Field value class guessing: Previously unseen fields are run through a cascading set of value-based parsers, which guess the Java class of field values - parsers for Boolean, Integer, Long, Float, Double, and Date are currently available.-. Automatic schema field addition, based on field value class(es): Previously unseen fields are added to the schema, based on field value Java classes, which are mapped to schema field types - see <<solr-field-types.adoc#,Solr Field Types>>.+. Automatic schema field addition, based on field value class(es): Previously unseen fields are added to the schema, based on field value Java classes, which are mapped to schema field types - see <<field-types.adoc#,Field Types>>.== Using the Schemaless Example@@ -70,9 +70,9 @@ As described above, there are three configuration elements that need to be in pl=== Enable Managed Schema-As described in the section <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>>, Managed Schema support is enabled by default, unless your configuration specifies that `ClassicIndexSchemaFactory` should be used.+As described in the section <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>>, Managed Schema support is enabled by default, unless your configuration specifies that `ClassicIndexSchemaFactory` should be used.-You can configure the `ManagedIndexSchemaFactory` (and control the resource file used, or disable future modifications) by adding an explicit `<schemaFactory/>` like the one below, please see <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for more details on the options available.+You can configure the `ManagedIndexSchemaFactory` (and control the resource file used, or disable future modifications) by adding an explicit `<schemaFactory/>` like the one below, please see <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for more details on the options available.[source,xml]----@@ -180,7 +180,7 @@ Once the UpdateRequestProcessorChain has been defined, you must instruct your UpThere are two ways to do this. The update chain shown above has a `default=true` attribute which will use it for any update handler.-An alternative, more explicit way is to use <<initparams-in-solrconfig.adoc#,InitParams>> to set the defaults on all `/update` request handlers:+An alternative, more explicit way is to use <<initparams.adoc#,InitParams>> to set the defaults on all `/update` request handlers:[source,xml]----@@ -282,7 +282,7 @@ In addition string versions of the text fields are indexed, using copyFields to====Even if you want to use schemaless mode for most fields, you can still use the <<schema-api.adoc#,Schema API>> to pre-emptively create some fields, with explicit types, before you index documents that use them.-Internally, the Schema API and the Schemaless Update Processors both use the same <<schema-factory-definition-in-solrconfig.adoc#,Managed Schema>> functionality.+Internally, the Schema API and the Schemaless Update Processors both use the same <<schema-factory.adoc#,Managed Schema>> functionality.Also, if you do not need the `*_str` version of a text field, you can simply remove the `copyField` definition from the auto-generated schema and it will not be re-added since the original field is now defined.====diff --git a/solr/solr-ref-guide/src/searching-in-solr.adoc b/solr/solr-ref-guide/src/searching-in-solr.adocnew file mode 100644index 00000000000..c92bc54c393--- /dev/null+++ b/solr/solr-ref-guide/src/searching-in-solr.adoc@@ -0,0 +1,89 @@+= Searching in Solr+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Solr offers a rich, flexible set of features for search.+To understand the extent of this flexibility, it's helpful to begin with an overview of the steps and components involved in a Solr search.++When a user runs a search in Solr, the search query is processed by a _request handler_.+A request handler is a Solr plug-in that defines the logic to be used when Solr processes a request.+Solr supports a variety of request handlers.+Some are designed for processing search queries, while others manage tasks such as index replication.++Search applications select a particular request handler by default.+In addition, applications can be configured to allow users to override the default selection in preference of a different request handler.++To process a search query, a request handler calls a _query parser_, which interprets the terms and parameters of a query.+Different query parsers support different syntax.+Solr's default query parser is known as the <<standard-query-parser.adoc#,Standard Query Parser>>,or more commonly just the "lucene" query parser.+Solr also includes the <<dismax-query-parser.adoc#,DisMax>>query parser, and the <<edismax-query-parser.adoc#,Extended DisMax>> (eDisMax) query parser.++The <<standard-query-parser.adoc#,standard>> query parser's syntax allows for greater precision in searches, but the DisMax query parser is much more tolerant of errors.+The DisMax query parser is designed to provide an experience similar to that of popular search engines such as Google, which rarely display syntax errors to users.+The Extended DisMax query parser is an improved version of DisMax that handles the full Lucene query syntax while still tolerating syntax errors.+It also includes several additional features.++In addition, there are <<common-query-parameters.adoc#,common query parameters>> that are accepted by all query parsers.++Input to a query parser can include:++* search strings---that is, _terms_ to search for in the index+* _parameters for fine-tuning the query_ by increasing the importance of particular strings or fields, by applying Boolean logic among the search terms, or by excluding content from the search results+* _parameters for controlling the presentation of the query response_, such as specifying the order in which results are to be presented or limiting the response to particular fields of the search application's schema.++Search parameters may also specify a _filter query_.+As part of a search response, a filter query runs a query against the entire index and caches the results.+Because Solr allocates a separate cache for filter queries, the strategic use of filter queries can improve search performance.++[INFO]+Despite their similar names, query filters are not related to analysis filters.+Filter queries perform queries at search time against data already in the index, while analysis filters, such as tokenizers, parse content for indexing, following specified rules.++A search query can request that certain terms be highlighted in the search response; that is, the selected terms will be displayed in colored boxes so that they "jump out" on the screen of search results.+<<highlighting.adoc#,_Highlighting_>> can make it easier to find relevant passages in long documents returned in a search.+Solr supports multi-term highlighting.+Solr includes a rich set of search parameters for controlling how terms are highlighted.++Search responses can also be configured to include _snippets_ (document excerpts) featuring highlighted text.++To help users zero in on the content they're looking for, Solr supports two special ways of grouping search results to aid further exploration: faceting and clustering.++<<faceting.adoc#,*Faceting*>> is the arrangement of search results into categories (which are based on indexed terms).+Within each category, Solr reports on the number of hits for relevant term, which is called a facet constraint.+Faceting makes it easy for users to explore search results on sites such as movie sites and product review sites, where there are many categories and many items within a category.++The screen shot below shows an example of faceting from the CNET Web site (CBS Interactive Inc.), which was the first site to use Solr.++image::images/searching-in-solr/cnet-faceting.png[image,width=600,height=300]++Faceting makes use of fields defined when the search applications were indexed.+In the example above, these fields include categories of information that are useful for describing digital cameras: manufacturer, resolution, and zoom range.++*Clustering* groups search results by similarities discovered when a search is executed, rather than when content is indexed.+The results of clustering often lack the neat hierarchical organization found in faceted search results, but clustering can be useful nonetheless.+It can reveal unexpected commonalities among search results, and it can help users rule out content that isn't pertinent to what they're really searching for.++Solr also supports a feature called <<morelikethis.adoc#,MoreLikeThis>>, which enables users to submit new queries that focus on particular terms returned in an earlier query.+MoreLikeThis queries can make use of faceting or clustering to provide additional aid to users.++A Solr component called a <<response-writers.adoc#,*response writer*>> manages the final presentation of the query response.+Solr includes a variety of response writers, including an <<response-writers.adoc#standard-xml-response-writer,XML Response Writer>> and a <<response-writers.adoc#json-response-writer,JSON Response Writer>>.++The diagram below summarizes some key elements of the search process.++image::images/searching-in-solr/search-process.png[image,width=624,height=401]diff --git a/solr/solr-ref-guide/src/searching-nested-documents.adoc b/solr/solr-ref-guide/src/searching-nested-documents.adocindex 0dcc46960d8..983cc7db480 100644--- a/solr/solr-ref-guide/src/searching-nested-documents.adoc+++ b/solr/solr-ref-guide/src/searching-nested-documents.adoc@@ -16,15 +16,14 @@// specific language governing permissions and limitations// under the License.-This section exposes potential techniques which can be used for searching deeply nested documents,-show casing how more complex queries can be constructed using some of Solr's query parsers and Doc Transformers.-These features require `\_root_` and `\_nest_path_` to be declared in the schema. +-Please refer to the <<indexing-nested-documents.adoc#, Indexing Nested Documents>>-section for more details about schema and index configuration.+This section exposes potential techniques which can be used for searching deeply nested documents, showcasing how more complex queries can be constructed using some of Solr's query parsers and document transformers.++These features require `\_root_` and `\_nest_path_` to be declared in the schema.+Please refer to <<indexing-nested-documents.adoc#, Indexing Nested Documents>> for details about schema and index configuration.[NOTE]-This section does not show case faceting on nested documents. For nested document faceting, please refer to the-<<json-facet-api#uniqueblock-and-block-join-counts, Block Join Facet Counts>> section.+This section does not demonstrate faceting on nested documents.+For nested document faceting, please refer to the <<json-facet-api#uniqueblock-and-block-join-counts,Block Join Facet Counts>> section.== Query Examples@@ -34,13 +33,14 @@ include::indexing-nested-documents.adoc[tag=sample-indexing-deeply-nested-docume=== Child Doc Transformer-By default, documents that match a query do not include any of their nested children in the response. The `[child]` Doc Transformer Can be used enrich query results with the documents' descendants.+By default, documents that match a query do not include any of their nested children in the response.+The `[child]` Doc Transformer Can be used enrich query results with the documents' descendants.-For a detailed explanation of this transformer, and specifics on it's syntax & limitations, please refer to the section <<transforming-result-documents.adoc#child-childdoctransformerfactory, [child] - ChildDocTransformerFactory>>.+For a detailed explanation of this transformer, and specifics on it's syntax & limitations, please refer to the section <<document-transformers.adoc#child-childdoctransformerfactory, [child] - ChildDocTransformerFactory>>.A simple query matching all documents with a description that includes "staplers":-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=description_t:staplers'{@@ -53,9 +53,10 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=descr}}-----The same query with the addition of the `[child]` transformer is shown below. Note that the `numFound` has not changed, we are still matching the same set of documents, but when returning those documents the nested children are also returned as pseudo-fields.+The same query with the addition of the `[child]` transformer is shown below.+Note that the `numFound` has not changed, we are still matching the same set of documents, but when returning those documents the nested children are also returned as pseudo-fields.-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=description_t:staplers&fl=*,[child]'{@@ -103,11 +104,12 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=descr=== Child Query Parser-The `{!child}` query parser can be used to search for the _descendent_ documents of parent documents matching a wrapped query. For a detailed explanation of this parser, see the section <<other-parsers.adoc#block-join-children-query-parser, Block Join Children Query Parser>>.+The `{!child}` query parser can be used to search for the _descendent_ documents of parent documents matching a wrapped query.+For a detailed explanation of this parser, see the section <<block-join-query-parser.adoc#block-join-children-query-parser, Block Join Children Query Parser>>.Let's consider again the `description_t:staplers` query used above -- if we wrap that query in a `{!child}` query parser then instead of "matching" & returning the product level documents, we instead match all of the _descendent_ child documents of the original query:-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' -d 'q={!child of="*:* -_nest_path_:*"}description_t:staplers'{@@ -143,12 +145,12 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' -}}-----In this example we've used `\*:* -\_nest_path_:*` as our <<other-parsers.adoc#block-mask,`of` parameter>> to indicate we want to consider all documents which don't have a nest path -- i.e., all "root" level document -- as the set of possible parents.+In this example we've used `\*:* -\_nest_path_:*` as our <<block-join-query-parser.adoc#block-mask,`of` parameter>> to indicate we want to consider all documents which don't have a nest path -- i.e., all "root" level document -- as the set of possible parents.By changing the `of` parameter to match ancestors at specific `\_nest_path_` levels, we can narrow down the list of children we return.In the query below, we search for all descendants of `skus` (using an `of` parameter that identifies all documents that do _not_ have a `\_nest_path_` with the prefix `/skus/*`) with a `price_i` less then `50`:-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' --data-urlencode 'q={!child of="*:* -_nest_path_:\\/skus\\/*"}(+price_i:[* TO 50] +_nest_path_:\/skus)'{@@ -173,21 +175,22 @@ Note that in the above example, the `/` characters in the `\_nest_path_` were "d(You can see that only a single level of of `\` escaping is needed in the body of the query string -- to prevent the Regex syntax -- because it's not a quoted string local param).-You may find it more convenient to use <<local-parameters-in-queries#parameter-dereferencing,parameter references>> in conjunction with <<other-parsers#other-parsers,other parsers>> that do not treat `/` as a special character to express the same query in a more verbose form:+You may find it more convenient to use <<local-params#parameter-dereferencing,parameter references>> in conjunction with <<other-parsers#other-parsers,other parsers>> that do not treat `/` as a special character to express the same query in a more verbose form:-[source,curl]+[source,text]-----curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' --data-urlencode 'q={!child of=$block_mask}(+price_i:[* TO 50] +{!field f="_nest_path_" v="/skus"})' --data-urlencode 'block_mask=(*:* -{!prefix f="_nest_path_" v="/skus/"})'+$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' --data-urlencode 'q={!child of=$block_mask}(+price_i:[* TO 50] +{!field f="_nest_path_" v="/skus"})' --data-urlencode 'block_mask=(*:* -{!prefix f="_nest_path_" v="/skus/"})'----======= Parent Query Parser-The inverse of the `{!child}` query parser is the `{!parent}` query parser, which lets you search for the _ancestor_ documents of some child documents matching a wrapped query. For a detailed explanation of this parser, see the section <<other-parsers.adoc#block-join-parent-query-parser,Block Join Parent Query Parser>>.+The inverse of the `{!child}` query parser is the `{!parent}` query parser, which lets you search for the _ancestor_ documents of some child documents matching a wrapped query.+For a detailed explanation of this parser, see the section <<block-join-query-parser.adoc#block-join-parent-query-parser,Block Join Parent Query Parser>>.Let's first consider this example of searching for all "manual" type documents that have exactly `1` page:-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=pages_i:1'{@@ -215,7 +218,7 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select?omitHeader=true&q=pagesWe can wrap that query in a `{!parent}` query to return the details of all products that are ancestors of these manuals:-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' --data-urlencode 'q={!parent which="*:* -_nest_path_:*"}(+_nest_path_:\/skus\/manuals +pages_i:1)'{@@ -233,11 +236,12 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' -}}-----In this example we've used `\*:* -\_nest_path_:*` as our <<other-parsers#block-mask,`which` parameter>> to indicate we want to consider all documents which don't have a nest path -- i.e., all "root" level document -- as the set of possible parents.+In this example we've used `\*:* -\_nest_path_:*` as our <<block-join-query-parser#block-mask,`which` parameter>> to indicate we want to consider all documents which don't have a nest path -- i.e., all "root" level document -- as the set of possible parents.-By changing the `which` parameter to match ancestors at specific `\_nest_path_` levels, we can change the type of ancestors we return. In the query below, we search for `skus` (using an `which` parameter that identifies all documents that do _not_ have a `\_nest_path_` with the prefix `/skus/*`) that are the ancestors of `manuals` with exactly `1` page:+By changing the `which` parameter to match ancestors at specific `\_nest_path_` levels, we can change the type of ancestors we return.+In the query below, we search for `skus` (using an `which` parameter that identifies all documents that do _not_ have a `\_nest_path_` with the prefix `/skus/*`) that are the ancestors of `manuals` with exactly `1` page:-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' --data-urlencode 'q={!parent which="*:* -_nest_path_:\\/skus\\/*"}(+_nest_path_:\/skus\/manuals +pages_i:1)'{@@ -272,7 +276,7 @@ Here for example is a query where:*** "lifetime guarantee" in their content* each return (sku) document also includes any descendent (manuals) documents it has-[source,curl]+[source,text]----$ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' -d 'fq=color_s:RED' --data-urlencode 'q={!child of="*:* -_nest_path_:*" filters=$parent_fq}' --data-urlencode 'parent_fq={!parent which="*:* -_nest_path_:*"}(+_nest_path_:"/manuals" +content_t:"lifetime guarantee")' -d 'fl=*,[child]'{diff --git a/solr/solr-ref-guide/src/searching.adoc b/solr/solr-ref-guide/src/searching.adocdeleted file mode 100644index 5fb71be77c1..00000000000--- a/solr/solr-ref-guide/src/searching.adoc+++ /dev/null@@ -1,88 +0,0 @@-= Searching-:page-children: overview-of-searching-in-solr, +- relevance, +- query-syntax-and-parsing, +- json-request-api, +- json-facet-api, +- faceting, +- highlighting, +- spell-checking, +- query-re-ranking, +- transforming-result-documents, +- searching-nested-documents, +- suggester, +- morelikethis, +- pagination-of-results, +- collapse-and-expand-results, +- result-grouping, +- result-clustering, +- spatial-search, +- the-terms-component, +- the-term-vector-component, +- the-stats-component, +- the-query-elevation-component, +- the-tagger-handler, +- response-writers, +- near-real-time-searching, +- realtime-get, +- exporting-result-sets, +- parallel-sql-interface, +- analytics--// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section describes how Solr works with search requests. It covers the following topics:--* <<overview-of-searching-in-solr.adoc#,Overview of Searching in Solr>>: An introduction to searching with Solr.-* <<relevance.adoc#,Relevance>>: Conceptual information about understanding relevance in search results.-* <<query-syntax-and-parsing.adoc#,Query Syntax and Parsing>>: A brief conceptual overview of query syntax and parsing. It also contains the following sub-sections:-** <<common-query-parameters.adoc#,Common Query Parameters>>: No matter the query parser, there are several parameters that are common to all of them.-** <<the-standard-query-parser.adoc#,The Standard Query Parser>>: Detailed information about the standard Lucene query parser.-** <<the-dismax-query-parser.adoc#,The DisMax Query Parser>>: Detailed information about Solr's DisMax query parser.-** <<the-extended-dismax-query-parser.adoc#,The Extended DisMax Query Parser>>: Detailed information about Solr's Extended DisMax (eDisMax) Query Parser.-** <<function-queries.adoc#,Function Queries>>: Detailed information about parameters for generating relevancy scores using values from one or more numeric fields.-** <<local-parameters-in-queries.adoc#,Local Parameters in Queries>>: How to add local arguments to queries.-** <<other-parsers.adoc#,Other Parsers>>: More parsers designed for use in specific situations.-* <<json-request-api.adoc#,JSON Request API>>: Overview of Solr's JSON Request API.-** <<json-query-dsl.adoc#,JSON Query DSL>>: Detailed information about a simple yet powerful query language for JSON Request API.-* <<json-facet-api.adoc#facet-analytics-module,JSON Facet API>>: Overview of Solr's JSON Facet API.-* <<faceting.adoc#,Faceting>>: Detailed information about categorizing search results based on indexed terms.-* <<highlighting.adoc#,Highlighting>>: Detailed information about Solr's highlighting capabilities, including multiple underlying highlighter implementations.-* <<spell-checking.adoc#,Spell Checking>>: Detailed information about Solr's spelling checker.-* <<query-re-ranking.adoc#,Query Re-Ranking>>: Detailed information about re-ranking top scoring documents from simple queries using more complex scores.-** <<learning-to-rank.adoc#,Learning To Rank>>: How to use LTR to run machine learned ranking models in Solr.--* <<transforming-result-documents.adoc#,Transforming Result Documents>>: Detailed information about using `DocTransformers` to add computed information to individual documents-* <<searching-nested-documents.adoc#,Searching Nested Documents>>: Detailed information about constructing nested and hierarchical queries.-* <<suggester.adoc#,Suggester>>: Detailed information about Solr's powerful autosuggest component.-* <<morelikethis.adoc#,MoreLikeThis>>: Detailed information about Solr's similar results query component.-* <<pagination-of-results.adoc#,Pagination of Results>>: Detailed information about fetching paginated results for display in a UI, or for fetching all documents matching a query.-* <<result-grouping.adoc#,Result Grouping>>: Detailed information about grouping results based on common field values.-* <<result-clustering.adoc#,Result Clustering>>: Detailed information about grouping search results based on cluster analysis applied to text fields. A bit like "unsupervised" faceting.-* <<spatial-search.adoc#,Spatial Search>>: How to use Solr's spatial search capabilities.-* <<the-terms-component.adoc#,The Terms Component>>: Detailed information about accessing indexed terms and the documents that include them.-* <<the-term-vector-component.adoc#,The Term Vector Component>>: How to get term information about specific documents.-* <<the-stats-component.adoc#,The Stats Component>>: How to return information from numeric fields within a document set.-* <<the-query-elevation-component.adoc#,The Query Elevation Component>>: How to force documents to the top of the results for certain queries.-* <<the-tagger-handler.adoc#,The Tagger Handler>>: The SolrTextTagger, for basic named entity tagging in text.-* <<response-writers.adoc#,Response Writers>>: Detailed information about configuring and using Solr's response writers.-* <<near-real-time-searching.adoc#,Near Real Time Searching>>: How to include documents in search results nearly immediately after they are indexed.-* <<realtime-get.adoc#,RealTime Get>>: How to get the latest version of a document without opening a searcher.-* <<exporting-result-sets.adoc#,Exporting Result Sets>>: Functionality to export large result sets out of Solr.-* <<parallel-sql-interface.adoc#,Parallel SQL Interface>>: An interface for sending SQL statements to Solr, and using advanced parallel query processing and relational algebra for complex data analysis.-* <<analytics.adoc#,The Analytics Component>>: A framework to compute complex analytics over a result set.diff --git a/solr/solr-ref-guide/src/securing-solr.adoc b/solr/solr-ref-guide/src/securing-solr.adocindex 9d8807c5afb..2499fc12d90 100644--- a/solr/solr-ref-guide/src/securing-solr.adoc+++ b/solr/solr-ref-guide/src/securing-solr.adoc@@ -1,5 +1,8 @@= Securing Solr-:page-children: authentication-and-authorization-plugins, enabling-ssl, zookeeper-access-control+:page-children: authentication-and-authorization-plugins, \+ audit-logging, \+ enabling-ssl, \+ zookeeper-access-control// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -19,52 +22,67 @@[WARNING]====-No Solr API, including the Admin UI, is designed to be exposed to non-trusted parties. Tune your firewall so that only trusted computers and people are allowed access. Because of this, the project will not regard e.g., Admin UI XSS issues as security vulnerabilities. However, we still ask you to report such issues in JIRA.+No Solr API, including the Admin UI, is designed to be exposed to non-trusted parties.+Tune your firewall so that only trusted computers and people are allowed access.+Because of this, the project will not regard e.g., Admin UI XSS issues as security vulnerabilities.+However, we still ask you to report such issues in JIRA.====When planning how to secure Solr, you should consider which of the available features or approaches are right for you:== Encryption with TLS (SSL) Certificates-Encrypting traffic to/from Solr and between Solr nodes prevents sensitive data to be leaked out on the network. TLS is also normally a requirement to prevent credential sniffing when using Authentication.+Encrypting traffic to/from Solr and between Solr nodes prevents sensitive data to be leaked out on the network.+TLS is also normally a requirement to prevent credential sniffing when using Authentication.-See the page <<enabling-ssl.adoc#,Enabling TLS (SSL)>> for details.+See the section <<enabling-ssl.adoc#,Enabling TLS (SSL)>> for details.-== Authentication, Authorization and Audit Logging+== Authentication and Authorization-See chapter <<authentication-and-authorization-plugins.adoc#,Configuring Authentication, Authorization and Audit Logging plugins>> to learn how to work with the `security.json` file.+See chapter <<authentication-and-authorization-plugins.adoc#,Configuring Authentication and Authorization>> to learn how to work with the `security.json` file.[#securing-solr-auth-plugins]=== Authentication Plugins-Authentication makes sure you know the identity of your users. The authentication plugins shipping with Solr are:+Authentication makes sure you know the identity of your users.+The authentication plugins that ship with Solr are:// tag::list-of-authentication-plugins[]-* <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>>-* <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>>-* <<hadoop-authentication-plugin.adoc#,Hadoop Authentication Plugin>>-* <<jwt-authentication-plugin.adoc#,JWT Authentication Plugin>>-* <<cert-authentication-plugin.adoc#,Certificate Authentication Plugin>>+[width=100%,cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>>+| <<kerberos-authentication-plugin.adoc#,Kerberos Authentication Plugin>>+| <<jwt-authentication-plugin.adoc#,JWT Authentication Plugin>>+| <<cert-authentication-plugin.adoc#,Certificate Authentication Plugin>>+| <<hadoop-authentication-plugin.adoc#,Hadoop Authentication Plugin>>+|+|===// end::list-of-authentication-plugins[]=== Authorization Plugins-Authorization makes sure that only users with the necessary roles/permissions can access any given resource. The authorization plugins shipping with Solr are:+Authorization makes sure that only users with the necessary roles/permissions can access any given resource.+The authorization plugins that ship with Solr are:// tag::list-of-authorization-plugins[]-* <<rule-based-authorization-plugin.adoc#,Rule-Based Authorization Plugin>>-* <<rule-based-authorization-plugin.adoc#,External Role Rule-Based Authorization Plugin>>+[width=100%,cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<rule-based-authorization-plugin.adoc#,Rule-Based Authorization Plugin>>+| <<rule-based-authorization-plugin.adoc#,External Role Rule-Based Authorization Plugin>>+|===// end::list-of-authorization-plugins[]-=== Audit Logging Plugins+== Audit Logging-Audit logging will record an audit trail of incoming reqests to your cluster, such as users being denied access to admin APIs. Learn more about audit logging and how to implement an audit logger plugin here in the <<audit-logging.adoc#,Audit Logging>> chapter.+Audit logging will record an audit trail of incoming reqests to your cluster, such as users being denied access to admin APIs.+Learn more about audit logging and how to implement an audit logger plugin in the section <<audit-logging.adoc#,Audit Logging>>.== Request Logging-Solr can optionally log every incoming HTTP(s) request in the standard https://en.wikipedia.org/wiki/Common_Log_Format[`NCSA format`]. You can enable request logging by setting `SOLR_REQUESTLOG_ENABLED=true` via environment variable or in `solr.in.sh`/`solr.in.cmd`.+Solr can optionally log every incoming HTTP(s) request in the standard https://en.wikipedia.org/wiki/Common_Log_Format[`NCSA format`].+You can enable request logging by setting `SOLR_REQUESTLOG_ENABLED=true` via environment variable or in `solr.in.sh`/`solr.in.cmd`.-== Enable IP Access Control+== IP Access ControlRestrict network access to specific hosts, by setting `SOLR_IP_WHITELIST`/`SOLR_IP_BLACKLIST` via environment variables or in `solr.in.sh`/`solr.in.cmd`.@@ -79,20 +97,24 @@ SOLR_IP_BLACKLIST="192.168.0.3, 192.168.0.4"== Securing ZooKeeper TrafficZooKeeper is a central and important part of a SolrCloud cluster and understanding how to secure-its content is covered in the <<zookeeper-access-control.adoc#,ZooKeeper Access Control>> page.+its content is covered in the section <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>.== Network Configuration// tag::security-network-binding-1[]-Administrators should consider their security setup carefully as an important step in moving to production. Solr provides a number of features out of the box to meet the security needs of users: authentication and authorization can be configured using a range of security plugins, privacy can be bolstered by enabling SSL/TLS, and (in SolrCloud) ZooKeeper data can be protected with ACL rules to prevent unauthorized reads and writes.+Administrators should consider their security setup carefully as an important step in moving to production.+Solr provides a number of features out of the box to meet the security needs of users: authentication and authorization can be configured using a range of security plugins, privacy can be bolstered by enabling SSL/TLS, and (in SolrCloud) ZooKeeper data can be protected with ACL rules to prevent unauthorized reads and writes.-Even if these measures or others are taken, it is strongly recommended that Solr always be protected by a firewall. Solr is not designed to be exposed on the open internet.+Even if these measures or others are taken, it is strongly recommended that Solr always be protected by a firewall.+Solr is not designed to be exposed on the open internet.-It is also strongly recommended that Solr listen to only those network interfaces that are strictly required. To prevent administrators from unintentionally exposing Solr more broadly, Solr only listens on the loopback interface ("127.0.0.1") by default. Most deployments will need to change this value to something less restrictive so that it can be reached from other boxes. This can be done by setting a `SOLR_JETTY_HOST` value in your environment's "include script" (`solr.in.sh` or `solr.in.cmd`):+It is also strongly recommended that Solr listen to only those network interfaces that are strictly required.+To prevent administrators from unintentionally exposing Solr more broadly, Solr only listens on the loopback interface ("127.0.0.1") by default.+Most deployments will need to change this value to something less restrictive so that it can be reached from other boxes.+This can be done by setting a `SOLR_JETTY_HOST` value in your environment's "include script" (`solr.in.sh` or `solr.in.cmd`):[source,bash]----SOLR_JETTY_HOST="0.0.0.0"----// end::security-network-binding-1[]-diff --git a/solr/solr-ref-guide/src/segments-info.adoc b/solr/solr-ref-guide/src/segments-info.adocdeleted file mode 100644index 8e38dcbd4c1..00000000000--- a/solr/solr-ref-guide/src/segments-info.adoc+++ /dev/null@@ -1,23 +0,0 @@-= Segments Info-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Segments Info screen lets you see a visualization of the various segments in the underlying Lucene index for this core, with information about the size of each segment – both bytes and in number of documents – as well as other basic metadata about those segments. Most visible is the the number of deleted documents, but you can hover your mouse over the segments to see additional numeric details.--image::images/segments-info/segments_info.png[image,width=486,height=250]--This information may be useful for people to help make decisions about the optimal <<indexconfig-in-solrconfig.adoc#merging-index-segments,merge settings>> for their data.diff --git a/solr/solr-ref-guide/src/shard-management.adoc b/solr/solr-ref-guide/src/shard-management.adocindex 2d5da7ce0a2..d8f7e0f9091 100644--- a/solr/solr-ref-guide/src/shard-management.adoc+++ b/solr/solr-ref-guide/src/shard-management.adoc@@ -142,7 +142,7 @@ A float value (default is 0.0f, must be smaller than 0.5f) that allows to vary tby this percentage of total shard range, odd shards being larger and even shards being smaller.`property._name_=_value_`::-Set core property _name_ to _value_. See the section <<defining-core-properties.adoc#,Defining core.properties>> for details on supported properties and values.+Set core property _name_ to _value_. See the section <<core-discovery.adoc#,Core Discovery>> for details on supported properties and values.`waitForFinalState`::If `true`, the request will complete only when all affected replicas become active. The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.@@ -274,7 +274,7 @@ The number of `tlog` replicas that should be created for the new shard (optionalThe number of `pull` replicas that should be created for the new shard (optional, the defaults for the collection is used if omitted)`property._name_=_value_`::-Set core property _name_ to _value_. See the section <<defining-core-properties.adoc#,Defining core.properties>> for details on supported properties and values.+Set core property _name_ to _value_. See the section <<core-discovery.adoc#,Core Discovery>> for details on supported properties and values.`waitForFinalState`::If `true`, the request will complete only when all affected replicas become active. The default is `false`, which means that the API will return the status of the single action, which may be before the new replica is online and active.diff --git a/solr/solr-ref-guide/src/solr-admin-ui.adoc b/solr/solr-ref-guide/src/solr-admin-ui.adocnew file mode 100644index 00000000000..a02ec19e69c--- /dev/null+++ b/solr/solr-ref-guide/src/solr-admin-ui.adoc@@ -0,0 +1,167 @@+= Solr Admin UI+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Solr features a Web interface that provides online access to many Solr configuration options and features.++== Overview of the Admin UI++=== Dashboard++Accessing the URL `\http://hostname:8983/solr/` will show the main dashboard, which is divided into two parts.++.Solr Dashboard+image::images/solr-admin-ui/dashboard.png[image,height=400]++The left-side of the screen is a menu under the Solr logo that provides the navigation through the screens of the UI.++The first set of links are for system-level information and configuration and provide access to <<configuring-logging.adoc#logging-screen,Logging Screen>>, <<collections-core-admin.adoc#,Collection/Core Administration>>, and <<jvm-settings.adoc#java-properties-screen,Java Properties Screen>>, among other things.++At the end of this information is at least one pulldown listing Solr cores configured for this instance.+On <<cluster-types.adoc#solrcloud-mode,SolrCloud>> nodes, an additional pulldown list shows all collections in this cluster.+Clicking on a collection or core name shows secondary menus of information for the specified collection or core, such as a <<schema-browser-screen.adoc#,Schema Browser>>, <<configuration-files.adoc#files-screen,Files Screen>>, <<plugins-stats-screen.adoc#,Plugins & Statistics>>, and an ability to perform <<query-screen.adoc#,Queries>> on indexed data.++The left-side navigation appears on every screen, while the center changes to the detail of the option selected.+The Dashboard shows several information items in the center of the screen, including system uptime, the version being run, system-level data, JVM arguments, and the security plugins enabled (if any).++Under the covers, the Solr Admin UI uses the same HTTP APIs available to all clients to access Solr-related data to drive an external interface.++[TIP]+====+The path to the Solr Admin UI given above is `\http://hostname:port/solr`, which redirects to `\http://hostname:port/solr/\#/`. A convenience redirect is also supported, so simply accessing the Admin UI at `\http://hostname:port/` will also redirect to `\http://hostname:port/solr/#/`.+====++=== Login Screen++If authentication has been enabled, Solr will present a login screen to unauthenticated users before allowing them further access to the Admin UI.++.Login Screen+image::images/solr-admin-ui/login.png[]++This login screen currently only works with Basic Authentication.+See the section <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>> for+ details on how to configure Solr to use this method of authentication.++Once logged in, the left-hand navigation will show the current user with an option to logout.++If Kerberos is enabled and the user has a valid ticket, the login screen will be skipped.+However, if the user does not have a valid ticket, they will see a message that they need to obtain a valid ticket before continuing.++=== Getting Assistance++At the bottom of each screen of the Admin UI is a set of links that can be used to get more assistance with configuring and using Solr.++.Assistance icons+image::images/solr-admin-ui/Assistance.png[Assistance icons]++These icons include the following links.++[%autowidth.stretch,options="header"]+|===+|Link |Description+|Documentation |Navigates to the Apache Solr documentation hosted on https://solr.apache.org/.+|Issue Tracker |Navigates to the JIRA issue tracking server for the Apache Solr project.+This server resides at https://issues.apache.org/jira/browse/SOLR.+|IRC Channel |Navigates to Solr's http://en.wikipedia.org/wiki/Internet_Relay_Chat[IRC] live-chat room: http://webchat.freenode.net/?channels=#solr.+|Community forum |Navigates to the Apache Wiki page which has further information about ways to engage in the Solr User community mailing lists: https://cwiki.apache.org/confluence/display/solr/UsingMailingLists.+|Solr Query Syntax |Navigates to the section <<query-syntax-and-parsers.adoc#,Query Syntax and Parsing>> in this Reference Guide.+|===++These links cannot be modified without editing the `index.html` in the `server/solr/solr-webapp` directory that contains the Admin UI files.++== Collection-Specific Tools++In the left-hand navigation bar, you will see a pull-down menu titled Collection Selector that can be used to access collection specific administration screens.++.Only Visible When Using SolrCloud+[NOTE]+====+The Collection Selector pull-down menu is only available on Solr instances running <<cluster-types.adoc#solrcloud-mode,SolrCloud>>.++User-managed clusters or single-node installations will not display this menu, instead the Collection specific UI pages described in this section will be available in the <<Core-Specific Tools,Core Selector pull-down menu>>.+====++Clicking on the Collection Selector pull-down menu will show a list of the collections in your Solr cluster, with a search box that can be used to find a specific collection by name.+When a collection is selected, the main body of the page will display some metadata about the collection.+A secondary menu will appear in the left nav with links to additional collection specific administration screens.++image::images/solr-admin-ui/collection_dashboard.png[image,width=482,height=250]++== Core-Specific Tools++The Core-Specific tools are a group of UI screens that allow you to see core-level information.++In the left-hand navigation bar, you will see a pull-down menu titled Core Selector.+Clicking on the menu will show a list of Solr cores hosted on this Solr node, with a search box that can be used to find a specific core by name.++When you select a core from the pull-down, the main display of the page will show some basic metadata about the core, and a secondary menu will appear in the left nav with links to additional core specific administration screens.++.Core overview screen+image::images/solr-admin-ui/core_dashboard.png[image,width=515,height=250]++== Links to UI Documentation+Here are sections throughout the Guide describing each screen of the Admin UI:++*Primary UI Tools*++****+// This tags the below list so it can be used in the parent page section list+// tag::ui-common-tools[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<configuring-logging.adoc#logging-screen,Logging Screen>>: Recent log messages and configuration of log levels.+| <<cloud-screens.adoc#,Cloud Screens>>: Access to SolrCloud node data and status.+| <<collections-core-admin.adoc#,Collections / Core Admin>>: Collection or Core management tools.+| <<jvm-settings.adoc#java-properties-screen,Java Properties Screen>>: Java information for each core.+| <<thread-dump.adoc#,Thread Dump Screen>>: Detailed thread and state information.+|+|===+// end::ui-common-tools[]+****++*Collection-Specific Tools*++****+// This tags the below list so it can be used in the parent page section list+// tag::ui-collection-tools[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<analysis-screen.adoc#,Analysis Screen>>: Test and validation tool for field type analyzers.+| <<documents-screen.adoc#,Documents Screen>>: Form-based document updates using the Admin UI.+| <<configuration-files.adoc#files-screen,Files Screen>>: Configuration file browser.+| <<query-screen.adoc#,Query Screen>>: Form-based query builder.+| <<stream-screen.adoc#,Stream Screen>>: Submit streaming expressions and see results and parsing explanations.+| <<schema-browser-screen.adoc#,Schema Browser Screen>>: Field-level schema details.+|===+// end::ui-collection-tools[]+****++*Core-Specific Tools*++****+// This tags the below list so it can be used in the parent page section list+// tag::ui-core-tools[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<ping.adoc#,Ping>>: Ping a named core to determine whether it is active.+| <<plugins-stats-screen.adoc#,Plugins/Stats Screen>>: Statistics for request handlers, search components, plugins, and other installed components.+| <<user-managed-index-replication.adoc#replication-screen,Replication Screen>>: Enable replication for a core and view current replication status.+| <<index-segments-merging.adoc#segments-info-screen,Segments Info Screen>>: Visualization of the underlying Lucene index segments.+|===+// end::ui-core-tools[]+****diff --git a/solr/solr-ref-guide/src/solr-concepts.adoc b/solr/solr-ref-guide/src/solr-concepts.adocnew file mode 100644index 00000000000..3daddd5ec09--- /dev/null+++ b/solr/solr-ref-guide/src/solr-concepts.adoc@@ -0,0 +1,40 @@+= Solr Concepts+:page-children: documents-fields-schema-design, \+ solr-indexing, \+ searching-in-solr, \+ relevance, \+ solr-glossary+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+This section covers essential information retrieval concepts and an introduction to how they are implemented in Solr.++****+// This tags the below list so it can be used in the parent page section list+// tag::concept-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<documents-fields-schema-design.adoc#,Documents, Fields, and Schema Design>>: An introduction to how Solr breaks documents into fields.+| <<solr-indexing.adoc#,Solr Indexing>>: The process of adding documents or other data to Solr for later searching.+| <<searching-in-solr.adoc#,Searching in Solr>>: The basic building blocks of asking Solr questions.+| <<relevance.adoc#,Relevance>>: A beginner's guide to understanding what relevance means in the search context.+| <<solr-glossary.adoc#,Solr Glossary>>: Commonly used Solr terminology.+|+|===+// end::concept-sections[]+****diff --git a/solr/solr-ref-guide/src/solr-configuration-files.adoc b/solr/solr-ref-guide/src/solr-configuration-files.adocdeleted file mode 100644index 862bfc48cc4..00000000000--- a/solr/solr-ref-guide/src/solr-configuration-files.adoc+++ /dev/null@@ -1,78 +0,0 @@-= Solr Configuration Files-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr has several configuration files that you will interact with during your implementation.--Many of these files are in XML format, although APIs that interact with configuration settings tend to accept JSON for programmatic access as needed.--== Solr Home-When Solr runs, it needs access to a home directory.--When you first install Solr, your home directory is `server/solr`. However, some examples may change this location (such as, if you run `bin/solr start -e cloud`, your home directory will be `example/cloud`).--The home directory contains important configuration information and is the place where Solr will store its index. The layout of the home directory will look a little different when you are running Solr in standalone mode vs. when you are running in SolrCloud mode.--The crucial parts of the Solr home directory are shown in these examples:--.Standalone Mode-[source,plain]------<solr-home-directory>/- solr.xml- core_name1/- core.properties- conf/- solrconfig.xml- managed-schema- data/- core_name2/- core.properties- conf/- solrconfig.xml- managed-schema- data/-------.SolrCloud Mode-[source,plain]------<solr-home-directory>/- solr.xml- core_name1/- core.properties- data/- core_name2/- core.properties- data/-------You may see other files, but the main ones you need to know are discussed in the next section.--== Configuration Files-Inside Solr's Home, you'll find these files:--* `solr.xml` specifies configuration options for your Solr server instance. For more information on `solr.xml` see <<solr-cores-and-solr-xml.adoc#,Solr Cores and solr.xml>>.-* Per Solr Core:-** `core.properties` defines specific properties for each core such as its name, the collection the core belongs to, the location of the schema, and other parameters. For more details on `core.properties`, see the section <<defining-core-properties.adoc#,Defining core.properties>>.-** `solrconfig.xml` controls high-level behavior. You can, for example, specify an alternate location for the data directory. For more information on `solrconfig.xml`, see <<configuring-solrconfig-xml.adoc#,Configuring solrconfig.xml>>.-** `managed-schema` (or `schema.xml` instead) describes the documents you will ask Solr to index. The Schema define a document as a collection of fields. You get to define both the field types and the fields themselves. Field type definitions are powerful and include information about how Solr processes incoming field values and query values. For more information on Solr Schemas, see <<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>> and the <<schema-api.adoc#,Schema API>>.-** `data/` The directory containing the low level index files.--Note that the SolrCloud example does not include a `conf` directory for each Solr Core (so there is no `solrconfig.xml` or Schema file). This is because the configuration files usually found in the `conf` directory are stored in ZooKeeper so they can be propagated across the cluster.--If you are using SolrCloud with the embedded ZooKeeper instance, you may also see `zoo.cfg` and `zoo.data` which are ZooKeeper configuration and data files. However, if you are running your own ZooKeeper ensemble, you would supply your own ZooKeeper configuration file when you start it and the copies in Solr would be unused. For more information about SolrCloud, see the section <<solrcloud.adoc#,SolrCloud>>.diff --git a/solr/solr-ref-guide/src/solr-control-script-reference.adoc b/solr/solr-ref-guide/src/solr-control-script-reference.adocindex 474a01bd407..c9be84a53bd 100644--- a/solr/solr-ref-guide/src/solr-control-script-reference.adoc+++ b/solr/solr-ref-guide/src/solr-control-script-reference.adoc@@ -20,15 +20,17 @@ Solr includes a script known as "`bin/solr`" that allows you to perform many comYou can start and stop Solr, create and delete collections or cores, perform operations on ZooKeeper and check the status of Solr and configured shards.-You can find the script in the `bin/` directory of your Solr installation. The `bin/solr` script makes Solr easier to work with by providing simple commands and options to quickly accomplish common goals.+You can find the script in the `bin/` directory of your Solr installation.+The `bin/solr` script makes Solr easier to work with by providing simple commands and options to quickly accomplish common goals.-More examples of `bin/solr` in use are available throughout the Solr Reference Guide, but particularly in the sections <<installing-solr.adoc#starting-solr,Starting Solr>> and <<getting-started-with-solrcloud.adoc#,Getting Started with SolrCloud>>.+More examples of `bin/solr` in use are available throughout this Guide, but particularly in the sections <<installing-solr.adoc#starting-solr,Starting Solr>> and <<tutorial-solrcloud.adoc#,Getting Started with SolrCloud>>.== Starting and Stopping=== Start and Restart-The `start` command starts Solr. The `restart` command allows you to restart Solr while it is already running or if it has been stopped already.+The `start` command starts Solr.+The `restart` command allows you to restart Solr while it is already running or if it has been stopped already.The `start` and `restart` commands have several options to allow you to run in SolrCloud mode, use an example configuration set, start with a hostname or port that is not the default and point to a local ZooKeeper ensemble.@@ -40,38 +42,45 @@ The `start` and `restart` commands have several options to allow you to run in S`bin/solr restart -help`-When using the `restart` command, you must pass all of the parameters you initially passed when you started Solr. Behind the scenes, a stop request is initiated, so Solr will be stopped before being started again. If no nodes are already running, restart will skip the step to stop and proceed to starting Solr.+When using the `restart` command, you must pass all of the parameters you initially passed when you started Solr.+Behind the scenes, a stop request is initiated, so Solr will be stopped before being started again.+If no nodes are already running, restart will skip the step to stop and proceed to starting Solr.==== Start Parameters-The `bin/solr` script provides many options to allow you to customize the server in common ways, such as changing the listening port. However, most of the defaults are adequate for most Solr installations, especially when just getting started.+The `bin/solr` script provides many options to allow you to customize the server in common ways, such as changing the listening port.+However, most of the defaults are adequate for most Solr installations, especially when just getting started.`-a "<string>"`::-Start Solr with additional JVM parameters, such as those starting with -X. If you are passing JVM parameters that begin with "-D", you can omit the -a option.+Start Solr with additional JVM parameters, such as those starting with `-X`.+If you are passing JVM parameters that begin with `-D`, you can omit the `-a` option.*Example*:[source,bash]bin/solr start -a "-Xdebug -Xrunjdwp:transport=dt_socket, server=y,suspend=n,address=1044"-`-cloud`::+`-cloud` or `-c`::Start Solr in SolrCloud mode, which will also launch the embedded ZooKeeper instance included with Solr.+This option can be shortened to simply `-c`.+-If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) or pass the -z parameter.+If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) or pass the -z parameter.+For more details, see the section <<SolrCloud Mode>> below.+*Example*: `bin/solr start -c``-d <dir>`::-Define a server directory, defaults to `server` (as in, `$SOLR_HOME/server`). It is uncommon to override this option. When running multiple instances of Solr on the same host, it is more common to use the same server directory for each instance and use a unique Solr home directory using the -s option.+Define a server directory, defaults to `server` (as in, `$SOLR_HOME/server`).+It is uncommon to override this option.+When running multiple instances of Solr on the same host, it is more common to use the same server directory for each instance and use a unique Solr home directory using the -s option.+*Example*: `bin/solr start -d newServerDir``-e <name>`::-Start Solr with an example configuration. These examples are provided to help you get started faster with Solr generally, or just try a specific feature.+Start Solr with an example configuration.+These examples are provided to help you get started faster with Solr generally, or just try a specific feature.+The available options are:@@ -84,48 +93,59 @@ See the section <<Running with Example Configurations>> below for more details o*Example*: `bin/solr start -e schemaless``-f`::-Start Solr in the foreground; you cannot use this option when running examples with the -e option.+Start Solr in the foreground.+You cannot use this option when running examples with the `-e` option.+*Example*: `bin/solr start -f``-h <hostname>`::-Start Solr with the defined hostname. If this is not specified, 'localhost' will be assumed.+Start Solr with the defined hostname.+If this is not specified, `localhost` will be assumed.+*Example*: `bin/solr start -h search.mysolr.com``-m <memory>`::-Start Solr with the defined value as the min (-Xms) and max (-Xmx) heap size for the JVM.+Start Solr with the defined value as the min (`-Xms`) and max (`-Xmx`) heap size for the JVM.+*Example*: `bin/solr start -m 1g``-noprompt`::-Start Solr and suppress any prompts that may be seen with another option. This would have the side effect of accepting all defaults implicitly.+Start Solr and suppress any prompts that may be seen with another option.+This would have the side effect of accepting all defaults implicitly.+-For example, when using the "cloud" example, an interactive session guides you through several options for your SolrCloud cluster. If you want to accept all of the defaults, you can simply add the -noprompt option to your request.+For example, when using the "cloud" example, an interactive session guides you through several options for your SolrCloud cluster.+If you want to accept all of the defaults, you can simply add the `-noprompt` option to your request.+*Example*: `bin/solr start -e cloud -noprompt``-p <port>`::-Start Solr on the defined port. If this is not specified, '8983' will be used.+Start Solr on the defined port.+If this is not specified, `8983` will be used.+*Example*: `bin/solr start -p 8655``-s <dir>`::-Sets the `solr.solr.home` system property; Solr will create core directories under this directory. This allows you to run multiple Solr instances on the same host while reusing the same server directory set using the `-d` parameter.+Sets the `solr.solr.home` system property.+Solr will create core directories under this directory.+This allows you to run multiple Solr instances on the same host while reusing the same server directory set using the `-d` parameter.+-If set, the specified directory should contain a `solr.xml` file, unless `solr.xml` exists in ZooKeeper. The default value is `server/solr`.+If set, the specified directory should contain a `solr.xml` file, unless `solr.xml` exists in ZooKeeper.+The default value is `server/solr`.+This parameter is ignored when running examples (`-e`), as the `solr.solr.home` depends on which example is run.+*Example*: `bin/solr start -s newHome``-v`::-Be more verbose. This changes the logging level of log4j from `INFO` to `DEBUG`, having the same effect as if you edited `log4j2.xml` accordingly.+Be more verbose.+This changes the logging level of Log4j from `INFO` to `DEBUG`, having the same effect as if you edited `log4j2.xml`.+*Example*: `bin/solr start -f -v``-q`::-Be more quiet. This changes the logging level of log4j from `INFO` to `WARN`, having the same effect as if you edited `log4j2.xml` accordingly. This can be useful in a production setting where you want to limit logging to warnings and errors.+Be more quiet.+This changes the logging level of Log4j from `INFO` to `WARN`, having the same effect as if you edited `log4j2.xml`.+This can be useful in a production setting where you want to limit logging to warnings and errors.+*Example*: `bin/solr start -f -q`@@ -135,12 +155,16 @@ Start Solr with verbose messages from the start script.*Example*: `bin/solr start -V``-z <zkHost>`::-Start Solr with the defined ZooKeeper connection string. This option is only used with the -c option, to start Solr in SolrCloud mode. If `ZK_HOST` is not specified in `solr.in.sh`/`solr.in.cmd` and this option is not provided, Solr will start the embedded ZooKeeper instance and use that instance for SolrCloud operations.+Start Solr with the defined ZooKeeper connection string.+++This option is only used with the `-c` option, to start Solr in SolrCloud mode.+If `ZK_HOST` is not specified in `solr.in.sh`/`solr.in.cmd` and this option is not provided, Solr will start the embedded ZooKeeper instance and use that instance for SolrCloud operations.+*Example*: `bin/solr start -c -z server1:2181,server2:2181``-force`::-If attempting to start Solr as the root user, the script will exit with a warning that running Solr as "root" can cause problems. It is possible to override this warning with the -force parameter.+If attempting to start Solr as the root user, the script will exit with a warning that running Solr as "root" can cause problems.+It is possible to override this warning with the `-force` parameter.+*Example*: `sudo bin/solr start -force`@@ -170,9 +194,10 @@ The `-c` and `-cloud` options are equivalent:If you specify a ZooKeeper connection string, such as `-z 192.168.1.4:2181`, then Solr will connect to ZooKeeper and join the cluster.-NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from all `bin/solr` commands.+NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from all `bin/solr` commands.- When starting Solr in SolrCloud mode, if you do not define `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` nor specify the `-z` option, then Solr will launch an embedded ZooKeeper server listening on the Solr port + 1000, i.e., if Solr is running on port 8983, then the embedded ZooKeeper will be listening on port 9983.+When starting Solr in SolrCloud mode, if you do not define `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` nor specify the `-z` option, then Solr will launch an embedded ZooKeeper server listening on the Solr port + 1000.+For example, if Solr is running on port 8983, then the embedded ZooKeeper will listen on port 9983.[IMPORTANT]====@@ -183,7 +208,7 @@ To do this use the `mkroot` command outlined below, for example: `bin/solr zk mkWhen starting in SolrCloud mode, the interactive script session will prompt you to choose a configset to use.-For more information about starting Solr in SolrCloud mode, see also the section <<getting-started-with-solrcloud.adoc#,Getting Started with SolrCloud>>.+For more information about starting Solr in SolrCloud mode, see also the section <<tutorial-solrcloud.adoc#,Getting Started with SolrCloud>>.==== Running with Example Configurations@@ -193,19 +218,23 @@ The example configurations allow you to get started quickly with a configurationEach example launches Solr with a managed schema, which allows use of the <<schema-api.adoc#,Schema API>> to make schema edits, but does not allow manual editing of a Schema file.-If you would prefer to manually modify a `schema.xml` file directly, you can change this default as described in the section <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>>.+If you would prefer to manually modify a `schema.xml` file directly, you can change this default as described in the section <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>>.-Unless otherwise noted in the descriptions below, the examples do not enable <<solrcloud.adoc#,SolrCloud>> nor <<schemaless-mode.adoc#,schemaless mode>>.+Unless otherwise noted in the descriptions below, the examples do not enable SolrCloud nor <<schemaless-mode.adoc#,schemaless mode>>.The following examples are provided:-* *cloud*: This example starts a 1-4 node SolrCloud cluster on a single machine. When chosen, an interactive session will start to guide you through options to select the initial configset to use, the number of nodes for your example cluster, the ports to use, and name of the collection to be created.+* *cloud*: This example starts a 1-4 node SolrCloud cluster on a single machine.+When chosen, an interactive session will start to guide you through options to select the initial configset to use, the number of nodes for your example cluster, the ports to use, and name of the collection to be created.+When using this example, you can choose from any of the available configsets found in `$SOLR_HOME/server/solr/configsets`.-* *techproducts*: This example starts Solr in standalone mode with a schema designed for the sample documents included in the `$SOLR_HOME/example/exampledocs` directory.++* *techproducts*: This example starts a single-node Solr instance with a schema designed for the sample documents included in the `$SOLR_HOME/example/exampledocs` directory.+The configset used can be found in `$SOLR_HOME/server/solr/configsets/sample_techproducts_configs`.-* *schemaless*: This example starts Solr in standalone mode using a managed schema, as described in the section <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>>, and provides a very minimal pre-defined schema. Solr will run in <<schemaless-mode.adoc#,Schemaless Mode>> with this configuration, where Solr will create fields in the schema on the fly and will guess field types used in incoming documents.++* *schemaless*: This example starts a single-node Solr instance using a managed schema, as described in the section <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>>, and provides a very minimal pre-defined schema.+Solr will run in <<schemaless-mode.adoc#,Schemaless Mode>> with this configuration, where Solr will create fields in the schema on the fly and will guess field types used in incoming documents.+The configset used can be found in `$SOLR_HOME/server/solr/configsets/_default`.@@ -216,7 +245,8 @@ The run in-foreground option (`-f`) is not compatible with the `-e` option since=== Stop-The `stop` command sends a STOP request to a running Solr node, which allows it to shutdown gracefully. The command will wait up to 180 seconds for Solr to stop gracefully and then will forcefully kill the process (kill -9).+The `stop` command sends a STOP request to a running Solr node, which allows it to shutdown gracefully.+The command will wait up to 180 seconds for Solr to stop gracefully and then will forcefully kill the process (`kill -9`).`bin/solr stop [options]`@@ -225,7 +255,8 @@ The `stop` command sends a STOP request to a running Solr node, which allows it==== Stop Parameters`-p <port>`::-Stop Solr running on the given port. If you are running more than one instance, or are running in SolrCloud mode, you either need to specify the ports in separate requests or use the -all option.+Stop Solr running on the given port.+If you are running more than one instance, or are running in SolrCloud mode, you either need to specify the ports in separate requests or use the `-all` option.+*Example*: `bin/solr stop -p 8983`@@ -246,7 +277,7 @@ Stop key used to protect from stopping Solr inadvertently; default is "solrrocksThe `version` command simply returns the version of Solr currently installed and immediately exists.-[source,plain]+[source,bash]----$ bin/solr versionX.Y.0@@ -293,12 +324,14 @@ Solr process 39827 running on port 8865=== Assert-The `assert` command sanity checks common issues with Solr installations. These include checking the ownership/existence of particular directories, and ensuring Solr is available on the expected URL. The command can either output a specified error message, or change its exit code to indicate errors.+The `assert` command sanity checks common issues with Solr installations.+These include checking the ownership/existence of particular directories, and ensuring Solr is available on the expected URL.+The command can either output a specified error message, or change its exit code to indicate errors.As an example:[source,bash]-bin/solr assert --exists /opt/bin/solr+$ bin/solr assert --exists /opt/bin/solrResults in the output below:@@ -309,7 +342,8 @@ Use `bin/solr assert -help` for a full list of options.=== Healthcheck-The `healthcheck` command generates a JSON-formatted health report for a collection when running in SolrCloud mode. The health report provides information about the state of every replica for all shards in a collection, including the number of committed documents and its current state.+The `healthcheck` command generates a JSON-formatted health report for a collection when running in SolrCloud mode.+The health report provides information about the state of every replica for all shards in a collection, including the number of committed documents and its current state.`bin/solr healthcheck [options]`@@ -323,7 +357,10 @@ The `healthcheck` command generates a JSON-formatted health report for a collect*Example*: `bin/solr healthcheck -c gettingstarted``-z <zkhost>`::-ZooKeeper connection string, defaults to `localhost:9983`. If you are running Solr on a port other than 8983, you will have to specify the ZooKeeper connection string. By default, this will be the Solr port + 1000. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+ZooKeeper connection string, defaults to `localhost:9983`.+If you are running Solr on a port other than 8983, you will have to specify the ZooKeeper connection string.+By default, this will be the Solr port + 1000.+This parameter is unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `bin/solr healthcheck -z localhost:2181`@@ -381,11 +418,13 @@ Below is an example healthcheck request and response using a non-standard ZooKee== Collections and Cores-The `bin/solr` script can also help you create new collections (in SolrCloud mode) or cores (in standalone mode), or delete collections.+The `bin/solr` script can also help you create new collections or cores, or delete collections or cores.=== Create a Core or Collection-The `create` command detects the mode that Solr is running in (standalone or SolrCloud) and then creates a core or collection depending on the mode.+The `create` command detects the mode that Solr is running in and creates either a core or collection depending on the mode.+When running with SolrCloud, a collection would be created.+When running a user-managed cluster or a single-node installation, a core would be created.`bin/solr create [options]`@@ -406,35 +445,42 @@ See the section <<Configuration Directories and SolrCloud>> below for more detai*Example*: `bin/solr create -d _default``-n <configName>`::-The configuration name. This defaults to the same name as the core or collection.+The configuration name.+This defaults to the same name as the core or collection.+*Example*: `bin/solr create -n basic``-p <port>`::-Port of a local Solr instance to send the create command to; by default the script tries to detect the port by looking for running Solr instances.+The port of a local Solr instance to send the create command to.+By default the script tries to detect the port by looking for running Solr instances.+-This option is useful if you are running multiple standalone Solr instances on the same host, thus requiring you to be specific about which instance to create the core in.+This option is useful if you are running multiple Solr instances on the same host, thus requiring you to be specific about which instance to create the core in.+*Example*: `bin/solr create -p 8983``-s <shards>` or `-shards`::-Number of shards to split a collection into, default is 1; only applies when Solr is running in SolrCloud mode.+Number of shards to split a collection into, default is `1`.+Only applies when Solr is running in SolrCloud mode.+*Example*: `bin/solr create -s 2``-rf <replicas>` or `-replicationFactor`::-Number of copies of each document in the collection. The default is 1 (no replication).+Number of copies of each document in the collection.+The default is `1` (no replication).+*Example*: `bin/solr create -rf 2``-force`::-If attempting to run create as "root" user, the script will exit with a warning that running Solr or actions against Solr as "root" can cause problems. It is possible to override this warning with the -force parameter.+If attempting to run create as "root" user, the script will exit with a warning that running Solr or actions against Solr as "root" can cause problems.+It is possible to override this warning with the -force parameter.+*Example*: `bin/solr create -c foo -force`==== Configuration Directories and SolrCloud-Before creating a collection in SolrCloud, the configuration directory used by the collection must be uploaded to ZooKeeper. The `create` command supports several use cases for how collections and configuration directories work. The main decision you need to make is whether a configuration directory in ZooKeeper should be shared across multiple collections.+Before creating a collection in SolrCloud, the configuration directory used by the collection must be uploaded to ZooKeeper.+The `create` command supports several use cases for how collections and configuration directories work.+The main decision you need to make is whether a configuration directory in ZooKeeper should be shared across multiple collections.Let's work through a few examples to illustrate how configuration directories work in SolrCloud.@@ -444,53 +490,67 @@ For example, the following command will result in the `_default` configuration bIf you create another collection with `bin/solr create -c contacts2`, then another copy of the `_default` directory will be uploaded to ZooKeeper under `/configs/contacts2`.-Any changes you make to the configuration for the contacts collection will not affect the `contacts2` collection. Put simply, the default behavior creates a unique copy of the configuration directory for each collection you create.+Any changes you make to the configuration for the contacts collection will not affect the `contacts2` collection.+Put simply, the default behavior creates a unique copy of the configuration directory for each collection you create.-You can override the name given to the configuration directory in ZooKeeper by using the `-n` option. For instance, the command `bin/solr create -c logs -d _default -n basic` will upload the `server/solr/configsets/_default/conf` directory to ZooKeeper as `/configs/basic`.+You can override the name given to the configuration directory in ZooKeeper by using the `-n` option.+For instance, the command `bin/solr create -c logs -d _default -n basic` will upload the `server/solr/configsets/_default/conf` directory to ZooKeeper as `/configs/basic`.-Notice that we used the `-d` option to specify a different configuration than the default. Solr provides several built-in configurations under `server/solr/configsets`. However you can also provide the path to your own configuration directory using the `-d` option. For instance, the command `bin/solr create -c mycoll -d /tmp/myconfigs`, will upload `/tmp/myconfigs` into ZooKeeper under `/configs/mycoll`.+Notice that we used the `-d` option to specify a different configuration than the default.+Solr provides several built-in configurations under `server/solr/configsets`. However you can also provide the path to your own configuration directory using the `-d` option.+For instance, the command `bin/solr create -c mycoll -d /tmp/myconfigs`, will upload `/tmp/myconfigs` into ZooKeeper under `/configs/mycoll`.To reiterate, the configuration directory is named after the collection unless you override it using the `-n` option.-Other collections can share the same configuration by specifying the name of the shared configuration using the `-n` option. For instance, the following command will create a new collection that shares the basic configuration created previously: `bin/solr create -c logs2 -n basic`.+Other collections can share the same configuration by specifying the name of the shared configuration using the `-n` option.+For instance, the following command will create a new collection that shares the basic configuration created previously: `bin/solr create -c logs2 -n basic`.==== Data-driven Schema and Shared Configurations-The `_default` schema can mutate as data is indexed, since it has schemaless functionality (i.e., data-driven changes to the schema). Consequently, we recommend that you do not share data-driven configurations between collections unless you are certain that all collections should inherit the changes made when indexing data into one of the collections. You can turn off schemaless functionality (i.e., data-driven changes to the schema) for a collection by the following, assuming the collection name is `mycollection` - see <<Set or Unset Configuration Properties>>:+The `_default` schema can mutate as data is indexed, since it has schemaless functionality (i.e., data-driven changes to the schema).+Consequently, we recommend that you do not share data-driven configurations between collections unless you are certain that all collections should inherit the changes made when indexing data into one of the collections.-[source,text]-bin/solr config -c mycollection -p 8983 -action set-user-property -property update.autoCreateFields -value false+You can turn off schemaless functionality for a collection with the following command, assuming the collection name is `mycollection`.++[source,bash]+$ bin/solr config -c mycollection -p 8983 -action set-user-property -property update.autoCreateFields -value false++See also the section <<Set or Unset Configuration Properties>>.=== Delete Core or Collection-The `delete` command detects the mode that Solr is running in (standalone or SolrCloud) and then deletes the specified core (standalone) or collection (SolrCloud) as appropriate.+The `delete` command detects the mode that Solr is running in and then deletes the specified core (user-managed or single-node) or collection (SolrCloud) as appropriate.`bin/solr delete [options]``bin/solr delete -help`-If running in SolrCloud mode, the delete command checks if the configuration directory used by the collection you are deleting is being used by other collections. If not, then the configuration directory is also deleted from ZooKeeper.+If running in SolrCloud mode, the `delete` command checks if the configuration directory used by the collection you are deleting is being used by other collections.+If not, then the configuration directory is also deleted from ZooKeeper.-For example, if you created a collection with `bin/solr create -c contacts`, then the delete command `bin/solr delete -c contacts` will check to see if the `/configs/contacts` configuration directory is being used by any other collections. If not, then the `/configs/contacts` directory is removed from ZooKeeper.+For example, if you created a collection with `bin/solr create -c contacts`, then the delete command `bin/solr delete -c contacts` will check to see if the `/configs/contacts` configuration directory is being used by any other collections.+If not, then the `/configs/contacts` directory is removed from ZooKeeper.==== Delete Core or Collection Parameters`-c <name>`::-Name of the core / collection to delete (required).+Name of the core or collection to delete (required).+*Example*: `bin/solr delete -c mycoll``-deleteConfig`::-Whether or not the configuration directory should also be deleted from ZooKeeper. The default is `true`.+Whether or not the configuration directory should also be deleted from ZooKeeper.+The default is `true`.+If the configuration directory is being used by another collection, then it will not be deleted even if you pass `-deleteConfig` as `true`.+*Example*: `bin/solr delete -deleteConfig false``-p <port>`::-The port of a local Solr instance to send the delete command to. By default the script tries to detect the port by looking for running Solr instances.+The port of a local Solr instance to send the delete command to.+By default the script tries to detect the port by looking for running Solr instances.+-This option is useful if you are running multiple standalone Solr instances on the same host, thus requiring you to be specific about which instance to delete the core from.+This option is useful if you are running multiple Solr instances on the same host, thus requiring you to be specific about which instance to delete the core from.+*Example*: `bin/solr delete -p 8983`@@ -504,7 +564,8 @@ Currently, this script only enables Basic Authentication, and is only availableThe command `bin/solr auth enable` configures Solr to use Basic Authentication when accessing the User Interface, using `bin/solr` and any API requests.-TIP: For more information about Solr's authentication plugins, see the section <<securing-solr.adoc#,Securing Solr>>. For more information on Basic Authentication support specifically, see the section <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>>.+TIP: For more information about Solr's authentication plugins, see the section <<securing-solr.adoc#,Securing Solr>>.+For more information on Basic Authentication support specifically, see the section <<basic-authentication-plugin.adoc#,Basic Authentication Plugin>>.The `bin/solr auth enable` command makes several changes to enable Basic Authentication:@@ -552,19 +613,24 @@ If prompt is preferred, pass *true* as a parameter to request the script to promEither `-credentials` or `-prompt` *must* be specified.`-blockUnknown`::-When *true*, blocks all unauthenticated users from accessing Solr. When *false*,unauthenticated users will still be able to access Solr, but only for operations not explicitly requiring a user role in the Authorization plugin configuration.+When *true*, blocks all unauthenticated users from accessing Solr.+When *false*,unauthenticated users will still be able to access Solr, but only for operations not explicitly requiring a user role in the Authorization plugin configuration.`-updateIncludeFileOnly`::When *true*, only the settings in `bin/solr.in.sh` or `bin\solr.in.cmd` will be updated, and `security.json` will not be created.`-z`::-Defines the ZooKeeper connect string. This is useful if you want to enable authentication before all your Solr nodes have come up. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+Defines the ZooKeeper connect string.+This is useful if you want to enable authentication before all your Solr nodes have come up.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.`-d`::-Defines the Solr server directory, by default `$SOLR_HOME/server`. It is not common to need to override the default, and is only needed if you have customized the `$SOLR_HOME` directory path.+Defines the Solr server directory, by default `$SOLR_HOME/server`.+It is not common to need to override the default, and is only needed if you have customized the `$SOLR_HOME` directory path.`-s`::-Defines the location of `solr.solr.home`, which by default is `server/solr`. If you have multiple instances of Solr on the same host, or if you have customized the `$SOLR_HOME` directory path, you likely need to define this.+Defines the location of `solr.solr.home`, which by default is `server/solr`.+If you have multiple instances of Solr on the same host, or if you have customized the `$SOLR_HOME` directory path, you likely need to define this.=== Disabling Basic Authentication@@ -596,7 +662,7 @@ To unset a previously set common property, specify `-action unset-property` with`bin/solr config -c mycollection -p 8983 -action unset-property -property updateHandler.autoCommit.maxDocs`-=== Set or Unset User-defined Properties+=== Set or Unset User-Defined PropertiesTo set the user-defined property `update.autoCreateFields` to `false` (to disable <<schemaless-mode.adoc#,Schemaless Mode>>):@@ -621,7 +687,8 @@ Name of the property to change (required).Set the property to this value.`-z <zkHost>`::-The ZooKeeper connection string, usable in SolrCloud mode. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string, usable in SolrCloud mode.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.`-p <port>`::`localhost` port of the Solr node to use when applying the configuration change.@@ -631,13 +698,17 @@ Base Solr URL, which can be used in SolrCloud mode to determine the ZooKeeper co== ZooKeeper Operations-The `bin/solr` script allows certain operations affecting ZooKeeper. These operations are for SolrCloud mode only. The operations are available as sub-commands, which each have their own set of options.+The `bin/solr` script allows certain operations affecting ZooKeeper.+These operations are for SolrCloud mode only.++The operations are available as sub-commands, which each have their own set of options.`bin/solr zk [sub-command] [options]``bin/solr zk -help`-NOTE: Solr should have been started at least once before issuing these commands to initialize ZooKeeper with the znodes Solr expects. Once ZooKeeper is initialized, Solr doesn't need to be running on any node to use these commands.+NOTE: Solr should have been started at least once before issuing these commands to initialize ZooKeeper with the znodes Solr expects.+Once ZooKeeper is initialized, Solr doesn't need to be running on any node to use these commands.=== Upload a Configuration Set@@ -648,7 +719,8 @@ Use the `zk upconfig` command to upload one of the pre-configured configurationAll parameters below are required.`-n <name>`::-Name of the configuration set in ZooKeeper. This command will upload the configuration set to the "configs" ZooKeeper node giving it the name specified.+Name of the configuration set in ZooKeeper.+This command will upload the configuration set to the "configs" ZooKeeper node giving it the name specified.+You can see all uploaded configuration sets in the Admin UI via the Cloud screens. Choose Cloud -> Tree -> configs to see them.+@@ -657,9 +729,11 @@ If a pre-existing configuration set is specified, it will be overwritten in ZooK*Example*: `-n myconfig``-d <configset dir>`::-The path of the configuration set to upload. It should have a `conf` directory immediately below it that in turn contains `solrconfig.xml` etc.+The path of the configuration set to upload.+It should have a `conf` directory immediately below it that in turn contains `solrconfig.xml` etc.+-If just a name is supplied, `$SOLR_HOME/server/solr/configsets` will be checked for this name. An absolute path may be supplied instead.+If just a name is supplied, `$SOLR_HOME/server/solr/configsets` will be checked for this name.+An absolute path may be supplied instead.+*Examples*:@@ -667,7 +741,8 @@ If just a name is supplied, `$SOLR_HOME/server/solr/configsets` will be checked* `-d /path/to/configset/source``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -679,7 +754,9 @@ bin/solr zk upconfig -z 111.222.333.444:2181 -n mynewconfig -d /path/to/configse.Reload Collections When Changing Configurations[WARNING]====-This command does *not* automatically make changes effective! It simply uploads the configuration sets to ZooKeeper. You can use the Collection API's <<collection-management.adoc#reload,RELOAD command>> to reload any collections that uses this configuration set.+This command does *not* automatically make changes effective!+It simply uploads the configuration sets to ZooKeeper.+You can use the Collection API's <<collection-management.adoc#reload,RELOAD command>> to reload any collections that uses this configuration set.======= Download a Configuration Set@@ -696,9 +773,11 @@ Name of the configset in ZooKeeper to download. The Admin UI Cloud -> Tree -> co*Example*: `-n myconfig``-d <configset dir>`::-The path to write the downloaded configuration set into. If just a name is supplied, `$SOLR_HOME/server/solr/configsets` will be the parent. An absolute path may be supplied as well.+The path to write the downloaded configuration set into.+If just a name is supplied, `$SOLR_HOME/server/solr/configsets` will be the parent.+An absolute path may be supplied as well.+-In either case, _pre-existing configurations at the destination will be overwritten!_+In either case, _pre-existing configurations at the destination will be overwritten_!+*Examples*:@@ -706,7 +785,8 @@ In either case, _pre-existing configurations at the destination will be overwrit* `-d /path/to/configset/destination``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -715,21 +795,28 @@ An example of this command with all parameters is:[source,bash]bin/solr zk downconfig -z 111.222.333.444:2181 -n mynewconfig -d /path/to/configset-A "best practice" is to keep your configuration sets in some form of version control as the system-of-record. In that scenario, `downconfig` should rarely be used.+A best practice is to keep your configuration sets in some form of version control as the system-of-record.+In that scenario, `downconfig` should rarely be used.=== Copy between Local Files and ZooKeeper znodes-Use the `zk cp` command for transferring files and directories between ZooKeeper znodes and your local drive. This command will copy from the local drive to ZooKeeper, from ZooKeeper to the local drive or from ZooKeeper to ZooKeeper.+Use the `zk cp` command for transferring files and directories between ZooKeeper znodes and your local drive.+This command will copy from the local drive to ZooKeeper, from ZooKeeper to the local drive or from ZooKeeper to ZooKeeper.==== ZK Copy Parameters`-r`::-Optional. Do a recursive copy. The command will fail if the <src> has children unless '-r' is specified.+Optional.+Do a recursive copy.+The command will fail if the <src> has children unless '-r' is specified.+*Example*: `-r``<src>`::-The file or path to copy from. If prefixed with `zk:` then the source is presumed to be ZooKeeper. If no prefix or the prefix is 'file:' this is the local drive. At least one of <src> or <dest> must be prefixed by `'zk:'` or the command will fail.+The file or path to copy from.+If prefixed with `zk:` then the source is presumed to be ZooKeeper.+If no prefix or the prefix is `file`: this is the local drive.+At least one of `<src>` or `<dest>` must be prefixed by `'zk:'` or the command will fail.+*Examples*:@@ -737,9 +824,12 @@ The file or path to copy from. If prefixed with `zk:` then the source is presume* `file:/Users/apache/configs/src``<dest>`::-The file or path to copy to. If prefixed with `zk:` then the source is presumed to be ZooKeeper. If no prefix or the prefix is `file:` this is the local drive.+The file or path to copy to.+If prefixed with `zk:` then the source is presumed to be ZooKeeper.+If no prefix or the prefix is `file:` this is the local drive.+-At least one of `<src>` or `<dest>` must be prefixed by `zk:` or the command will fail. If `<dest>` ends in a slash character it names a directory.+At least one of `<src>` or `<dest>` must be prefixed by `zk:` or the command will fail.+If `<dest>` ends in a slash character it names a directory.+*Examples*:@@ -747,7 +837,8 @@ At least one of `<src>` or `<dest>` must be prefixed by `zk:` or the command wil* `file:/Users/apache/configs/src``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -768,14 +859,16 @@ Use the `zk rm` command to remove a znode (and optionally all child nodes) from==== ZK Remove Parameters`-r`::-Optional. Do a recursive removal. The command will fail if the <path> has children unless '-r' is specified.+Optional.+Do a recursive removal.+The command will fail if the `<path>` has children unless `-r` is specified.+*Example*: `-r``<path>`::The path to remove from ZooKeeper, either a parent or leaf node.+-There are limited safety checks, you cannot remove '/' or '/zookeeper' nodes.+There are limited safety checks, you cannot remove `/` or `/zookeeper` nodes.+The path is assumed to be a ZooKeeper node, no `zk:` prefix is necessary.+@@ -786,7 +879,8 @@ The path is assumed to be a ZooKeeper node, no `zk:` prefix is necessary.* `/configs/myconfigset/solrconfig.xml``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -804,17 +898,20 @@ Use the `zk mv` command to move (rename) a ZooKeeper znode.==== ZK Move Parameters`<src>`::-The znode to rename. The `zk:` prefix is assumed.+The znode to rename.+The `zk:` prefix is assumed.+*Example*: `/configs/oldconfigset``<dest>`::-The new name of the znode. The `zk:` prefix is assumed.+The new name of the znode.+The `zk:` prefix is assumed.+*Example*: `/configs/newconfigset``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -829,8 +926,9 @@ Use the `zk ls` command to see the children of a znode.==== ZK List Parameters-`-r`-Optional. Recursively list all descendants of a znode.+`-r`::+Optional.+Recursively list all descendants of a znode.+*Example*: `-r`@@ -840,7 +938,8 @@ The path on ZooKeeper to list.*Example*: `/collections/mycollection``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -853,17 +952,22 @@ An example of this command with the parameters is:=== Create a znode (supports chroot)-Use the `zk mkroot` command to create a znode. The primary use-case for this command to support ZooKeeper's "chroot" concept. However, it can also be used to create arbitrary paths.+Use the `zk mkroot` command to create a znode.+The primary use-case for this command to support ZooKeeper's "chroot" concept.+However, it can also be used to create arbitrary paths.==== Create znode Parameters`<path>`::-The path on ZooKeeper to create. Intermediate znodes will be created if necessary. A leading slash is assumed even if not specified.+The path on ZooKeeper to create.+Intermediate znodes will be created if necessary.+A leading slash is assumed even if not specified.+*Example*: `/solr``-z <zkHost>`::-The ZooKeeper connection string. Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+The ZooKeeper connection string.+Unnecessary if `ZK_HOST` is defined in `solr.in.sh` or `solr.in.cmd`.+*Example*: `-z 123.321.23.43:2181`@@ -886,19 +990,25 @@ All documents can be exported, or only those that match a query.The `bin/solr export` command takes the following parameters:`format`::-The file format of the export, `jsonl` (default) or `javabin`. Choosing `javabin` exports to a file with extension `.javabin` which is the native Solr format. This is compact and faster to import.+The file format of the export, `jsonl` (default) or `javabin`.+Choosing `javabin` exports to a file with extension `.javabin` which is the native Solr format.+This is compact and faster to import.-`out`::-The file name of the export. If the file name ends with `json.gz` the output will be compressed into a .gz file.+`-out`::+The file name of the export.+If the file name ends with `json.gz` the output will be compressed into a .gz file.`query`::-A custom query. The default is `\*:*` which will export all documents.+A custom query.+The default is `\*:*` which will export all documents.`fields`::A comma separated list of fields to be exported.`limit`::-The number of documents to export. The default is `100`. The value `-1` will export all documents.+The number of documents to export.+The default is `100`.+The value `-1` will export all documents.*Examples*@@ -916,7 +1026,7 @@ bin/solr export -url http://localhost:8983/solr/gettingstarted -1 -out 1MDocs.js=== Importing Documents to a Collection-Once you have exported documents in a file, you can use the <<uploading-data-with-index-handlers.adoc#,/update request handler>> to import them to a new Solr collection.+Once you have exported documents in a file, you can use the <<indexing-with-update-handlers.adoc#,/update request handler>> to import them to a new Solr collection.*Example: import `jsonl` files*diff --git a/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc b/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adocdeleted file mode 100644index a8b30a8e09e..00000000000--- a/solr/solr-ref-guide/src/solr-cores-and-solr-xml.adoc+++ /dev/null@@ -1,36 +0,0 @@-= Solr Cores and solr.xml-:page-children: format-of-solr-xml, defining-core-properties, coreadmin-api, config-sets-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--In Solr, the term _core_ is used to refer to a single index and associated transaction log and configuration files (including the `solrconfig.xml` and Schema files, among others). Your Solr installation can have multiple cores if needed, which allows you to index data with different structures in the same server, and maintain more control over how your data is presented to different audiences. In SolrCloud mode you will be more familiar with the term _collection._ Behind the scenes a collection consists of one or more cores.--Cores can be created using `bin/solr` script or as part of SolrCloud collection creation using the APIs. Core-specific properties (such as the directories to use for the indexes or configuration files, the core name, and other options) are defined in a `core.properties` file. Any `core.properties` file in any directory of your Solr installation (or in a directory under where `solr_home` is defined) will be found by Solr and the defined properties will be used for the core named in the file.--In standalone mode, `solr.xml` must reside in `solr_home`. In SolrCloud mode, `solr.xml` will be loaded from ZooKeeper if it exists, with fallback to `solr_home`.--[NOTE]-====-In older versions of Solr, cores had to be predefined as `<core>` tags in `solr.xml` in order for Solr to know about them. Now, however, Solr supports automatic discovery of cores and they no longer need to be explicitly defined. The recommended way is to dynamically create cores/collections using the APIs.-====--The following sections describe these options in more detail.--* *<<format-of-solr-xml.adoc#,Format of solr.xml>>*: Details on how to define `solr.xml`, including the acceptable parameters for the `solr.xml` file-* *<<defining-core-properties.adoc#,Defining core.properties>>*: Details on placement of `core.properties` and available property options.-* *<<coreadmin-api.adoc#,CoreAdmin API>>*: Tools and commands for core administration using a REST API.-* *<<config-sets.adoc#,Configsets>>*: How to use configsets to avoid duplicating effort when defining a new core.diff --git a/solr/solr-ref-guide/src/solr-field-types.adoc b/solr/solr-ref-guide/src/solr-field-types.adocdeleted file mode 100644index 0b4d4d21b5b..00000000000--- a/solr/solr-ref-guide/src/solr-field-types.adoc+++ /dev/null@@ -1,38 +0,0 @@-= Solr Field Types-:page-children: field-type-definitions-and-properties, field-types-included-with-solr, working-with-currencies-and-exchange-rates, working-with-dates, working-with-enum-fields, working-with-external-files-and-processes, field-properties-by-use-case-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The field type defines how Solr should interpret data in a field and how the field can be queried. There are many field types included with Solr by default, and they can also be defined locally.--Topics covered in this section:--* <<field-type-definitions-and-properties.adoc#,Field Type Definitions and Properties>>--* <<field-types-included-with-solr.adoc#,Field Types Included with Solr>>--* <<working-with-currencies-and-exchange-rates.adoc#,Working with Currencies and Exchange Rates>>--* <<working-with-dates.adoc#,Working with Dates>>--* <<working-with-enum-fields.adoc#,Working with Enum Fields>>--* <<working-with-external-files-and-processes.adoc#,Working with External Files and Processes>>--* <<field-properties-by-use-case.adoc#,Field Properties by Use Case>>--TIP: See also the {solr-javadocs}/core/org/apache/solr/schema/FieldType.html[FieldType Javadoc].diff --git a/solr/solr-ref-guide/src/solr-glossary.adoc b/solr/solr-ref-guide/src/solr-glossary.adocindex 296eccbbbde..100dccb7eef 100644--- a/solr/solr-ref-guide/src/solr-glossary.adoc+++ b/solr/solr-ref-guide/src/solr-glossary.adoc@@ -31,7 +31,7 @@ Where possible, terms are linked to relevant parts of the Solr Reference Guide f[[SolrGlossary-A]]=== A-[[atomicupdates]]<<updating-parts-of-documents.adoc#atomic-updates,Atomic updates>>::+[[atomicupdates]]<<partial-document-updates.adoc#atomic-updates,Atomic updates>>::An approach to updating only one or more fields of a document, instead of reindexing the entire document.@@ -45,18 +45,25 @@ These control the inclusion or exclusion of keywords in a query by using operato=== C[[cluster]]Cluster::-In Solr, a cluster is a set of Solr nodes operating in coordination with each other via <<zookeeper,ZooKeeper>>, and managed as a unit. A cluster may contain many collections. See also <<solrclouddef,SolrCloud>>.+In Solr, a cluster is a set of Solr nodes operating in coordination with each other via <<zookeeper,ZooKeeper>>, and managed as a unit.+A cluster may contain many collections.+See also <<solrclouddef,SolrCloud>>.[[collection]]Collection::In Solr, one or more <<document,Documents>> grouped together in a single logical index using a single configuration and Schema.+-In <<solrclouddef,SolrCloud>> a collection may be divided up into multiple logical shards, which may in turn be distributed across many nodes, or in a Single node Solr installation, a collection may be a single <<core,Core>>.+In <<solrclouddef,SolrCloud>> a collection may be divided up into multiple logical shards, which may in turn be distributed across many nodes.+++Single-node installations and user-managed clusters use instead the concept of a <<core,Core>>.[[defcommit]]Commit::-To make document changes permanent in the index. In the case of added documents, they would be searchable after a _commit_.+To make document changes permanent in the index.+In the case of added documents, they would be searchable after a _commit_.[[core]]Core::-An individual Solr instance (represents a logical index). Multiple cores can run on a single node. See also <<solrclouddef,SolrCloud>>.+An individual Solr instance (represents a logical index).+Multiple cores can run on a single node.+See also <<solrclouddef,SolrCloud>>.[[corereload]]Core reload::To re-initialize a Solr core after changes to `schema.xml`, `solrconfig.xml` or other configuration files.@@ -68,7 +75,9 @@ To re-initialize a Solr core after changes to `schema.xml`, `solrconfig.xml` orDistributed search is one where queries are processed across more than one <<shard,Shard>>.[[document]]Document::-A group of <<field,fields>> and their values. Documents are the basic unit of data in a <<collection,collection>>. Documents are assigned to <<shard,shards>> using standard hashing, or by specifically assigning a shard within the document ID. Documents are versioned after each write operation.+A group of <<field,fields>> and their values.+Documents are the basic unit of data in a <<collection,collection>>. Documents are assigned to <<shard,shards>> using standard hashing, or by specifically assigning a shard within the document ID.+Documents are versioned after each write operation.[[SolrGlossary-E]]=== E@@ -89,16 +98,23 @@ The content to be indexed/searched along with metadata defining how the content=== I[[idf]]Inverse document frequency (IDF)::-A measure of the general importance of a term. It is calculated as the number of total Documents divided by the number of Documents that a particular word occurs in the collection. See http://en.wikipedia.org/wiki/Tf-idf and {lucene-javadocs}/core/org/apache/lucene/search/similarities/TFIDFSimilarity.html[the Lucene TFIDFSimilarity javadocs] for more info on TF-IDF based scoring and Lucene scoring in particular. See also <<termfrequency,Term frequency>>.+A measure of the general importance of a term.+It is calculated as the number of total Documents divided by the number of Documents that a particular word occurs in the collection.+See http://en.wikipedia.org/wiki/Tf-idf and {lucene-javadocs}/core/org/apache/lucene/search/similarities/TFIDFSimilarity.html[the Lucene TFIDFSimilarity javadocs] for more info on TF-IDF based scoring and Lucene scoring in particular.+See also <<termfrequency,Term frequency>>.[[invertedindex]]Inverted index::-A way of creating a searchable index that lists every word and the documents that contain those words, similar to an index in the back of a book which lists words and the pages on which they can be found. When performing keyword searches, this method is considered more efficient than the alternative, which would be to create a list of documents paired with every word used in each document. Since users search using terms they expect to be in documents, finding the term before the document saves processing resources and time.+A way of creating a searchable index that lists every word and the documents that contain those words, similar to an index in the back of a book which lists words and the pages on which they can be found.+When performing keyword searches, this method is considered more efficient than the alternative, which would be to create a list of documents paired with every word used in each document.+Since users search using terms they expect to be in documents, finding the term before the document saves processing resources and time.[[SolrGlossary-L]]=== L[[leader]]Leader::-A single <<replica,Replica>> for each <<shard,Shard>> that takes charge of coordinating index updates (document additions or deletions) to other replicas in the same shard. This is a transient responsibility assigned to a node via an election, if the current Shard Leader goes down, a new node will automatically be elected to take its place. See also <<solrclouddef,SolrCloud>>.+A single <<replica,Replica>> for each <<shard,Shard>> that takes charge of coordinating index updates (document additions or deletions) to other replicas in the same shard.+This is a transient responsibility assigned to a node via an election, if the current Shard Leader goes down, a new node will automatically be elected to take its place.+See also <<solrclouddef,SolrCloud>>.[[SolrGlossary-M]]=== M@@ -113,16 +129,20 @@ Literally, _data about data_. Metadata is information about a document, such asA search that is entered as a user would normally speak or write, as in, "What is aspirin?"[[node]]Node::-A JVM instance running Solr. Also known as a Solr server.+A JVM instance running Solr.+Also known as a Solr server.[[SolrGlossary-O]]=== O-[[optimisticconcurrency]]<<updating-parts-of-documents.adoc#optimistic-concurrency,Optimistic concurrency>>::+[[optimisticconcurrency]]<<partial-document-updates.adoc#optimistic-concurrency,Optimistic concurrency>>::Also known as "optimistic locking", this is an approach that allows for updates to documents currently in the index while retaining locking or version control.[[overseer]]Overseer::-A single node in <<solrclouddef,SolrCloud>> that is responsible for processing and coordinating actions involving the entire cluster. It keeps track of the state of existing nodes, collections, shards, and replicas, and assigns new replicas to nodes. This is a transient responsibility assigned to a node via an election, if the current Overseer goes down, a new node will be automatically elected to take its place. See also <<solrclouddef,SolrCloud>>.+A single node in <<solrclouddef,SolrCloud>> that is responsible for processing and coordinating actions involving the entire cluster.+It keeps track of the state of existing nodes, collections, shards, and replicas, and assigns new replicas to nodes.+This is a transient responsibility assigned to a node via an election, if the current Overseer goes down, a new node will be automatically elected to take its place.+See also <<solrclouddef,SolrCloud>>.[[SolrGlossary-Q]]=== Q@@ -142,49 +162,58 @@ The appropriateness of a document to the search conducted by the user.[[replica]]Replica::A <<core,Core>> that acts as a physical copy of a <<shard,Shard>> in a <<solrclouddef,SolrCloud>> <<collection,Collection>>.-[[replication]]<<index-replication.adoc#,Replication>>::+[[replication]]<<user-managed-index-replication.adoc#,Replication>>::A method of copying a leader index from one server to one or more "follower" or "child" servers.-[[requesthandler]]<<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,RequestHandler>>::+[[requesthandler]]<<requesthandlers-searchcomponents.adoc#,RequestHandler>>::Logic and configuration parameters that tell Solr how to handle incoming "requests", whether the requests are to return search results, to index documents, or to handle other custom situations.[[SolrGlossary-S]]=== S-[[searchcomponent]]<<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,SearchComponent>>::-Logic and configuration parameters used by request handlers to process query requests. Examples of search components include faceting, highlighting, and "more like this" functionality.+[[searchcomponent]]<<requesthandlers-searchcomponents.adoc#,SearchComponent>>::+Logic and configuration parameters used by request handlers to process query requests.+Examples of search components include faceting, highlighting, and "more like this" functionality.[[shard]]Shard::-In SolrCloud, a logical partition of a single <<collection,Collection>>. Every shard consists of at least one physical <<replica,Replica>>, but there may be multiple Replicas distributed across multiple <<node,Nodes>> for fault tolerance. See also <<solrclouddef,SolrCloud>>.+In SolrCloud, a logical partition of a single <<collection,Collection>>. Every shard consists of at least one physical <<replica,Replica>>, but there may be multiple Replicas distributed across multiple <<node,Nodes>> for fault tolerance.+See also <<solrclouddef,SolrCloud>>.-[[solrclouddef]]<<solrcloud.adoc#,SolrCloud>>::+[[solrclouddef]]<<cluster-types.adoc#solrcloud-mode,SolrCloud>>::Umbrella term for a suite of functionality in Solr which allows managing a <<cluster,Cluster>> of Solr <<node,Nodes>> for scalability, fault tolerance, and high availability.-[[schema]]<<documents-fields-and-schema-design.adoc#,Solr Schema (managed-schema or schema.xml)>>::-The Solr index Schema defines the fields to be indexed and the type for the field (text, integers, etc.) By default schema data can be "managed" at run time using the <<schema-api.adoc#,Schema API>> and is typically kept in a file named `managed-schema` which Solr modifies as needed, but a collection may be configured to use a static Schema, which is only loaded on startup from a human edited configuration file - typically named `schema.xml`. See <<schema-factory-definition-in-solrconfig.adoc#,Schema Factory Definition in SolrConfig>> for details.+[[schema]]<<solr-schema.adoc#,Solr Schema (managed-schema or schema.xml)>>::+The Solr index Schema defines the fields to be indexed and the type for the field (text, integers, etc.) By default schema data can be "managed" at run time using the <<schema-api.adoc#,Schema API>> and is typically kept in a file named `managed-schema` which Solr modifies as needed, but a collection may be configured to use a static Schema, which is only loaded on startup from a human edited configuration file - typically named `schema.xml`. See <<schema-factory.adoc#,Schema Factory Definition in SolrConfig>> for details.-[[solrconfig]]<<the-well-configured-solr-instance.adoc#,SolrConfig (solrconfig.xml)>>::+[[solrconfig]]<<configuring-solrconfig-xml.adoc#,SolrConfig (solrconfig.xml)>>::-The Apache Solr configuration file. Defines indexing options, RequestHandlers, highlighting, spellchecking and various other configurations. The file, `solrconfig.xml`, is located in the Solr home `conf` directory.+The Apache Solr configuration file.+Defines indexing options, RequestHandlers, highlighting, spellchecking and various other configurations.+The file, `solrconfig.xml`, is located in the Solr home `conf` directory.[[spellcheck]]<<spell-checking.adoc#,Spell Check>>::The ability to suggest alternative spellings of search terms to a user, as a check against spelling errors causing few or zero results.[[stopwords]]Stopwords::-Generally, words that have little meaning to a user's search but which may have been entered as part of a <<naturallanguagequery,natural language>> query. Stopwords are generally very small pronouns, conjunctions and prepositions (such as, "the", "with", or "and")+Generally, words that have little meaning to a user's search but which may have been entered as part of a <<naturallanguagequery,natural language>> query.+Stopwords are generally very small pronouns, conjunctions and prepositions (such as, "the", "with", or "and")[[suggesterdef]]<<suggester.adoc#,Suggester>>::Functionality in Solr that provides the ability to suggest possible query terms to users as they type.[[synonyms]]Synonyms::-Synonyms generally are terms which are near to each other in meaning and may substitute for one another. In a search engine implementation, synonyms may be abbreviations as well as words, or terms that are not consistently hyphenated. Examples of synonyms in this context would be "Inc." and "Incorporated" or "iPod" and "i-pod".+Synonyms generally are terms which are near to each other in meaning and may substitute for one another.+In a search engine implementation, synonyms may be abbreviations as well as words, or terms that are not consistently hyphenated.+Examples of synonyms in this context would be "Inc." and "Incorporated" or "iPod" and "i-pod".[[SolrGlossary-T]]=== T[[termfrequency]]Term frequency::-The number of times a word occurs in a given document. See http://en.wikipedia.org/wiki/Tf-idf and {lucene-javadocs}/core/org/apache/lucene/search/similarities/TFIDFSimilarity.html[the Lucene TFIDFSimilarity javadocs] for more info on TF-IDF based scoring and Lucene scoring in particular. See also <<idf,Inverse document frequency (IDF)>>.+The number of times a word occurs in a given document.+See http://en.wikipedia.org/wiki/Tf-idf and {lucene-javadocs}/core/org/apache/lucene/search/similarities/TFIDFSimilarity.html[the Lucene TFIDFSimilarity javadocs] for more info on TF-IDF based scoring and Lucene scoring in particular.+See also <<idf,Inverse document frequency (IDF)>>.[[transactionlog]]Transaction log::An append-only log of write operations maintained by each <<replica,Replica>>. This log is required with SolrCloud implementations and is created and managed automatically by Solr.@@ -199,4 +228,6 @@ A wildcard allows a substitution of one or more letters of a word to account for=== Z[[zookeeper]]ZooKeeper::-Also known as http://zookeeper.apache.org/[Apache ZooKeeper]. The system used by SolrCloud to keep track of configuration files and node names for a cluster. A ZooKeeper cluster is used as the central configuration store for the cluster, a coordinator for operations requiring distributed synchronization, and the system of record for cluster topology. See also <<solrclouddef,SolrCloud>>.+Also known as http://zookeeper.apache.org/[Apache ZooKeeper]. The system used by SolrCloud to keep track of configuration files and node names for a cluster.+A ZooKeeper cluster is used as the central configuration store for the cluster, a coordinator for operations requiring distributed synchronization, and the system of record for cluster topology.+See also <<solrclouddef,SolrCloud>>.diff --git a/solr/solr-ref-guide/src/running-solr-in-docker.adoc b/solr/solr-ref-guide/src/solr-in-docker.adocsimilarity index 99%rename from solr/solr-ref-guide/src/running-solr-in-docker.adocrename to solr/solr-ref-guide/src/solr-in-docker.adocindex 02798176842..a248ce822a8 100644--- a/solr/solr-ref-guide/src/running-solr-in-docker.adoc+++ b/solr/solr-ref-guide/src/solr-in-docker.adoc@@ -1,5 +1,6 @@-= Running Solr in Docker-:page-children: docker-faq, docker-networking+= Solr in Docker+:page-children: docker-faq, \+ docker-networking// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional informationdiff --git a/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc b/solr/solr-ref-guide/src/solr-indexing.adocsimilarity index 60%rename from solr/solr-ref-guide/src/introduction-to-solr-indexing.adocrename to solr/solr-ref-guide/src/solr-indexing.adocindex 52faff23d36..48db7922ad7 100644--- a/solr/solr-ref-guide/src/introduction-to-solr-indexing.adoc+++ b/solr/solr-ref-guide/src/solr-indexing.adoc@@ -1,4 +1,4 @@-= Introduction to Solr Indexing+= Solr Indexing// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -24,27 +24,36 @@ A Solr index can accept data from many different sources, including XML files, cHere are the three most common ways of loading data into a Solr index:-* Using the <<uploading-data-with-solr-cell-using-apache-tika.adoc#,Solr Cell>> framework built on Apache Tika for ingesting binary files or structured files such as Office, Word, PDF, and other proprietary formats.+* Using the <<indexing-with-tika.adoc#,Solr Cell>> framework built on Apache Tika for ingesting binary files or structured files such as Office, Word, PDF, and other proprietary formats.* Uploading XML files by sending HTTP requests to the Solr server from any environment where such requests can be generated.-* Writing a custom Java application to ingest data through Solr's Java Client API (which is described in more detail in <<client-apis.adoc#,Client APIs>>). Using the Java API may be the best choice if you're working with an application, such as a Content Management System (CMS), that offers a Java API.+* Writing a custom Java application to ingest data through Solr's Java Client API (which is described in more detail in <<client-apis.adoc#,Client APIs>>).+Using the Java API may be the best choice if you're working with an application, such as a Content Management System (CMS), that offers a Java API.-Regardless of the method used to ingest data, there is a common basic data structure for data being fed into a Solr index: a _document_ containing multiple _fields,_ each with a _name_ and containing _content,_ which may be empty. One of the fields is usually designated as a unique ID field (analogous to a primary key in a database), although the use of a unique ID field is not strictly required by Solr.+Regardless of the method used to ingest data, there is a common basic data structure for data being fed into a Solr index: a _document_ containing multiple _fields,_ each with a _name_ and containing _content,_ which may be empty.+One of the fields is usually designated as a unique ID field (analogous to a primary key in a database), although the use of a unique ID field is not strictly required by Solr.-If the field name is defined in the Schema that is associated with the index, then the analysis steps associated with that field will be applied to its content when the content is tokenized. Fields that are not explicitly defined in the Schema will either be ignored or mapped to a dynamic field definition (see <<documents-fields-and-schema-design.adoc#,Documents, Fields, and Schema Design>>), if one matching the field name exists.+If the field name is defined in the Schema that is associated with the index, then the analysis steps associated with that field will be applied to its content when the content is tokenized.+Fields that are not explicitly defined in the Schema will either be ignored or mapped to a dynamic field definition (see <<fields-and-schema-design.adoc#,Fields and Schema Design>>), if one matching the field name exists.== The Solr Example Directory-When starting Solr with the "-e" option, the `example/` directory will be used as base directory for the example Solr instances that are created. This directory also includes an `example/exampledocs/` subdirectory containing sample documents in a variety of formats that you can use to experiment with indexing into the various examples.+When starting Solr with the "-e" option, the `example/` directory will be used as base directory for the example Solr instances that are created.+This directory also includes an `example/exampledocs/` subdirectory containing sample documents in a variety of formats that you can use to experiment with indexing into the various examples.== The curl Utility for Transferring Files-Many of the instructions and examples in this section make use of the `curl` utility for transferring content through a URL. `curl` posts and retrieves data over HTTP, FTP, and many other protocols. Most Linux distributions include a copy of `curl`. You'll find curl downloads for Linux, Windows, and many other operating systems at http://curl.haxx.se/download.html. Documentation for `curl` is available here: http://curl.haxx.se/docs/manpage.html.+Many of the instructions and examples in this section make use of the `curl` utility for transferring content through a URL.+`curl` posts and retrieves data over HTTP, FTP, and many other protocols.+Most Linux distributions include a copy of `curl`.+You'll find curl downloads for Linux, Windows, and many other operating systems at http://curl.haxx.se/download.html.+Documentation for `curl` is available here: http://curl.haxx.se/docs/manpage.html.[IMPORTANT]====-Using `curl` or other command line tools for posting data is just fine for examples or tests, but it's not the recommended method for achieving the best performance for updates in production environments. You will achieve better performance with Solr Cell or the other methods described in this section.+Using `curl` or other command line tools for posting data is just fine for examples or tests, but it's not the recommended method for achieving the best performance for updates in production environments.+You will achieve better performance with Solr Cell or the other methods described in this section.Instead of `curl`, you can use utilities such as GNU `wget` (http://www.gnu.org/software/wget/) or manage GETs and POSTS with Perl, although the command line options will differ.====diff --git a/solr/solr-ref-guide/src/solr-jdbc-apache-zeppelin.adoc b/solr/solr-ref-guide/src/solr-jdbc-apache-zeppelin.adocdeleted file mode 100644index 1b2a3d06034..00000000000--- a/solr/solr-ref-guide/src/solr-jdbc-apache-zeppelin.adoc+++ /dev/null@@ -1,79 +0,0 @@-= Solr JDBC - Apache Zeppelin-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The Solr JDBC driver can support Apache Zeppelin.--IMPORTANT: This requires Apache Zeppelin 0.6.0 or greater which contains the JDBC interpreter.--To use http://zeppelin.apache.org[Apache Zeppelin] with Solr, you will need to create a JDBC interpreter for Solr. This will add SolrJ to the interpreter classpath. Once the interpreter has been created, you can create a notebook to issue queries. The http://zeppelin.apache.org/docs/latest/interpreter/jdbc.html[Apache Zeppelin JDBC interpreter documentation] provides additional information about JDBC prefixes and other features.--== Create the Apache Solr JDBC Interpreter--.Click "Interpreter" in the top navigation-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_1.png[image,height=400]--.Click "Create"-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_2.png[image,height=400]--.Enter information about your Solr installation-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_3.png[image,height=400]--[NOTE]-====-For most installations, Apache Zeppelin configures PostgreSQL as the JDBC interpreter default driver. The default driver can either be replaced by the Solr driver as outlined above or you can add a separate JDBC interpreter prefix as outlined in the http://zeppelin.apache.org/docs/latest/interpreter/jdbc.html[Apache Zeppelin JDBC interpreter documentation].-====--== Create a Notebook--.Click Notebook -> Create new note-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_4.png[image,width=517,height=400]--.Provide a name and click "Create Note"-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_5.png[image,width=839,height=400]--== JDBC Interpreter Copy Sheet--To facilitate easy copying the parameters mentioned in the screenshots, here is a consolidated list of the parameters:--[source,text,subs=attributes]------Name : Solr-Interpreter : jdbc-default.url : jdbc:solr://SOLR_ZK_CONNECTION_STRING?collection=<collection_name>-default.driver : org.apache.solr.client.solrj.io.sql.DriverImpl-default.user : solr-dependency : org.apache.solr:solr-solrj:{solr-docs-version}.0-------== Query with the Notebook--[IMPORTANT]-====-For some notebooks, the JDBC interpreter will not be bound to the notebook by default. Instructions on how to bind the JDBC interpreter to a notebook are available https://zeppelin.apache.org/docs/latest/interpreter/jdbc.html#bind-to-notebook[here].-====--.Results of Solr query-image::images/solr-jdbc-apache-zeppelin/zeppelin_solrjdbc_6.png[image,width=481,height=400]--The below code block assumes that the Apache Solr driver is setup as the default JDBC interpreter driver. If that is not the case, instructions for using a different prefix is available https://zeppelin.apache.org/docs/latest/interpreter/jdbc.html#how-to-use[here].--[source,text]------%jdbc-select fielda, fieldb, from test limit 10-----diff --git a/solr/solr-ref-guide/src/solr-jdbc-dbvisualizer.adoc b/solr/solr-ref-guide/src/solr-jdbc-dbvisualizer.adocdeleted file mode 100644index 107a3d9984c..00000000000--- a/solr/solr-ref-guide/src/solr-jdbc-dbvisualizer.adoc+++ /dev/null@@ -1,121 +0,0 @@-= Solr JDBC - DbVisualizer-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Solr's JDBC driver supports DBVisualizer for querying Solr.--For https://www.dbvis.com/[DbVisualizer], you will need to create a new driver for Solr using the DbVisualizer Driver Manager. This will add several SolrJ client .jars to the DbVisualizer classpath. The files required are:--* all .jars found in `$SOLR_HOME/dist/solrj-lib`-* the SolrJ .jar found at `$SOLR_HOME/dist/solr-solrj-<version>.jar`--Once the driver has been created, you can create a connection to Solr with the connection string format outlined in the generic section and use the SQL Commander to issue queries.--== Setup Driver--=== Open Driver Manager--From the Tools menu, choose Driver Manager to add a driver.--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_1.png[image,width=673,height=400]---=== Create a New Driver--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_2.png[image,width=532,height=400]---=== Name the Driver in Driver Manager--Provide a name for the driver, and provide the URL format: `jdbc:solr://<zk_connection_string>/?collection=<collection>`. Do not fill in values for the variables "```zk_connection_string```" and "```collection```", those will be provided later when the connection to Solr is configured. The Driver Class will also be automatically added when the driver .jars are added.--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_3.png[image,width=532,height=400]---=== Add Driver Files to Classpath--The driver files to be added are:--* all .jars in `$SOLR_HOME/dist/solrj-lib`-* the SolrJ .jar found in `$SOLR_HOME/dist/solr-solrj-<version>.jar`--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_4.png[image,width=535,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_5.png[image,width=664,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_6.png[image,width=653,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_7.png[image,width=655,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_9.png[image,width=651,height=400]---=== Review and Close Driver Manager--Once the driver files have been added, you can close the Driver Manager.--== Create a Connection--Next, create a connection to Solr using the driver just created.--=== Use the Connection Wizard--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_11.png[image,width=763,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_12.png[image,width=807,height=400]---=== Name the Connection--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_13.png[image,width=402,height=400]---=== Select the Solr driver--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_14.png[image,width=399,height=400]---=== Specify the Solr URL--Provide the Solr URL, using the ZooKeeper host and port and the collection. For example, `jdbc:solr://localhost:9983?collection=test`--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_15.png[image,width=401,height=400]---== Open and Connect to Solr--Once the connection has been created, double-click on it to open the connection details screen and connect to Solr.--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_16.png[image,width=625,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_17.png[image,width=592,height=400]---== Open SQL Commander to Enter Queries--When the connection is established, you can use the SQL Commander to issue queries and view data.--image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_19.png[image,width=577,height=400]---image::images/solr-jdbc-dbvisualizer/dbvisualizer_solrjdbc_20.png[image,width=556,height=400]diff --git a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc b/solr/solr-ref-guide/src/solr-on-hdfs.adocsimilarity index 64%rename from solr/solr-ref-guide/src/running-solr-on-hdfs.adocrename to solr/solr-ref-guide/src/solr-on-hdfs.adocindex 919a1fb9d94..9f31235efd9 100644--- a/solr/solr-ref-guide/src/running-solr-on-hdfs.adoc+++ b/solr/solr-ref-guide/src/solr-on-hdfs.adoc@@ -1,4 +1,4 @@-= Running Solr on HDFS+= Solr on HDFS// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -22,17 +22,22 @@ Solr has support for writing and reading its index and transaction log files toThis does not use Hadoop MapReduce to process Solr data, rather it only uses the HDFS filesystem for index and transaction log file storage.-To use HDFS rather than a local filesystem, you must be using Hadoop 2.x and you will need to instruct Solr to use the `HdfsDirectoryFactory`. There are also several additional parameters to define. These can be set in one of three ways:+To use HDFS rather than a local filesystem, you must be using Hadoop 2.x and you will need to instruct Solr to use the `HdfsDirectoryFactory`.+There are also several additional parameters to define.+These can be set in one of three ways:-* Pass JVM arguments to the `bin/solr` script. These would need to be passed every time you start Solr with `bin/solr`.+* Pass JVM arguments to the `bin/solr` script.+These would need to be passed every time you start Solr with `bin/solr`.* Modify `solr.in.sh` (or `solr.in.cmd` on Windows) to pass the JVM arguments automatically when using `bin/solr` without having to set them manually.-* Define the properties in `solrconfig.xml`. These configuration changes would need to be repeated for every collection, so is a good option if you only want some of your collections stored in HDFS.+* Define the properties in `solrconfig.xml`.+These configuration changes would need to be repeated for every collection, so is a good option if you only want some of your collections stored in HDFS.== Starting Solr on HDFS-=== Standalone Solr Instances+=== User-Managed Cluters and Single-Node Installations-For standalone Solr instances, there are a few parameters you should modify before starting Solr. These can be set in `solrconfig.xml` (more on that <<HdfsDirectoryFactory Parameters,below>>), or passed to the `bin/solr` script at startup.+For user-managed clusters or single-node Solr installations, there are a few parameters you should modify before starting Solr.+These can be set in `solrconfig.xml` (more on that <<HdfsDirectoryFactory Parameters,below>>), or passed to the `bin/solr` script at startup.* You need to use an `HdfsDirectoryFactory` and a data directory in the form `hdfs://host:port/path`* You need to specify an `updateLog` location in the form `hdfs://host:port/path`@@ -48,11 +53,12 @@ bin/solr start -Dsolr.directoryFactory=HdfsDirectoryFactory-Dsolr.updatelog=hdfs://host:port/path-----This example will start Solr in standalone mode, using the defined JVM properties (explained in more detail <<HdfsDirectoryFactory Parameters,below>>).+This example will start Solr using the defined JVM properties (explained in more detail <<HdfsDirectoryFactory Parameters,below>>).=== SolrCloud Instances-In SolrCloud mode, it's best to leave the data and update log directories as the defaults Solr comes with and simply specify the `solr.hdfs.home`. All dynamically created collections will create the appropriate directories automatically under the `solr.hdfs.home` root directory.+In SolrCloud mode, it's best to leave the data and update log directories as the defaults Solr comes with and simply specify the `solr.hdfs.home`.+All dynamically created collections will create the appropriate directories automatically under the `solr.hdfs.home` root directory.* Set `solr.hdfs.home` in the form `hdfs://host:port/path`* You should specify a lock factory type of `'hdfs'` or none.@@ -64,12 +70,14 @@ bin/solr start -c -Dsolr.directoryFactory=HdfsDirectoryFactory-Dsolr.hdfs.home=hdfs://host:port/path-----This command starts Solr in SolrCloud mode, using the defined JVM properties.+This command starts Solr using the defined JVM properties.=== Modifying solr.in.sh (*nix) or solr.in.cmd (Windows)-The examples above assume you will pass JVM arguments as part of the start command every time you use `bin/solr` to start Solr. However, `bin/solr` looks for an include file named `solr.in.sh` (`solr.in.cmd` on Windows) to set environment variables. By default, this file is found in the `bin` directory, and you can modify it to permanently add the `HdfsDirectoryFactory` settings and ensure they are used every time Solr is started.+The examples above assume you will pass JVM arguments as part of the start command every time you use `bin/solr` to start Solr.+However, `bin/solr` looks for an include file named `solr.in.sh` (`solr.in.cmd` on Windows) to set environment variables.+By default, this file is found in the `bin` directory, and you can modify it to permanently add the `HdfsDirectoryFactory` settings and ensure they are used every time Solr is started.For example, to set JVM arguments to always use HDFS when running in SolrCloud mode (as shown above), you would add a section such as this:@@ -83,7 +91,11 @@ For example, to set JVM arguments to always use HDFS when running in SolrCloud m== The Block Cache-For performance, the `HdfsDirectoryFactory` uses a Directory that will cache HDFS blocks. This caching mechanism replaces the standard file system cache that Solr utilizes. By default, this cache is allocated off-heap. This cache will often need to be quite large and you may need to raise the off-heap memory limit for the specific JVM you are running Solr in. For the Oracle/OpenJDK JVMs, the following is an example command-line parameter that you can use to raise the limit when starting Solr:+For performance, the `HdfsDirectoryFactory` uses a Directory that will cache HDFS blocks.+This caching mechanism replaces the standard file system cache that Solr utilizes.+By default, this cache is allocated off-heap.+This cache will often need to be quite large and you may need to raise the off-heap memory limit for the specific JVM you are running Solr in.+For the Oracle/OpenJDK JVMs, the following is an example command-line parameter that you can use to raise the limit when starting Solr:[source,bash]----@@ -97,35 +109,48 @@ The `HdfsDirectoryFactory` has a number of settings defined as part of the `dire=== Solr HDFS Settings`solr.hdfs.home`::-A root location in HDFS for Solr to write collection data to. Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location. The structure of this parameter is `hdfs://host:port/path/solr`.+A root location in HDFS for Solr to write collection data to.+Rather than specifying an HDFS location for the data directory or update log directory, use this to specify one root location and have everything automatically created within this HDFS location.+The structure of this parameter is `hdfs://host:port/path/solr`.=== Block Cache Settings`solr.hdfs.blockcache.enabled`::-Enable the blockcache. The default is `true`.+Enable the blockcache.+The default is `true`.`solr.hdfs.blockcache.read.enabled`::-Enable the read cache. The default is `true`.+Enable the read cache.+The default is `true`.`solr.hdfs.blockcache.direct.memory.allocation`::-Enable direct memory allocation. If this is `false`, heap is used. The default is `true`.+Enable direct memory allocation.+If this is `false`, heap is used.+The default is `true`.`solr.hdfs.blockcache.slab.count`::-Number of memory slabs to allocate. Each slab is 128 MB in size. The default is `1`.+Number of memory slabs to allocate.+Each slab is 128 MB in size.+The default is `1`.`solr.hdfs.blockcache.global`::-Enable/Disable using one global cache for all SolrCores. The settings used will be from the first HdfsDirectoryFactory created. The default is `true`.+Enable/Disable using one global cache for all SolrCores.+The settings used will be from the first HdfsDirectoryFactory created.+The default is `true`.=== NRTCachingDirectory Settings-`solr.hdfs.nrtcachingdirectory.enable`:: true |-Enable the use of NRTCachingDirectory. The default is `true`.+`solr.hdfs.nrtcachingdirectory.enable`::+Enable the use of NRTCachingDirectory.+The default is `true`.`solr.hdfs.nrtcachingdirectory.maxmergesizemb`::-NRTCachingDirectory max segment size for merges. The default is `16`.+NRTCachingDirectory max segment size for merges.+The default is `16`.`solr.hdfs.nrtcachingdirectory.maxcachedmb`::-NRTCachingDirectory max cache size. The default is `192`.+NRTCachingDirectory max cache size.+The default is `192`.=== HDFS Client Configuration Settings@@ -134,10 +159,13 @@ Pass the location of HDFS client configuration files - needed for HDFS HA for ex=== Kerberos Authentication Settings-Hadoop can be configured to use the Kerberos protocol to verify user identity when trying to access core services like HDFS. If your HDFS directories are protected using Kerberos, then you need to configure Solr's HdfsDirectoryFactory to authenticate using Kerberos in order to read and write to HDFS. To enable Kerberos authentication from Solr, you need to set the following parameters:+Hadoop can be configured to use the Kerberos protocol to verify user identity when trying to access core services like HDFS.+If your HDFS directories are protected using Kerberos, then you need to configure Solr's HdfsDirectoryFactory to authenticate using Kerberos in order to read and write to HDFS.+To enable Kerberos authentication from Solr, you need to set the following parameters:`solr.hdfs.security.kerberos.enabled`::-Set to `true` to enable Kerberos authentication. The default is `false`.+Set to `true` to enable Kerberos authentication.+The default is `false`.`solr.hdfs.security.kerberos.keytabfile`::A keytab file contains pairs of Kerberos principals and encrypted keys which allows for password-less authentication when Solr attempts to authenticate with secure Hadoop.@@ -177,4 +205,3 @@ If using Kerberos, you will need to add the three Kerberos related properties to<str name="solr.hdfs.security.kerberos.principal">solr/[email protected]</str></directoryFactory>-----diff --git a/solr/solr-ref-guide/src/solr-plugins.adoc b/solr/solr-ref-guide/src/solr-plugins.adocindex 565b7bbd5af..084dabd6950 100644--- a/solr/solr-ref-guide/src/solr-plugins.adoc+++ b/solr/solr-ref-guide/src/solr-plugins.adoc@@ -47,17 +47,15 @@ Plugins are packaged into a Java jar file and may have other dependent jar filesThe next sections describe some options:-* <<libs.adoc#lib-directories,Lib Directories>>:-Describes where to put the plugin's JAR files on the file system; either in one of the special places or a place convenient to you along with a `<lib/>` directive in `solrconfig.xml`.-This has been the standard approach since Solr's inception.-It's simple and reliable but it's entirely on you to ensure that all nodes in a cluster have them.-Contrib modules ship with Solr so there's no effort for them but not so for other plugins (yours or 3rd party).--* <<package-manager.adoc#,Package Management>>:-Describes a new and experimental system to manage packages of plugins in SolrCloud.-It includes CLI commands, cluster-wide installation, use of plugin registries that host plugins, cryptographically signed plugins for security, and more.-Only some plugins support this as of now (support for more types of plugins coming soon).--* <<cluster-plugins.adoc#,Cluster Plugins>>:-Describes the API used for managing cluster-level plugins such as request handlers,-cluster-level event producer and <<replica-placement-plugins.adoc#,replica placement plugins>>.+****+// This tags the below list so it can be used in the parent page section list+// tag::plugin-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<libs.adoc#,Lib Directories>>: Plugins as libraries on the filesystem.+| <<package-manager.adoc#,Package Management>>: Package-based plugins.+| <<cluster-plugins.adoc#,Cluster Plugins>>: Cluster-level plugins.+| <<replica-placement-plugins.adoc#,Replica Placement Plugins>>: Plugins specifically for replica placement.+|===+// end::plugin-sections[]+****diff --git a/solr/solr-ref-guide/src/solr-schema.adoc b/solr/solr-ref-guide/src/solr-schema.adocnew file mode 100644index 00000000000..7cb45bc3f73--- /dev/null+++ b/solr/solr-ref-guide/src/solr-schema.adoc@@ -0,0 +1,40 @@+= Solr Schema+:page-children: schema-elements, \+ schema-api, \+ schemaless-mode, \+ schema-browser-screen+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+Solr's schema governs how documents are indexed and how terms in a query are interpreted.++The following sections cover how to work with the schema generally.+Note, however, that many of the elements of a schema are covered in the section <<fields-and-schema-design.adoc#,Fields and Schema Design>>.++****+// This tags the below list so it can be used in the parent page section list+// tag::schema-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<schema-elements.adoc#,Schema Elements>>: The structure of Solr's schema.+| <<schema-api.adoc#,Schema API>>: An API to read a schema or create new fields and copyField rules.+| <<schemaless-mode.adoc#,Schemaless Mode>>: Automatically add previously unknown fields using field type guessing.+| <<schema-browser-screen.adoc#,Schema Browser>>: Schema details through the Admin UI.+|===+// end::schema-sections[]+****diff --git a/solr/solr-ref-guide/src/solr-tutorial.adoc b/solr/solr-ref-guide/src/solr-tutorial.adocindex c2817ec1687..4ebbe7ce8e0 100644--- a/solr/solr-ref-guide/src/solr-tutorial.adoc+++ b/solr/solr-ref-guide/src/solr-tutorial.adoc@@ -1,5 +1,10 @@-= Solr Tutorial+= Solr Tutorials:experimental:+:page-children: tutorial-techproducts, \+ tutorial-films, \+ tutorial-diy, \+ tutorial-solrcloud, \+ tutorial-aws// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -20,11 +25,12 @@This tutorial covers getting Solr up and running, ingesting a variety of data sources into Solr collections,and getting a feel for the Solr administrative and search interfaces.-The tutorial is organized into three sections that each build on the one before it. The <<exercise-1,first exercise>> will ask you to start Solr, create a collection, index some basic documents, and then perform some searches.+The tutorial is organized into three sections that each build on the one before it.+The <<tutorial-techproducts.adoc#,first exercise>> will ask you to start Solr, create a collection, index some basic documents, and then perform some searches.-The <<exercise-2,second exercise>> works with a different set of data, and explores requesting facets with the dataset.+The <<tutorial-films.adoc#,second exercise>> works with a different set of data, and explores requesting facets with the dataset.-The <<exercise-3,third exercise>> encourages you to begin to work with your own data and start a plan for your implementation.+The <<tutorial-diy.adoc#,third exercise>> encourages you to begin to work with your own data and start a plan for your implementation.Finally, we'll introduce <<Spatial Queries,spatial search>> and show you how to get your Solr instance back into a clean state.@@ -34,7 +40,8 @@ To follow along with this tutorial, you will need...// TODO possibly remove this system requirements or only replace the link. To meet the {solr-javadocs}/SYSTEM_REQUIREMENTS.html[system requirements]-. An Apache Solr release http://solr.apache.org/downloads.html[download]. This tutorial is designed for Apache Solr {solr-docs-version}.+. An Apache Solr release http://solr.apache.org/downloads.html[download].+This tutorial is designed for Apache Solr {solr-docs-version}.For best results, please run the browser showing this tutorial and the Solr server on the same machine so tutorial links will correctly point to your Solr server.@@ -54,903 +61,13 @@ solr-{solr-docs-version}.0.zipIf you'd like to know more about Solr's directory layout before moving to the first exercise, see the section <<installing-solr.adoc#directory-layout,Directory Layout>> for details.-[[exercise-1]]-== Exercise 1: Index Techproducts Example Data--This exercise will walk you through how to start Solr as a two-node cluster (both nodes on the same machine) and create a collection during startup. Then you will index some sample data that ships with Solr and do some basic searches.--=== Launch Solr in SolrCloud Mode-To launch Solr, run: `bin/solr start -e cloud` on Unix or MacOS; `bin\solr.cmd start -e cloud` on Windows.--This will start an interactive session that will start two Solr "servers" on your machine. This command has an option to run without prompting you for input (`-noprompt`), but we want to modify two of the defaults so we won't use that option now.--[source,subs="verbatim,attributes+"]------solr-{solr-docs-version}.0:$ ./bin/solr start -e cloud--Welcome to the SolrCloud example!--This interactive session will help you launch a SolrCloud cluster on your local workstation.-To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2]:------The first prompt asks how many nodes we want to run. Note the `[2]` at the end of the last line; that is the default number of nodes. Two is what we want for this example, so you can simply press kbd:[enter].--[source,subs="verbatim,attributes+"]------Ok, let's start up 2 Solr nodes for your example SolrCloud cluster.-Please enter the port for node1 [8983]:------This will be the port that the first node runs on. Unless you know you have something else running on port 8983 on your machine, accept this default option also by pressing kbd:[enter]. If something is already using that port, you will be asked to choose another port.--[source,subs="verbatim,attributes+"]------Please enter the port for node2 [7574]:------This is the port the second node will run on. Again, unless you know you have something else running on port 7574 on your machine, accept this default option also by pressing kbd:[enter]. If something is already using that port, you will be asked to choose another port.--Solr will now initialize itself and start running on those two nodes. The script will print the commands it uses for your reference.--[source,subs="verbatim,attributes+"]------Starting up 2 Solr nodes for your example SolrCloud cluster.--Creating Solr home directory /solr-{solr-docs-version}.0/example/cloud/node1/solr-Cloning /solr-{solr-docs-version}.0/example/cloud/node1 into- /solr-{solr-docs-version}.0/example/cloud/node2--Starting up Solr on port 8983 using command:-"bin/solr" start -cloud -p 8983 -s "example/cloud/node1/solr"--Waiting up to 180 seconds to see Solr running on port 8983 [\]-Started Solr server on port 8983 (pid=34942). Happy searching!---Starting up Solr on port 7574 using command:-"bin/solr" start -cloud -p 7574 -s "example/cloud/node2/solr" -z localhost:9983--Waiting up to 180 seconds to see Solr running on port 7574 [\]-Started Solr server on port 7574 (pid=35036). Happy searching!--INFO - 2017-07-27 12:28:02.835; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready-------Notice that two instances of Solr have started on two nodes. Because we are starting in SolrCloud mode, and did not define any details about an external ZooKeeper cluster, Solr launches its own ZooKeeper and connects both nodes to it.--After startup is complete, you'll be prompted to create a collection to use for indexing data.--[source,subs="verbatim,attributes+"]------Now let's create a new collection for indexing documents in your 2-node cluster.-Please provide a name for your new collection: [gettingstarted]-------Here's the first place where we'll deviate from the default options. This tutorial will ask you to index some sample data included with Solr, called the "techproducts" data. Let's name our collection "techproducts" so it's easy to differentiate from other collections we'll create later. Enter `techproducts` at the prompt and hit kbd:[enter].--[source,subs="verbatim,attributes+"]------How many shards would you like to split techproducts into? [2]-------This is asking how many <<solr-glossary.adoc#shard,shards>> you want to split your index into across the two nodes. Choosing "2" (the default) means we will split the index relatively evenly across both nodes, which is a good way to start. Accept the default by hitting kbd:[enter].--[source,subs="verbatim,attributes+"]------How many replicas per shard would you like to create? [2]-------A replica is a copy of the index that's used for failover (see also the <<solr-glossary.adoc#replica,Solr Glossary definition>>). Again, the default of "2" is fine to start with here also, so accept the default by hitting kbd:[enter].--[source,subs="verbatim,attributes+"]------Please choose a configuration for the techproducts collection, available options are:-_default or sample_techproducts_configs [_default]-------We've reached another point where we will deviate from the default option. Solr has two sample sets of configuration files (called a configset) available out-of-the-box.--A collection must have a configset, which at a minimum includes the two main configuration files for Solr: the schema file (named either `managed-schema` or `schema.xml`), and `solrconfig.xml`. The question here is which configset you would like to start with. The `_default` is a bare-bones option, but note there's one whose name includes "techproducts", the same as we named our collection. This configset is specifically designed to support the sample data we want to use, so enter `sample_techproducts_configs` at the prompt and hit kbd:[enter].--At this point, Solr will create the collection and again output to the screen the commands it issues.--[source,subs="verbatim,attributes+"]------Uploading /solr-{solr-docs-version}.0/server/solr/configsets/_default/conf for config techproducts to ZooKeeper at localhost:9983--Connecting to ZooKeeper at localhost:9983 ...-INFO - 2017-07-27 12:48:59.289; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready-Uploading /solr-{solr-docs-version}.0/server/solr/configsets/sample_techproducts_configs/conf for config techproducts to ZooKeeper at localhost:9983--Creating new collection 'techproducts' using command:-http://localhost:8983/solr/admin/collections?action=CREATE&name=techproducts&numShards=2&replicationFactor=2&collection.configName=techproducts--{- "responseHeader":{- "status":0,- "QTime":5460},- "success":{- "192.168.0.110:7574_solr":{- "responseHeader":{- "status":0,- "QTime":4056},- "core":"techproducts_shard1_replica_n1"},- "192.168.0.110:8983_solr":{- "responseHeader":{- "status":0,- "QTime":4056},- "core":"techproducts_shard2_replica_n2"}}}--Enabling auto soft-commits with maxTime 3 secs using the Config API--POSTing request to Config API: http://localhost:8983/solr/techproducts/config-{"set-property":{"updateHandler.autoSoftCommit.maxTime":"3000"}}-Successfully set-property updateHandler.autoSoftCommit.maxTime to 3000--SolrCloud example running, please visit: http://localhost:8983/solr-------*Congratulations!* Solr is ready for data!--You can see that Solr is running by launching the Solr Admin UI in your web browser: http://localhost:8983/solr/. This is the main starting point for administering Solr.--Solr will now be running two "nodes", one on port 7574 and one on port 8983. There is one collection created automatically, `techproducts`, a two shard collection, each with two replicas.--The http://localhost:8983/solr/#/~cloud[Cloud tab] in the Admin UI diagrams the collection nicely:--.SolrCloud Diagram-image::images/solr-tutorial/tutorial-solrcloud.png[]--=== Index the Techproducts Data--Your Solr server is up and running, but it doesn't contain any data yet, so we can't do any queries.--Solr includes the `bin/post` tool in order to facilitate indexing various types of documents easily. We'll use this tool for the indexing examples below.--You'll need a command shell to run some of the following examples, rooted in the Solr install directory; the shell from where you launched Solr works just fine.--NOTE: Currently the `bin/post` tool does not have a comparable Windows script, but the underlying Java program invoked is available. We'll show examples below for Windows, but you can also see the <<post-tool.adoc#post-tool-windows-support,Windows section>> of the Post Tool documentation for more details.--The data we will index is in the `example/exampledocs` directory. The documents are in a mix of document formats (JSON, CSV, etc.), and fortunately we can index them all at once:--.Linux/Mac-[source,subs="verbatim,attributes+"]------solr-{solr-docs-version}.0:$ bin/post -c techproducts example/exampledocs/*-------.Windows-[source,subs="verbatim,attributes+"]------C:\solr-{solr-docs-version}.0> java -jar -Dc=techproducts -Dauto example\exampledocs\post.jar example\exampledocs\*-------You should see output similar to the following:--[source,subs="verbatim,attributes+"]------SimplePostTool version 5.0.0-Posting files to [base] url http://localhost:8983/solr/techproducts/update...-Entering auto mode. File endings considered are xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log-POSTing file books.csv (text/csv) to [base]-POSTing file books.json (application/json) to [base]/json/docs-POSTing file gb18030-example.xml (application/xml) to [base]-POSTing file hd.xml (application/xml) to [base]-POSTing file ipod_other.xml (application/xml) to [base]-POSTing file ipod_video.xml (application/xml) to [base]-POSTing file manufacturers.xml (application/xml) to [base]-POSTing file mem.xml (application/xml) to [base]-POSTing file money.xml (application/xml) to [base]-POSTing file monitor.xml (application/xml) to [base]-POSTing file monitor2.xml (application/xml) to [base]-POSTing file more_books.jsonl (application/json) to [base]/json/docs-POSTing file mp500.xml (application/xml) to [base]-POSTing file post.jar (application/octet-stream) to [base]/extract-POSTing file sample.html (text/html) to [base]/extract-POSTing file sd500.xml (application/xml) to [base]-POSTing file solr-word.pdf (application/pdf) to [base]/extract-POSTing file solr.xml (application/xml) to [base]-POSTing file test_utf8.sh (application/octet-stream) to [base]/extract-POSTing file utf8-example.xml (application/xml) to [base]-POSTing file vidcard.xml (application/xml) to [base]-21 files indexed.-COMMITting Solr index changes to http://localhost:8983/solr/techproducts/update...-Time spent: 0:00:00.822-------Congratulations again! You have data in your Solr!--Now we're ready to start searching.--[[tutorial-searching]]-=== Basic Searching--Solr can be queried via REST clients, curl, wget, Chrome POSTMAN, etc., as well as via native clients available for many programming languages.--The Solr Admin UI includes a query builder interface via the Query tab for the `techproducts` collection (at http://localhost:8983/solr/#/techproducts/query). If you click the btn:[Execute Query] button without changing anything in the form, you'll get 10 documents in JSON format:--.Query Screen-image::images/solr-tutorial/tutorial-query-screen.png[Solr Quick Start: techproducts Query screen with results]--The URL sent by the Admin UI to Solr is shown in light grey near the top right of the above screenshot. If you click on it, your browser will show you the raw response.--To use curl, give the same URL shown in your browser in quotes on the command line:--`curl "http://localhost:8983/solr/techproducts/select?indent=on&q=\*:*"`--What's happening here is that we are using Solr's query parameter (`q`) with a special syntax that requests all documents in the index (`\*:*`). All of the documents are not returned to us, however, because of the default for a parameter called `rows`, which you can see in the form is `10`. You can change the parameter in the UI or in the defaults if you wish.--Solr has very powerful search options, and this tutorial won't be able to cover all of them. But we can cover some of the most common types of queries.--==== Search for a Single Term--To search for a term, enter it as the `q` parameter value in the Solr Admin UI Query screen, replacing `\*:*` with the term you want to find.--Enter "foundation" and hit btn:[Execute Query] again.--If you prefer curl, enter something like this:--`curl "http://localhost:8983/solr/techproducts/select?q=foundation"`--You'll see something like this:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":8,- "params":{- "q":"foundation"}},- "response":{"numFound":4,"start":0,"maxScore":2.7879646,"docs":[- {- "id":"0553293354",- "cat":["book"],- "name":"Foundation",- "price":7.99,- "price_c":"7.99,USD",- "inStock":true,- "author":"Isaac Asimov",- "author_s":"Isaac Asimov",- "series_t":"Foundation Novels",- "sequence_i":1,- "genre_s":"scifi",- "_version_":1574100232473411586,- "price_c____l_ns":799}]-}}--The response indicates that there are 4 hits (`"numFound":4`). We've only included one document the above sample output, but since 4 hits is lower than the `rows` parameter default of 10 to be returned, you should see all 4 of them.--Note the `responseHeader` before the documents. This header will include the parameters you have set for the search. By default it shows only the parameters _you_ have set for this query, which in this case is only your query term.--The documents we got back include all the fields for each document that were indexed. This is, again, default behavior. If you want to restrict the fields in the response, you can use the `fl` parameter, which takes a comma-separated list of field names. This is one of the available fields on the query form in the Admin UI.--Put "id" (without quotes) in the "fl" box and hit btn:[Execute Query] again. Or, to specify it with curl:--`curl "http://localhost:8983/solr/techproducts/select?q=foundation&fl=id"`--You should only see the IDs of the matching records returned.--==== Field Searches--All Solr queries look for documents using some field. Often you want to query across multiple fields at the same time, and this is what we've done so far with the "foundation" query. This is possible with the use of copy fields, which are set up already with this set of configurations. We'll cover copy fields a little bit more in Exercise 2.--Sometimes, though, you want to limit your query to a single field. This can make your queries more efficient and the results more relevant for users.--Much of the data in our small sample data set is related to products. Let's say we want to find all the "electronics" products in the index. In the Query screen, enter "electronics" (without quotes) in the `q` box and hit btn:[Execute Query]. You should get 14 results, such as:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":6,- "params":{- "q":"electronics"}},- "response":{"numFound":14,"start":0,"maxScore":1.5579545,"docs":[- {- "id":"IW-02",- "name":"iPod & iPod Mini USB 2.0 Cable",- "manu":"Belkin",- "manu_id_s":"belkin",- "cat":["electronics",- "connector"],- "features":["car power adapter for iPod, white"],- "weight":2.0,- "price":11.5,- "price_c":"11.50,USD",- "popularity":1,- "inStock":false,- "store":"37.7752,-122.4232",- "manufacturedate_dt":"2006-02-14T23:55:59Z",- "_version_":1574100232554151936,- "price_c____l_ns":1150}]-}}--This search finds all documents that contain the term "electronics" anywhere in the indexed fields. However, we can see from the above there is a `cat` field (for "category"). If we limit our search for only documents with the category "electronics", the results will be more precise for our users.--Update your query in the `q` field of the Admin UI so it's `cat:electronics`. Now you get 12 results:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":6,- "params":{- "q":"cat:electronics"}},- "response":{"numFound":12,"start":0,"maxScore":0.9614112,"docs":[- {- "id":"SP2514N",- "name":"Samsung SpinPoint P120 SP2514N - hard drive - 250 GB - ATA-133",- "manu":"Samsung Electronics Co. Ltd.",- "manu_id_s":"samsung",- "cat":["electronics",- "hard drive"],- "features":["7200RPM, 8MB cache, IDE Ultra ATA-133",- "NoiseGuard, SilentSeek technology, Fluid Dynamic Bearing (FDB) motor"],- "price":92.0,- "price_c":"92.0,USD",- "popularity":6,- "inStock":true,- "manufacturedate_dt":"2006-02-13T15:26:37Z",- "store":"35.0752,-97.032",- "_version_":1574100232511160320,- "price_c____l_ns":9200}]- }}--Using curl, this query would look like this:--`curl "http://localhost:8983/solr/techproducts/select?q=cat:electronics"`--==== Phrase Search--To search for a multi-term phrase, enclose it in double quotes: `q="multiple terms here"`. For example, search for "CAS latency" by entering that phrase in quotes to the `q` box in the Admin UI.--If you're following along with curl, note that the space between terms must be converted to "+" in a URL, as so:--`curl "http://localhost:8983/solr/techproducts/select?q=\"CAS+latency\""`--We get 2 results:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":7,- "params":{- "q":"\"CAS latency\""}},- "response":{"numFound":2,"start":0,"maxScore":5.937691,"docs":[- {- "id":"VDBDB1A16",- "name":"A-DATA V-Series 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - OEM",- "manu":"A-DATA Technology Inc.",- "manu_id_s":"corsair",- "cat":["electronics",- "memory"],- "features":["CAS latency 3, 2.7v"],- "popularity":0,- "inStock":true,- "store":"45.18414,-93.88141",- "manufacturedate_dt":"2006-02-13T15:26:37Z",- "payloads":"electronics|0.9 memory|0.1",- "_version_":1574100232590852096},- {- "id":"TWINX2048-3200PRO",- "name":"CORSAIR XMS 2GB (2 x 1GB) 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) Dual Channel Kit System Memory - Retail",- "manu":"Corsair Microsystems Inc.",- "manu_id_s":"corsair",- "cat":["electronics",- "memory"],- "features":["CAS latency 2, 2-3-3-6 timing, 2.75v, unbuffered, heat-spreader"],- "price":185.0,- "price_c":"185.00,USD",- "popularity":5,- "inStock":true,- "store":"37.7752,-122.4232",- "manufacturedate_dt":"2006-02-13T15:26:37Z",- "payloads":"electronics|6.0 memory|3.0",- "_version_":1574100232584560640,- "price_c____l_ns":18500}]- }}--==== Combining Searches--By default, when you search for multiple terms and/or phrases in a single query, Solr will only require that one of them is present in order for a document to match. Documents containing more terms will be sorted higher in the results list.--You can require that a term or phrase is present by prefixing it with a `+`; conversely, to disallow the presence of a term or phrase, prefix it with a `-`.--To find documents that contain both terms "electronics" and "music", enter `+electronics +music` in the `q` box in the Admin UI Query tab.--If you're using curl, you must encode the `+` character because it has a reserved purpose in URLs (encoding the space character). The encoding for `+` is `%2B` as in:--`curl "http://localhost:8983/solr/techproducts/select?q=%2Belectronics%20%2Bmusic"`--You should only get a single result.--To search for documents that contain the term "electronics" but *don't* contain the term "music", enter `+electronics -music` in the `q` box in the Admin UI. For curl, again, URL encode `+` as `%2B` as in:--`curl "http://localhost:8983/solr/techproducts/select?q=%2Belectronics+-music"`--This time you get 13 results.--==== More Information on Searching--We have only scratched the surface of the search options available in Solr. For more Solr search options, see the section on <<searching.adoc#,Searching>>.--=== Exercise 1 Wrap Up--At this point, you've seen how Solr can index data and have done some basic queries. You can choose now to continue to the next example which will introduce more Solr concepts, such as faceting results and managing your schema, or you can strike out on your own.--If you decide not to continue with this tutorial, the data we've indexed so far is likely of little value to you. You can delete your installation and start over, or you can use the `bin/solr` script we started out with to delete this collection:--`bin/solr delete -c techproducts`--And then create a new collection:--`bin/solr create -c <yourCollection> -s 2 -rf 2`--To stop both of the Solr nodes we started, issue the command:--`bin/solr stop -all`--For more information on start/stop and collection options with `bin/solr`, see <<solr-control-script-reference.adoc#,Solr Control Script Reference>>.--[[exercise-2]]-== Exercise 2: Modify the Schema and Index Films Data--This exercise will build on the last one and introduce you to the index schema and Solr's powerful faceting features.--=== Restart Solr--Did you stop Solr after the last exercise? No? Then go ahead to the next section.--If you did, though, and need to restart Solr, issue these commands:--`./bin/solr start -c -p 8983 -s example/cloud/node1/solr`--This starts the first node. When it's done start the second node, and tell it how to connect to to ZooKeeper:--`./bin/solr start -c -p 7574 -s example/cloud/node2/solr -z localhost:9983`--NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.--=== Create a New Collection--We're going to use a whole new data set in this exercise, so it would be better to have a new collection instead of trying to reuse the one we had before.--One reason for this is we're going to use a feature in Solr called "field guessing", where Solr attempts to guess what type of data is in a field while it's indexing it. It also automatically creates new fields in the schema for new fields that appear in incoming documents. This mode is called "Schemaless". We'll see the benefits and limitations of this approach to help you decide how and where to use it in your real application.--.What is a "schema" and why do I need one?-[sidebar]-****-Solr's schema is a single file (in XML) that stores the details about the fields and field types Solr is expected to understand. The schema defines not only the field or field type names, but also any modifications that should happen to a field before it is indexed. For example, if you want to ensure that a user who enters "abc" and a user who enters "ABC" can both find a document containing the term "ABC", you will want to normalize (lower-case it, in this case) "ABC" when it is indexed, and normalize the user query to be sure of a match. These rules are defined in your schema.--Earlier in the tutorial we mentioned copy fields, which are fields made up of data that originated from other fields. You can also define dynamic fields, which use wildcards (such as `*_t` or `*_s`) to dynamically create fields of a specific field type. These types of rules are also defined in the schema.-****--When you initially started Solr in the first exercise, we had a choice of a configset to use. The one we chose had a schema that was pre-defined for the data we later indexed. This time, we're going to use a configset that has a very minimal schema and let Solr figure out from the data what fields to add.--The data you're going to index is related to movies, so start by creating a collection named "films" that uses the `_default` configset:--`bin/solr create -c films -s 2 -rf 2`--Whoa, wait. We didn't specify a configset! That's fine, the `_default` is appropriately named, since it's the default and is used if you don't specify one at all.--We did, however, set two parameters `-s` and `-rf`. Those are the number of shards to split the collection across (2) and how many replicas to create (2). This is equivalent to the options we had during the interactive example from the first exercise.--You should see output like:--[source,subs="verbatim,attributes+"]------WARNING: Using _default configset. Data driven schema functionality is enabled by default, which is- NOT RECOMMENDED for production use.-- To turn it off:- bin/solr config -c films -p 7574 -action set-user-property -property update.autoCreateFields -value false--Connecting to ZooKeeper at localhost:9983 ...-INFO - 2017-07-27 15:07:46.191; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready-Uploading /{solr-docs-version}.0/server/solr/configsets/_default/conf for config films to ZooKeeper at localhost:9983--Creating new collection 'films' using command:-http://localhost:7574/solr/admin/collections?action=CREATE&name=films&numShards=2&replicationFactor=2&collection.configName=films--{- "responseHeader":{- "status":0,- "QTime":3830},- "success":{- "192.168.0.110:8983_solr":{- "responseHeader":{- "status":0,- "QTime":2076},- "core":"films_shard2_replica_n1"},- "192.168.0.110:7574_solr":{- "responseHeader":{- "status":0,- "QTime":2494},- "core":"films_shard1_replica_n2"}}}-------The first thing the command printed was a warning about not using this configset in production. That's due to some of the limitations we'll cover shortly.--Otherwise, though, the collection should be created. If we go to the Admin UI at http://localhost:8983/solr/#/films/collection-overview we should see the overview screen.--==== Preparing Schemaless for the Films Data--There are two parallel things happening with the schema that comes with the `_default` configset.--First, we are using a "managed schema", which is configured to only be modified by Solr's Schema API. That means we should not hand-edit it so there isn't confusion about which edits come from which source. Solr's Schema API allows us to make changes to fields, field types, and other types of schema rules.--Second, we are using "field guessing", which is configured in the `solrconfig.xml` file (and includes most of Solr's various configuration settings). Field guessing is designed to allow us to start using Solr without having to define all the fields we think will be in our documents before trying to index them. This is why we call it "schemaless", because you can start quickly and let Solr create fields for you as it encounters them in documents.--Sounds great! Well, not really, there are limitations. It's a bit brute force, and if it guesses wrong, you can't change much about a field after data has been indexed without having to reindex. If we only have a few thousand documents that might not be bad, but if you have millions and millions of documents, or, worse, don't have access to the original data anymore, this can be a real problem.--For these reasons, the Solr community does not recommend going to production without a schema that you have defined yourself. By this we mean that the schemaless features are fine to start with, but you should still always make sure your schema matches your expectations for how you want your data indexed and how users are going to query it.--It is possible to mix schemaless features with a defined schema. Using the Schema API, you can define a few fields that you know you want to control, and let Solr guess others that are less important or which you are confident (through testing) will be guessed to your satisfaction. That's what we're going to do here.--===== Create the "names" Field-The films data we are going to index has a small number of fields for each movie: an ID, director name(s), film name, release date, and genre(s).--If you look at one of the files in `example/films`, you'll see the first film is named _.45_, released in 2006. As the first document in the dataset, Solr is going to guess the field type based on the data in the record. If we go ahead and index this data, that first film name is going to indicate to Solr that that field type is a "float" numeric field, and will create a "name" field with a type `FloatPointField`. All data after this record will be expected to be a float.--Well, that's not going to work. We have titles like _A Mighty Wind_ and _Chicken Run_, which are strings - decidedly not numeric and not floats. If we let Solr guess the "name" field is a float, what will happen is later titles will cause an error and indexing will fail. That's not going to get us very far.--What we can do is set up the "name" field in Solr before we index the data to be sure Solr always interprets it as a string. At the command line, enter this curl command:--[source,bash]-curl -X POST -H 'Content-type:application/json' --data-binary '{"add-field": {"name":"name", "type":"text_general", "multiValued":false, "stored":true}}' http://localhost:8983/solr/films/schema--This command uses the Schema API to explicitly define a field named "name" that has the field type "text_general" (a text field). It will not be permitted to have multiple values, but it will be stored (meaning it can be retrieved by queries).--You can also use the Admin UI to create fields, but it offers a bit less control over the properties of your field. It will work for our case, though:--.Creating a field-image::images/solr-tutorial/tutorial-add-field.png[Adding a Field,640,480,pdfwidth=75%]--===== Create a "catchall" Copy Field--There's one more change to make before we start indexing.--In the first exercise when we queried the documents we had indexed, we didn't have to specify a field to search because the configuration we used was set up to copy fields into a `text` field, and that field was the default when no other field was defined in the query.--The configuration we're using now doesn't have that rule. We would need to define a field to search for every query. We can, however, set up a "catchall field" by defining a copy field that will take all data from all fields and index it into a field named `\_text_`. Let's do that now.--You can use either the Admin UI or the Schema API for this.--At the command line, use the Schema API again to define a copy field:--[source,bash]-curl -X POST -H 'Content-type:application/json' --data-binary '{"add-copy-field" : {"source":"*","dest":"_text_"}}' http://localhost:8983/solr/films/schema--In the Admin UI, choose btn:[Add Copy Field], then fill out the source and destination for your field, as in this screenshot.--.Creating a copy field-image::images/solr-tutorial/tutorial-add-copy-field.png[Adding a copy field,640,480,pdfwidth=75%]--What this does is make a copy of all fields and put the data into the "\_text_" field.--TIP: It can be very expensive to do this with your production data because it tells Solr to effectively index everything twice. It will make indexing slower, and make your index larger. With your production data, you will want to be sure you only copy fields that really warrant it for your application.--OK, now we're ready to index the data and start playing around with it.--=== Index Sample Film Data--The films data we will index is located in the `example/films` directory of your installation. It comes in three formats: JSON, XML and CSV. Pick one of the formats and index it into the "films" collection (in each example, one command is for Unix/MacOS and the other is for Windows):--.To Index JSON Format-[source,subs="verbatim,attributes+"]------bin/post -c films example/films/films.json--C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dauto example\exampledocs\post.jar example\films\*.json-------.To Index XML Format-[source,subs="verbatim,attributes+"]------bin/post -c films example/films/films.xml--C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dauto example\exampledocs\post.jar example\films\*.xml-------.To Index CSV Format-[source,subs="verbatim,attributes+"]------bin/post -c films example/films/films.csv -params "f.genre.split=true&f.directed_by.split=true&f.genre.separator=|&f.directed_by.separator=|"--C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dparams=f.genre.split=true&f.directed_by.split=true&f.genre.separator=|&f.directed_by.separator=| -Dauto example\exampledocs\post.jar example\films\*.csv-------Each command includes these main parameters:--* `-c films`: this is the Solr collection to index data to.-* `example/films/films.json` (or `films.xml` or `films.csv`): this is the path to the data file to index. You could simply supply the directory where this file resides, but since you know the format you want to index, specifying the exact file for that format is more efficient.--Note the CSV command includes extra parameters. This is to ensure multi-valued entries in the "genre" and "directed_by" columns are split by the pipe (`|`) character, used in this file as a separator. Telling Solr to split these columns this way will ensure proper indexing of the data.--Each command will produce output similar to the below seen while indexing JSON:--[source,bash,subs="verbatim,attributes"]------$ ./bin/post -c films example/films/films.json-/bin/java -classpath /solr-{solr-docs-version}.0/dist/solr-core-{solr-docs-version}.0.jar -Dauto=yes -Dc=films -Ddata=files org.apache.solr.util.SimplePostTool example/films/films.json-SimplePostTool version 5.0.0-Posting files to [base] url http://localhost:8983/solr/films/update...-Entering auto mode. File endings considered are xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log-POSTing file films.json (application/json) to [base]/json/docs-1 files indexed.-COMMITting Solr index changes to http://localhost:8983/solr/films/update...-Time spent: 0:00:00.878-------Hooray!--If you go to the Query screen in the Admin UI for films (http://localhost:8983/solr/#/films/query) and hit btn:[Execute Query] you should see 1100 results, with the first 10 returned to the screen.--Let's do a query to see if the "catchall" field worked properly. Enter "comedy" in the `q` box and hit btn:[Execute Query] again. You should see get 417 results. Feel free to play around with other searches before we move on to faceting.--[[tutorial-faceting]]-=== Faceting--One of Solr's most popular features is faceting. Faceting allows the search results to be arranged into subsets (or buckets, or categories), providing a count for each subset. There are several types of faceting: field values, numeric and date ranges, pivots (decision tree), and arbitrary query faceting.--==== Field Facets--In addition to providing search results, a Solr query can return the number of documents that contain each unique value in the whole result set.--On the Admin UI Query tab, if you check the `facet` checkbox, you'll see a few facet-related options appear:--.Facet options in the Query screen-image::images/solr-tutorial/tutorial-admin-ui-facet-options.png[Solr Quick Start: Query tab facet options]--To see facet counts from all documents (`q=\*:*`): turn on faceting (`facet=true`), and specify the field to facet on via the `facet.field` parameter. If you only want facets, and no document contents, specify `rows=0`. The `curl` command below will return facet counts for the `genre_str` field:--`curl "http://localhost:8983/solr/films/select?q=\*:*&rows=0&facet=true&facet.field=genre_str"`--In your terminal, you'll see something like:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":11,- "params":{- "q":"*:*",- "facet.field":"genre_str",- "rows":"0",- "facet":"true"}},- "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]- },- "facet_counts":{- "facet_queries":{},- "facet_fields":{- "genre_str":[- "Drama",552,- "Comedy",389,- "Romance Film",270,- "Thriller",259,- "Action Film",196,- "Crime Fiction",170,- "World cinema",167]},- "facet_ranges":{},- "facet_intervals":{},- "facet_heatmaps":{}}}--We've truncated the output here a little bit, but in the `facet_counts` section, you see by default you get a count of the number of documents using each genre for every genre in the index. Solr has a parameter `facet.mincount` that you could use to limit the facets to only those that contain a certain number of documents (this parameter is not shown in the UI). Or, perhaps you do want all the facets, and you'll let your application's front-end control how it's displayed to users.--If you wanted to control the number of items in a bucket, you could do something like this:--`curl "http://localhost:8983/solr/films/select?=&q=\*:*&facet.field=genre_str&facet.mincount=200&facet=on&rows=0"`--You should only see 4 facets returned.--There are a great deal of other parameters available to help you control how Solr constructs the facets and facet lists. We'll cover some of them in this exercise, but you can also see the section <<faceting.adoc#,Faceting>> for more detail.--==== Range Facets--For numerics or dates, it's often desirable to partition the facet counts into ranges rather than discrete values. A prime example of numeric range faceting, using the example techproducts data from our previous exercise, is `price`.-The films data includes the release date for films, and we could use that to create date range facets, which are another common use for range facets.--The Solr Admin UI doesn't yet support range facet options, so you will need to use curl or similar command line tool for the following examples.--If we construct a query that looks like this:--[source,bash]-curl 'http://localhost:8983/solr/films/select?q=*:*&rows=0'\- '&facet=true'\- '&facet.range=initial_release_date'\- '&facet.range.start=NOW-20YEAR'\- '&facet.range.end=NOW'\- '&facet.range.gap=%2B1YEAR'--This will request all films and ask for them to be grouped by year starting with 20 years ago (our earliest release date is in 2000) and ending today. Note that this query again URL encodes a `+` as `%2B`.--In the terminal you will see:--[source,json]-{- "responseHeader":{- "zkConnected":true,- "status":0,- "QTime":8,- "params":{- "facet.range":"initial_release_date",- "facet.limit":"300",- "q":"*:*",- "facet.range.gap":"+1YEAR",- "rows":"0",- "facet":"on",- "facet.range.start":"NOW-20YEAR",- "facet.range.end":"NOW"}},- "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]- },- "facet_counts":{- "facet_queries":{},- "facet_fields":{},- "facet_ranges":{- "initial_release_date":{- "counts":[- "1997-07-28T17:12:06.919Z",0,- "1998-07-28T17:12:06.919Z",0,- "1999-07-28T17:12:06.919Z",48,- "2000-07-28T17:12:06.919Z",82,- "2001-07-28T17:12:06.919Z",103,- "2002-07-28T17:12:06.919Z",131,- "2003-07-28T17:12:06.919Z",137,- "2004-07-28T17:12:06.919Z",163,- "2005-07-28T17:12:06.919Z",189,- "2006-07-28T17:12:06.919Z",92,- "2007-07-28T17:12:06.919Z",26,- "2008-07-28T17:12:06.919Z",7,- "2009-07-28T17:12:06.919Z",3,- "2010-07-28T17:12:06.919Z",0,- "2011-07-28T17:12:06.919Z",0,- "2012-07-28T17:12:06.919Z",1,- "2013-07-28T17:12:06.919Z",1,- "2014-07-28T17:12:06.919Z",1,- "2015-07-28T17:12:06.919Z",0,- "2016-07-28T17:12:06.919Z",0],- "gap":"+1YEAR",- "start":"1997-07-28T17:12:06.919Z",- "end":"2017-07-28T17:12:06.919Z"}},- "facet_intervals":{},- "facet_heatmaps":{}}}--==== Pivot Facets--Another faceting type is pivot facets, also known as "decision trees", allowing two or more fields to be nested for all the various possible combinations. Using the films data, pivot facets can be used to see how many of the films in the "Drama" category (the `genre_str` field) are directed by a director. Here's how to get at the raw data for this scenario:--`curl "http://localhost:8983/solr/films/select?q=\*:*&rows=0&facet=on&facet.pivot=genre_str,directed_by_str"`--This results in the following response, which shows a facet for each category and director combination:--[source,json]-{"responseHeader":{- "zkConnected":true,- "status":0,- "QTime":1147,- "params":{- "q":"*:*",- "facet.pivot":"genre_str,directed_by_str",- "rows":"0",- "facet":"on"}},- "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]- },- "facet_counts":{- "facet_queries":{},- "facet_fields":{},- "facet_ranges":{},- "facet_intervals":{},- "facet_heatmaps":{},- "facet_pivot":{- "genre_str,directed_by_str":[{- "field":"genre_str",- "value":"Drama",- "count":552,- "pivot":[{- "field":"directed_by_str",- "value":"Ridley Scott",- "count":5},- {- "field":"directed_by_str",- "value":"Steven Soderbergh",- "count":5},- {- "field":"directed_by_str",- "value":"Michael Winterbottom",- "count":4}}]}]}}}--We've truncated this output as well - you will see a lot of genres and directors in your screen.--=== Exercise 2 Wrap Up--In this exercise, we learned a little bit more about how Solr organizes data in the indexes, and how to work with the Schema API to manipulate the schema file. We also learned a bit about facets in Solr, including range facets and pivot facets. In both of these things, we've only scratched the surface of the available options. If you can dream it, it might be possible!--Like our previous exercise, this data may not be relevant to your needs. We can clean up our work by deleting the collection. To do that, issue this command at the command line:--`bin/solr delete -c films`--[[exercise-3]]-== Exercise 3: Index Your Own Data--For this last exercise, work with a dataset of your choice. This can be files on your local hard drive, a set of data you have worked with before, or maybe a sample of the data you intend to index to Solr for your production application.--This exercise is intended to get you thinking about what you will need to do for your application:--* What sorts of data do you need to index?-* What will you need to do to prepare Solr for your data (such as, create specific fields, set up copy fields, determine analysis rules, etc.)-* What kinds of search options do you want to provide to users?-* How much testing will you need to do to ensure everything works the way you expect?--=== Create Your Own Collection--Before you get started, create a new collection, named whatever you'd like. In this example, the collection will be named "localDocs"; replace that name with whatever name you choose if you want to.--`./bin/solr create -c localDocs -s 2 -rf 2`--Again, as we saw from Exercise 2 above, this will use the `_default` configset and all the schemaless features it provides. As we noted previously, this may cause problems when we index our data. You may need to iterate on indexing a few times before you get the schema right.--=== Indexing Ideas--Solr has lots of ways to index data. Choose one of the approaches below and try it out with your system:--Local Files with bin/post::-If you have a local directory of files, the Post Tool (`bin/post`) can index a directory of files. We saw this in action in our first exercise.-+-We used only JSON, XML and CSV in our exercises, but the Post Tool can also handle HTML, PDF, Microsoft Office formats (such as MS Word), plain text, and more.-+-In this example, assume there is a directory named "Documents" locally. To index it, we would issue a command like this (correcting the collection name after the `-c` parameter as needed):-+-`./bin/post -c localDocs ~/Documents`-+-You may get errors as it works through your documents. These might be caused by the field guessing, or the file type may not be supported. Indexing content such as this demonstrates the need to plan Solr for your data, which requires understanding it and perhaps also some trial and error.--SolrJ::-SolrJ is a Java-based client for interacting with Solr. Use <<using-solrj.adoc#,SolrJ>> for JVM-based languages or other <<client-apis.adoc#,Solr clients>> to programmatically create documents to send to Solr.--Documents Screen::-Use the Admin UI <<documents-screen.adoc#,Documents tab>> (at http://localhost:8983/solr/#/localDocs/documents) to paste in a document to be indexed, or select `Document Builder` from the `Document Type` dropdown to build a document one field at a time. Click on the btn:[Submit Document] button below the form to index your document.--=== Updating Data--You may notice that even if you index content in this tutorial more than once, it does not duplicate the results found. This is because the example Solr schema (a file named either `managed-schema` or `schema.xml`) specifies a `uniqueKey` field called `id`. Whenever you POST commands to Solr to add a document with the same value for the `uniqueKey` as an existing document, it automatically replaces it for you.--You can see that that has happened by looking at the values for `numDocs` and `maxDoc` in the core-specific Overview section of the Solr Admin UI.--`numDocs` represents the number of searchable documents in the index (and will be larger than the number of XML, JSON, or CSV files since some files contained more than one document). The `maxDoc` value may be larger as the `maxDoc` count includes logically deleted documents that have not yet been physically removed from the index. You can re-post the sample files over and over again as much as you want and `numDocs` will never increase, because the new documents will constantly be replacing the old.--Go ahead and edit any of the existing example data files, change some of the data, and re-run the PostTool (`bin/post`). You'll see your changes reflected in subsequent searches.--=== Deleting Data--If you need to iterate a few times to get your schema right, you may want to delete documents to clear out the collection and try again. Note, however, that merely removing documents doesn't change the underlying field definitions. Essentially, this will allow you to reindex your data after making changes to fields for your needs.--You can delete data by POSTing a delete command to the update URL and specifying the value of the document's unique key field, or a query that matches multiple documents (be careful with that one!). We can use `bin/post` to delete documents also if we structure the request properly.--Execute the following command to delete a specific document:--`bin/post -c localDocs -d "<delete><id>SP2514N</id></delete>"`--To delete all documents, you can use "delete-by-query" command like:--`bin/post -c localDocs -d "<delete><query>*:*</query></delete>"`--You can also modify the above to only delete documents that match a specific query.--=== Exercise 3 Wrap Up--At this point, you're ready to start working on your own.--Jump ahead to the overall <<Wrapping Up,wrap up>> when you're ready to stop Solr and remove all the examples you worked with and start fresh.== Spatial QueriesSolr has sophisticated geospatial support, including searching within a specified distance range of a given location (or within a bounding box), sorting by distance, or even boosting results by the distance.-Some of the example techproducts documents we indexed in Exercise 1 have locations associated with them to illustrate the spatial capabilities. To reindex this data, see <<index-the-techproducts-data,Exercise 1>>.+Some of the example techproducts documents we indexed in Exercise 1 have locations associated with them to illustrate the spatial capabilities.+To reindex this data, see <<tutorial-techproducts.adoc#,Exercise 1>>.To learn more about Solr's spatial capabilities, see the section <<spatial-search.adoc#,Spatial Search>>.@@ -968,7 +85,8 @@ Nice work!== Cleanup-As you work through this tutorial, you may want to stop Solr and reset the environment back to the starting point. The following command line will stop Solr and remove the directories for each of the two nodes that were created all the way back in Exercise 1:+As you work through this tutorial, you may want to stop Solr and reset the environment back to the starting point.+The following command line will stop Solr and remove the directories for each of the two nodes that were created all the way back in Exercise 1:`bin/solr stop -all ; rm -Rf example/cloud/`@@ -976,4 +94,5 @@ As you work through this tutorial, you may want to stop Solr and reset the envirThis Guide will be your best resource for learning more about Solr.-Solr also has a robust community made up of people happy to help you get started. For more information, check out the Solr website's http://solr.apache.org/resources.html[Resources page].+Solr also has a robust community made up of people happy to help you get started.+For more information, check out the Solr website's http://solr.apache.org/resources.html[Resources page].diff --git a/solr/solr-ref-guide/src/solr-upgrade-notes.adoc b/solr/solr-ref-guide/src/solr-upgrade-notes.adocindex cdbedf4b85e..8863a7bd683 100644--- a/solr/solr-ref-guide/src/solr-upgrade-notes.adoc+++ b/solr/solr-ref-guide/src/solr-upgrade-notes.adoc@@ -1,5 +1,8 @@= Solr Upgrade Notes-:page-children: major-changes-in-solr-9, major-changes-in-solr-8, major-changes-in-solr-7, major-changes-from-solr-5-to-solr-6+:page-children: major-changes-in-solr-9, \+ major-changes-in-solr-8, \+ major-changes-in-solr-7, \+ major-changes-in-solr-6:toclevels: 3// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file@@ -60,7 +63,7 @@ The old format is officially deprecated, and support for it is likely to be remoFor the time-being the old format can be created by defining a parameter `incremental=false`.Again, though, this support is likely to be removed in Solr 9.0.-More documentation on backups is available at <<making-and-restoring-backups.adoc#,Making and Restoring Backups>>.+More documentation on backups is available at <<backup-restore.adoc#,Backup and Restore>>.New Collections API commands for backups:@@ -70,7 +73,7 @@ See <<collection-management.adoc#listbackup,List Backups>> for more details.A new option for backup repository is also available in 8.9, which is to use Google Cloud Storage (GCS).This is a contrib (located in `contrib/gcs-repository`).-See <<making-and-restoring-backups.adoc#gcsbackuprepository,GCSBackupRepository>> for configuration details.+See <<backup-restore.adoc#gcsbackuprepository,GCSBackupRepository>> for configuration details.The Solr community is working to add support for S3 buckets in the near future.*Nested Docs*@@ -90,7 +93,7 @@ See <<collapse-and-expand-results.adoc#block-collapsing,Block Collapsing>> for d*In-Place Updates*A new request parameter `update.partial.requireInPlace=true` allows telling Solr to "fail fast" if all of the necessary conditions are not satisfied to allow an in-place update to succeed.-See also <<updating-parts-of-documents.adoc#in-place-updates,In-Place Updates>>.+See also <<partial-document-updates.adoc#in-place-updates,In-Place Updates>>.*Metrics History*@@ -140,7 +143,7 @@ For examples using this feature, see the section <<learning-to-rank.adoc#running*Prometheus Exporter** The `./bin` scripts included with the Prometheus Exporter contrib now allow use of custom java options with environment variables.-See the section <<monitoring-solr-with-prometheus-and-grafana.adoc#environment-variable-options,Environment Variable Options>> for more details.+See the section <<monitoring-with-prometheus-and-grafana.adoc#environment-variable-options,Environment Variable Options>> for more details.* The default Grafana dashboards now include panels for query performance monitoring.The default Prometheus Exporter configuration includes metrics like queries-per-second (QPS) and 95th percentiles (P95) to populate the new panels.* The default Prometheus Exporter configuration also includes the two new metrics mentioned in the Metrics above.@@ -184,15 +187,14 @@ The Solr community is working on pluggable API to replace this functionality, wi* Support for detecting spinning disks has been removed in LUCENE-9576. Corresponding`spins` metrics in Solr still exist but now they always return `false` and will be removed in Solr 9.0.-*Legacy Scaling (non-SolrCloud) Terminology Updated*+*User-Managed Cluster Terminology Updated** Solr has replaced the terms "master" and "slave" in the codebase and all documentation with "leader" and "follower".+-This impacts the functionality described in the section <<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>.This functionality has only changed in terms of parameter names changed, and we do not expect any back-compatibility issues on upgrade to 8.7 or even 9.0 later.+However, users should update their `solrconfig.xml` files after completing the upgrade on all nodes of a cluster.-Comparing your configuration to the updated configuration examples in <<index-replication.adoc#,Index Replication>> will show examples of what needs to change, but here are the main changes:+Comparing your configuration to the updated configuration examples in <<user-managed-index-replication.adoc#,Index Replication>> will show examples of what needs to change, but here are the main changes:+. On the replication leader, in the definition of the `/replication` request handler:.. Replace "master" with "leader".@@ -301,7 +303,7 @@ When using the `solr.xml` file that ships with 8.6, you can configure the list oPlease see `bin/solr.in.sh` or `bin\solr.in.cmd` for example usage.Using the value `*` will allow any path as in earlier versions.+-For more on this, see the section <<format-of-solr-xml.adoc#the-solr-element,Solr.xml Parameters>>.+For more on this, see the section <<configuring-solr-xml.adoc#the-solr-element,Solr.xml Parameters>>.+Windows SMB shares on the UNC format, such as `\\myhost\myshare\mypath` are now always disallowed.Please use drive letter mounts instead, i.e., `S:\mypath`.@@ -407,7 +409,7 @@ In Solr 9.0, this will be the default.Solr has two new parameters to allow you to restrict access to Solr using IP addresses. Use `SOLR_IP_WHITELIST` to configure a whitelist, and `SOLR_IP_BLACKLIST` to configure a blacklist. These properties are defined in `solr.in.sh` or `solr.in.cmd`.-See also the section <<securing-solr.adoc#enable-ip-access-control,Enable IP Access Control>>.+See also the section <<securing-solr.adoc#ip-access-control,Enable IP Access Control>>.*BlockJoin Facet Deprecation*@@ -534,8 +536,8 @@ If you are using these together, you will need to modify your queries.* SolrJ now supports the `shards.preference` parameter for single-shardscenarios to ensure multi-shard and single-shard request routing works in the same way.+-See <<using-solrj.adoc#cloud-request-routing,Cloud Request Routing>> and-<<distributed-requests.adoc#shards-preference-parameter,shards.preference Parameter>> for details.+See <<solrj.adoc#cloud-request-routing,Cloud Request Routing>> and+<<solrcloud-distributed-requests.adoc#shards-preference-parameter,shards.preference Parameter>> for details.* `QueryResponse.getExplainMap()` type has changed from `Map<String, String>` to `Map<String, Object>` in order to support structured explanations.+@@ -593,7 +595,7 @@ Since caching has a direct impact on the performance of your Solr* A new parameter, `maxIdleTime`, allows automatic eviction of cache items that have not been used in the defined amount of time. This allows the cache to release some memory and should aid those who want or need to fine-tune their caches.-See the section <<query-settings-in-solrconfig.adoc#,Query Settings in SolrConfig>> for more details about these and other cache options and parameters.+See the section <<caches-warming.adoc#,Caches and Query Warming>> for more details about these and other cache options and parameters.=== Solr 8.2@@ -610,7 +612,7 @@ It is recommended that external ensembles set up to work with Solr also be updatThis ZooKeeper release includes many new security features.In order for Solr's Admin UI to work with 3.5.5, the `zoo.cfg` file must allow access to ZooKeeper's "four-letter commands".At a minimum, `ruok`, `conf`, and `mntr` must be enabled, but other commands can optionally be enabled if you choose.-See the section <<setting-up-an-external-zookeeper-ensemble.adoc#configuration-for-a-zookeeper-ensemble,Configuration for a ZooKeeper Ensemble>> for details.+See the section <<zookeeper-ensemble.adoc#configuration-for-a-zookeeper-ensemble,Configuration for a ZooKeeper Ensemble>> for details.[WARNING]Until 8.3, https://issues.apache.org/jira/browse/SOLR-13672[SOLR-13672] causes the ZK Status screen in the Admin UI to not be able to report status. This only impacts the UI, ZooKeeper still operates correctly.@@ -631,7 +633,7 @@ Collections created with older versions will continue to work.*Distributed Tracing Support*-This release adds support for tracing requests in Solr. Please review the section <<solr-tracing.adoc#,Distributed Solr Tracing>> for details on how to configure this feature.+This release adds support for tracing requests in Solr. Please review the section <<distributed-tracing.adoc#,Distributed Tracing>> for details on how to configure this feature.=== Solr 8.1@@ -649,7 +651,7 @@ An identical parameter is available in `solrconfig.xml` for limiting the size of+If your use case demands that you a lot of OR or AND clauses in your queries, upon upgrade to 8.1 you may need to adjust the global `maxBooleanClauses` parameter since between 7.0 and 8.1 the limit was effectively unbounded.+-For more information about the new parameter, see the section <<format-of-solr-xml.adoc#global-maxbooleanclauses,Format of solr.xml: maxBooleanClauses>>.+For more information about the new parameter, see the section <<configuring-solr-xml.adoc#global-maxbooleanclauses,Format of solr.xml: maxBooleanClauses>>.*Security*@@ -702,4 +704,4 @@ Users upgrading from versions of Solr prior to 7.x are strongly encouraged to coThe upgrade from Solr 6.x to Solr 7.0 introduced several *major* changes that you should be aware of before upgrading. Please do a thorough review of the section <<major-changes-in-solr-7.adoc#,Major Changes in Solr 7>> before starting your upgrade.-A summary of the significant changes between Solr 5.x and Solr 6.0 is in the section <<major-changes-from-solr-5-to-solr-6.adoc#,Major Changes from Solr 5 to Solr 6>>.+A summary of the significant changes between Solr 5.x and Solr 6.0 is in the section <<major-changes-in-solr-6.adoc#,Major Changes in Solr 6>>.diff --git a/solr/solr-ref-guide/src/solrcloud-clusters.adoc b/solr/solr-ref-guide/src/solrcloud-clusters.adocnew file mode 100644index 00000000000..83b3fde57d5--- /dev/null+++ b/solr/solr-ref-guide/src/solrcloud-clusters.adoc@@ -0,0 +1,67 @@+= SolrCloud Clusters+:page-children: solrcloud-shards-indexing, \+ solrcloud-recoveries-and-write-tolerance, \+ solrcloud-distributed-requests, \+ aliases, \+ cluster-node-management, \+ shard-management, \+ replica-management, \+ collection-management, \+ alias-management, \+ zookeeper-ensemble, \+ zookeeper-file-management, \+ zookeeper-utilities, \+ solrcloud-with-legacy-configuration-files, \+ collections-core-admin, \+ cloud-screens+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+SolrCloud is Solr's solution for fault tolerant and high availability clusters.+It provides central coordination of cluster activities and configurations and automatic load balancing and failover.++ZooKeeper is a critical component of SolrCloud, used to manage shard and replica locations, as well as document and query routing.++****+// This tags the below list so it can be used in the parent page section list+// tag::solrcloud-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+2+^h| Functionality Overview+| <<solrcloud-shards-indexing.adoc#,SolrCloud Shards and Indexing>>: Leaders and replica types, and routing documents during indexing.+| <<solrcloud-recoveries-and-write-tolerance.adoc#,SolrCloud Recoveries and Write Tolerance>>: Recovery in a SolrCloud cluster.+| <<solrcloud-distributed-requests.adoc#,SolrCloud Distributed Requests>>: Query routing in a SolrCloud cluster.+| <<aliases.adoc#,Aliases>>: Alternative names for collections.+2+^h| Collections API+|<<cluster-node-management.adoc#,Cluster and Node Management>>: Cluster management commands of the Collections API.+| <<shard-management.adoc#,Shard Management>>: Shard management commands of the Collections API.+| <<replica-management.adoc#,Replica Management>>: Replica management commands of the Collections API.+|<<collection-management.adoc#,Collection Management>>: Collection management commands of the Collections API.+|<<alias-management.adoc#,Alias Management>>: Alias management commands of the Collections API.+|+2+^h| ZooKeeper Configuration+|<<zookeeper-ensemble.adoc#,ZooKeeper Ensemble>>: External ZooKeeper configuration.+| <<zookeeper-file-management.adoc#,ZooKeeper File Management>>: Uploading files to ZooKeeper.+| <<zookeeper-utilities.adoc#,ZooKeeper Utilities>>: ZooKeeper CLI.+| <<solrcloud-with-legacy-configuration-files.adoc#,SolrCloud with Legacy Configuration Files>>: Migration from user-managed clusters to SolrCloud.+2+^h| Admin UI+| <<collections-core-admin.adoc#,Collections / Core Admin>>: Admin UI for collections and cores.+| <<cloud-screens.adoc#,Cloud Screens>>: Admin UI for SolrCloud status and ZooKeeper files.+|===+// end::solrcloud-sections[]+****diff --git a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc b/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adocdeleted file mode 100644index dce441a040d..00000000000--- a/solr/solr-ref-guide/src/solrcloud-configuration-and-parameters.adoc+++ /dev/null@@ -1,32 +0,0 @@-= SolrCloud Configuration and Parameters-:page-children: setting-up-an-external-zookeeper-ensemble, using-zookeeper-to-manage-configuration-files, collections-api, parameter-reference, command-line-utilities, solrcloud-with-legacy-configuration-files, configsets-api-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--In this section, we'll cover the various configuration options for SolrCloud.--The following sections cover these topics:--* <<setting-up-an-external-zookeeper-ensemble.adoc#,Setting Up an External ZooKeeper Ensemble>>-* <<using-zookeeper-to-manage-configuration-files.adoc#,Using ZooKeeper to Manage Configuration Files>>-* <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>-* <<collections-api.adoc#,Collections API>>--* <<parameter-reference.adoc#,Parameter Reference>>-* <<command-line-utilities.adoc#,Command Line Utilities>>-* <<solrcloud-with-legacy-configuration-files.adoc#,SolrCloud with Legacy Configuration Files>>-* <<configsets-api.adoc#,Configsets API>>diff --git a/solr/solr-ref-guide/src/solrcloud-distributed-requests.adoc b/solr/solr-ref-guide/src/solrcloud-distributed-requests.adocnew file mode 100644index 00000000000..5755cab846b--- /dev/null+++ b/solr/solr-ref-guide/src/solrcloud-distributed-requests.adoc@@ -0,0 +1,454 @@+= SolrCloud Distributed Requests+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++When a Solr node receives a search request, the request is automatically routed to a replica of a shard that is part of the collection being searched.++The chosen replica acts as an aggregator: it creates internal requests to randomly chosen replicas of every shard in the collection, coordinates the responses, issues any subsequent internal requests as needed (for example, to refine facets values, or request additional stored fields), and constructs the final response for the client.++== Query Fault Tolerance++In a SolrCloud cluster each individual node load balances read requests across all the replicas in a collection.+You may still need a load balancer on the 'outside' that talks to the cluster.+Or you need a client which understands how to read and interact with Solr's metadata in ZooKeeper and only requests the ZooKeeper ensemble's address to discover which nodes should receive requests.+Solr provides a smart Java SolrJ client called {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[CloudSolrClient] which is capable of this.++Even if some nodes in the cluster are offline or unreachable, a Solr node will be able to correctly respond to a search request as long as it can communicate with at least one replica of every shard, or one replica of every _relevant_ shard if the user limited the search via the `shards` or `\_route_` parameters.+The more replicas there are of every shard, the more likely that the Solr cluster will be able to handle search results in the event of node failures.++=== zkConnected Parameter++A Solr node will return the results of a search request as long as it can communicate with at least one replica of every shard that it knows about, even if it can _not_ communicate with ZooKeeper at the time it receives the request.+This is normally the preferred behavior from a fault tolerance standpoint, but may result in stale or incorrect results if there have been major changes to the collection structure that the node has not been informed of via ZooKeeper (i.e., shards may have been added or removed, or split into sub-shards).++A `zkConnected` header is included in every search response indicating if the node that processed the request was connected with ZooKeeper at the time:++.Solr Response with zkConnected+[source,json]+----+{+ "responseHeader": {+ "status": 0,+ "zkConnected": true,+ "QTime": 20,+ "params": {+ "q": "*:*"+ }+ },+ "response": {+ "numFound": 107,+ "start": 0,+ "docs": [ "..." ]+ }+}+----++To prevent stale or incorrect results in the event that the request-serving node can't communicate with ZooKeeper, set the <<shards-tolerant-parameter,`shards.tolerant`>> parameter to `requireZkConnected`.+This will cause requests to fail rather than setting a `zkConnected` header to `false`.++=== shards.tolerant Parameter++In the event that one or more shards queried are unavailable, then Solr's default behavior is to fail the request.+However, there are many use-cases where partial results are acceptable and so Solr provides a boolean `shards.tolerant` parameter (default `false`).+In addition to `true` and `false`, `shards.tolerant` may also be set to `requireZkConnected` - see below.++If `shards.tolerant=true` then partial results may be returned.+If the returned response does not contain results from all the appropriate shards then the response header contains a special flag called `partialResults`.++If `shards.tolerant=requireZkConnected` and the node serving the search request cannot communicate with ZooKeeper, the request will fail, rather than returning potentially stale or incorrect results.+This will also cause requests to fail when one or more queried shards are completely unavailable, just like when `shards.tolerant=false`.++// TODO: add section on shards.info with examples+The client can specify `shards.info` along with the `shards.tolerant` parameter to retrieve more fine-grained details.++Example response with `partialResults` flag set to `true`:++.Solr Response with partialResults+[source,json]+----+{+ "responseHeader": {+ "status": 0,+ "zkConnected": true,+ "partialResults": true,+ "QTime": 20,+ "params": {+ "q": "*:*"+ }+ },+ "response": {+ "numFound": 77,+ "start": 0,+ "docs": [ "..." ]+ }+}+----++=== distrib.singlePass Parameter++If set to `true`, the `distrib.singlePass` parameter changes the distributed search algorithm to fetch all requested stored fields from each shard in the first phase itself.+This eliminates the need for making a second request to fetch the stored fields.++This can be faster when requesting a very small number of fields containing small values.+However, if large fields are requested or if a lot of fields are requested then the overhead of fetching them over the network from all shards can make the request slower as compared to the normal distributed search path.++Note that this optimization only applies to distributed search.+Certain features such as faceting may make additional network requests for refinements, etc.++== Routing Queries++There are several ways to control how queries are routed.++=== Limiting Which Shards are Queried++While one of the advantages of using SolrCloud is the ability to query very large collections distributed across various shards, in some cases you may have configured Solr so you know <<solrcloud-shards-indexing.adoc#document-routing,you are only interested in results from a specific subset of shards>>.+You have the option of searching over all of your data or just parts of it.++Because SolrCloud automatically load balances queries, a query across all shards for a collection is simply a query that does not define a `shards` parameter:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*+----++This is in contrast to user-managed clusters, where the `shards` parameter is required in order to distribute the query.++To limit the query to just one shard, use the `shards` parameter to specify the shard by its logical ID, as in:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1+----++If you want to search a group of shards, you can specify each shard separated by a comma in one request:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1,shard2+----++In both of the above examples, while only the specific shards are queried, any random replica of the shard will get the request.++You could instead specify a list of replicas you wish to use in place of a shard IDs by separating the replica IDs with commas:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=localhost:7574/solr/gettingstarted,localhost:8983/solr/gettingstarted+----++Or you can specify a list of replicas to choose from for a single shard (for load balancing purposes) by using the pipe symbol (|) between different replica IDs:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=localhost:7574/solr/gettingstarted|localhost:7500/solr/gettingstarted+----++Finally, you can specify a list of shards (separated by commas) each defined by a list of replicas (separated by pipes).++In the following example, 2 shards are queried, the first being a random replica from shard1, the second being a random replica from the explicit pipe delimited list:++[source,text]+----+http://localhost:8983/solr/gettingstarted/select?q=*:*&shards=shard1,localhost:7574/solr/gettingstarted|localhost:7500/solr/gettingstarted+----++=== shards.preference Parameter++Solr allows you to pass an optional string parameter named `shards.preference` to indicate that a distributed query should sort the available replicas in the given order of precedence within each shard.++The syntax is: `shards.preference=_property_:__value__`. The order of the properties and the values are significant: the first one is the primary sort, the second is secondary, etc.++IMPORTANT: `shards.preference` is supported for single shard scenarios only when using the SolrJ clients.+Queries that do not use the SolrJ client cannot use `shards.preference` in single shard collections.++The properties that can be specified are as follows:++`replica.type`::+One or more replica types that are preferred. Any combination of `PULL`, `TLOG` and `NRT` is allowed.++`replica.location`::+One or more replica locations that are preferred.+++A location starts with `http://hostname:port`. Matching is done for the given string as a prefix, so it's possible to e.g., leave out the port.+++A special value `local` may be used to denote any local replica running on the same Solr instance as the one handling the query. This is useful when a query requests many fields or large fields to be returned per document because it avoids moving large amounts of data over the network when it is available locally. In addition, this feature can be useful for minimizing the impact of a problematic replica with degraded performance, as it reduces the likelihood that the degraded replica will be hit by other healthy replicas.+++The value of `replica.location:local` diminishes as the number of shards (that have no locally-available replicas) in a collection increases because the query controller will have to direct the query to non-local replicas for most of the shards.+++In other words, this feature is mostly useful for optimizing queries directed towards collections with a small number of shards and many replicas.+++Also, this option should only be used if you are load balancing requests across all nodes that host replicas for the collection you are querying, as Solr's `CloudSolrClient` will do. If not load-balancing, this feature can introduce a hotspot in the cluster since queries won't be evenly distributed across the cluster.++`replica.base`::+Applied after sorting by inherent replica attributes, this property defines a fallback ordering among sets of preference-equivalent replicas; if specified, only one value may be specified for this property, and it must be specified last.+++`random`, the default, randomly shuffles replicas for each request. This distributes requests evenly, but can result in sub-optimal cache usage for shards with replication factor > 1.+++`stable:dividend:_paramName_` parses an integer from the value associated with the given parameter name; this integer is used as the dividend (mod equivalent replica count) to determine (via list rotation) order of preference among equivalent replicas.+++`stable[:hash[:_paramName_]]` the string value associated with the given parameter name is hashed to a dividend that is used to determine replica preference order (analogous to the explicit `dividend` property above); `_paramName_` defaults to `q` if not specified, providing stable routing keyed to the string value of the "main query".+Note that this may be inappropriate for some use cases (e.g., static main queries that leverage parameter substitution)++`replica.leader`::+Prefer replicas based on their leader status, set to either `true` or `false`.+++Consider a shard with two `TLOG` replicas and four `PULL` replicas (six replicas in total, one of which is the leader).+With `shards.preference=replica.leader:false`, 5 out of 6 replicas will be preferred.+Contrast this with `shards.preference=replica.type:PULL` where only 4 of 6 replicas will be preferred.+++Note that the non-leader `TLOG` replica behaves like a `PULL` replica from a search perspective; it pulls index updates from the leader just like a `PULL` replica and does not perform soft-commits.+The difference is that the non-leader `TLOG` replica also captures updates in its TLOG, so that it is a candidate to replace the current leader if it is lost.++`node.sysprop`::+Query will be routed to nodes with same defined system properties as the current one.+For example, if you start Solr nodes on different racks, you'll want to identify those nodes by a <<property-substitution.adoc#jvm-system-properties,system property>> (e.g., `-Drack=rack1`).+Then, queries can contain `shards.preference=node.sysprop:sysprop.rack`, to make sure you always hit shards with the same value of `rack`.++*Examples*:++* Prefer stable routing (keyed to client "sessionId" parameter) among otherwise equivalent replicas:+++[source,text]+shards.preference=replica.base:stable:hash:sessionId&sessionId=abc123++* Prefer PULL replicas:+++[source,text]+shards.preference=replica.type:PULL++* Prefer PULL replicas, or TLOG replicas if PULL replicas are not available:+++[source,text]+shards.preference=replica.type:PULL,replica.type:TLOG++* Prefer any local replicas:+++[source,text]+shards.preference=replica.location:local++* Prefer any replicas on a host called "server1" with "server2" as the secondary option:+++[source,text]+shards.preference=replica.location:http://server1,replica.location:http://server2++* Prefer PULL replicas if available, otherwise TLOG replicas, and local replicas among those:+++[source,text]+shards.preference=replica.type:PULL,replica.type:TLOG,replica.location:local++* Prefer local replicas, and among them PULL replicas when available, otherwise TLOG replicas:+++[source,text]+shards.preference=replica.location:local,replica.type:PULL,replica.type:TLOG++* Prefer any replica that is not a leader:+++[source,text]+`shards.preference=replica.leader:false`++Note that if you provide these parameters in a query string, they need to be properly URL-encoded.++=== collection Parameter++The `collection` parameter allows you to specify a collection or a number of collections on which the query should be executed.+This allows you to query multiple collections at once and the features of Solr which work in a distributed manner will work across collections.++[source,plain]+----+http://localhost:8983/solr/collection1/select?collection=collection1,collection2,collection3+----++=== \_route_ Parameter++The `\_route_` parameter can be used to specify a route key which is used to figure out the corresponding shards.+For example, if you have a document with a unique key "user1!123", then specifying the route key as "_route_=user1!" (notice the trailing '!' character) will route the request to the shard which hosts that user.+You can specify multiple route keys separated by comma.+This parameter can be leveraged when we have shard data by users.+See <<solrcloud-shards-indexing.adoc#document-routing,Document Routing>> for more information++[source,plain]+----+http://localhost:8983/solr/collection1/select?q=*:*&_route_=user1!+http://localhost:8983/solr/collection1/select?q=*:*&_route_=user1!,user2!+----++== Near Real Time (NRT) Use Cases++Near Real Time (NRT) search means that documents are available for search soon after being indexed.+NRT searching is one of the main features of SolrCloud and is rarely attempted in user-managed clusters or single-node installations.++Document durability and searchability are controlled by `commits`.+The "Near" in "Near Real Time" is configurable to meet the needs of your application.+Commits are either "hard" or "soft" and can be issued by a client (say SolrJ), via a REST call or configured to occur automatically in `solrconfig.xml`.+The recommendation usually gives is to configure your commit strategy in `solrconfig.xml` (see below) and avoid issuing commits externally.++Typically in NRT applications, hard commits are configured with `openSearcher=false`, and soft commits are configured to make documents visible for search.++When a commit occurs, various background tasks are initiated, segment merging for example.+These background tasks do not block additional updates to the index nor do they delay the availability of the documents for search.++When configuring for NRT, pay special attention to cache and autowarm settings as they can have a significant impact on NRT performance.+For extremely short autoCommit intervals, consider disabling caching and autowarming completely.++== Configuring the ShardHandlerFactory++For finer-grained control, you can directly configure and tune aspects of the concurrency and thread-pooling used within distributed search in Solr. The default configuration favors throughput over latency.++This is done by defining a `shardHandlerFactory` in the configuration for your search handler.++To add a `shardHandlerFactory` to the standard search handler, provide a configuration in `solrconfig.xml`, as in this example:++[source,xml]+----+<requestHandler name="/select" class="solr.SearchHandler">+ <!-- other params go here -->+ <shardHandlerFactory class="HttpShardHandlerFactory">+ <int name="socketTimeout">1000</int>+ <int name="connTimeout">5000</int>+ </shardHandlerFactory>+</requestHandler>+----++`HttpShardHandlerFactory` is the only `ShardHandlerFactory` implementation included out of the box with Solr.++NOTE:: The `shardHandlerFactory` is reliant on the `allowUrls` parameter configured in `solr.xml`, which controls which nodes are allowed to talk to each other.+This means that the configuration of hosts is global instead of per-core or per-collection.+See the section <<configuring-solr-xml.adoc#allow-urls, Format of solr.allowUrls>> for details.++The `HttpShardHandlerFactory` accepts the following parameters:++`socketTimeout`::+++[%autowidth,frame=none]+|===+|Optional |Default: `0`+|===+++The amount of time in milliseconds that a socket is allowed to wait.+The default is `0`, where the operating system's default will be used.++`connTimeout`::+++[%autowidth,frame=none]+|===+|Optional |Default: `0`+|===+++The amount of time in milliseconds that is accepted for binding / connecting a socket.+The default is `0`, where the operating system's default will be used.++`maxConnectionsPerHost`::+++[%autowidth,frame=none]+|===+|Optional |Default: `100000`+|===+++The maximum number of concurrent connections that is made to each individual shard in a distributed search.++`corePoolSize`::+++[%autowidth,frame=none]+|===+|Optional |Default: `0`+|===+++The retained lowest limit on the number of threads used in coordinating distributed search.++`maximumPoolSize`::+++[%autowidth,frame=none]+|===+|Optional |Default: `Integer.MAX_VALUE`+|===+++The maximum number of threads used for coordinating distributed search.++`maxThreadIdleTime`::+++[%autowidth,frame=none]+|===+|Optional |Default: `5`+|===+++The amount of time in seconds to wait for before threads are scaled back in response to a reduction in load.++`sizeOfQueue`::+++[%autowidth,frame=none]+|===+|Optional |Default: `-1`+|===+++If specified, the thread pool will use a backing queue instead of a direct handoff buffer.+High throughput systems will want to configure this to be a direct hand off (with `-1`).+Systems that desire better latency will want to configure a reasonable size of queue to handle variations in requests.++`fairnessPolicy`::+++[%autowidth,frame=none]+|===+|Optional |Default: `false`+|===+++Chooses the JVM specifics dealing with fair policy queuing.+If enabled distributed searches will be handled in a first-in-first-out fashion at a cost to throughput.+If disabled throughput will be favored over latency.+++[[distributedidf]]+== Distributed Inverse Document Frequency (IDF)++Document and term statistics are needed in order to calculate relevancy.+In a distributed system, these statistics can vary from node to node, introducing bias or inaccuracies into scoring calculations.++Solr stores the document and term statistics in a cache called the `statsCache`.+There are four implementations out of the box when it comes to document statistics calculation:++* `LocalStatsCache`: This uses only local term and document statistics to compute relevance.+In cases with uniform term distribution across shards, this works reasonably well.+This option is the default if no `<statsCache>` is configured.++* `ExactStatsCache`: This implementation uses global values (across the collection) for document frequency.+It's recommended to choose this option if precise scoring across nodes is important for your implementation.++* `ExactSharedStatsCache`: This is like the `ExactStatsCache` in its functionality but the global stats are reused for subsequent requests with the same terms.++* `LRUStatsCache`: This implementation uses a least-recently-used cache to hold global stats, which are shared between requests.++The implementation can be selected by setting `<statsCache>` in `solrconfig.xml`.+For example, the following line makes Solr use the `ExactStatsCache` implementation:++[source,xml]+----+<statsCache class="org.apache.solr.search.stats.ExactStatsCache"/>+----++== Avoiding Distributed Deadlock++Each shard serves top-level query requests and then makes sub-requests to all of the other shards.+Care should be taken to ensure that the max number of threads serving HTTP requests is greater than the possible number of requests from both top-level clients and other shards.+If this is not the case, the configuration may result in a distributed deadlock.++For example, a deadlock might occur in the case of two shards, each with just a single thread to service HTTP requests.+Both threads could receive a top-level request concurrently, and make sub-requests to each other.+Because there are no more remaining threads to service requests, the incoming requests will be blocked until the other pending requests are finished, but they will not finish since they are waiting for the sub-requests.+By ensuring that Solr is configured to handle a sufficient number of threads, you can avoid deadlock situations like this.++== Distributed Tracing and Debugging++The `debug` parameter with a value of `track` can be used to trace the request as well as find timing information for each phase of a distributed request.diff --git a/solr/solr-ref-guide/src/solrcloud-query-routing-and-read-tolerance.adoc b/solr/solr-ref-guide/src/solrcloud-query-routing-and-read-tolerance.adocdeleted file mode 100644index 7aaa30427d8..00000000000--- a/solr/solr-ref-guide/src/solrcloud-query-routing-and-read-tolerance.adoc+++ /dev/null@@ -1,134 +0,0 @@-= SolrCloud Query Routing And Read Tolerance-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--SolrCloud is highly available and fault tolerant in reads and writes.---== Read Side Fault Tolerance--In a SolrCloud cluster each individual node load balances read requests across all the replicas in a collection. You still need a load balancer on the 'outside' that talks to the cluster, or you need a smart client which understands how to read and interact with Solr's metadata in ZooKeeper and only requests the ZooKeeper ensemble's address to start discovering to which nodes it should send requests. (Solr provides a smart Java SolrJ client called {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[CloudSolrClient].)--Even if some nodes in the cluster are offline or unreachable, a Solr node will be able to correctly respond to a search request as long as it can communicate with at least one replica of every shard, or one replica of every _relevant_ shard if the user limited the search via the `shards` or `\_route_` parameters. The more replicas there are of every shard, the more likely that the Solr cluster will be able to handle search results in the event of node failures.--== Query Parameters for Query Routing--=== zkConnected Parameter--A Solr node will return the results of a search request as long as it can communicate with at least one replica of every shard that it knows about, even if it can _not_ communicate with ZooKeeper at the time it receives the request. This is normally the preferred behavior from a fault tolerance standpoint, but may result in stale or incorrect results if there have been major changes to the collection structure that the node has not been informed of via ZooKeeper (i.e., shards may have been added or removed, or split into sub-shards).--A `zkConnected` header is included in every search response indicating if the node that processed the request was connected with ZooKeeper at the time:--.Solr Response with zkConnected-[source,json]------{- "responseHeader": {- "status": 0,- "zkConnected": true,- "QTime": 20,- "params": {- "q": "*:*"- }- },- "response": {- "numFound": 107,- "start": 0,- "docs": [ "..." ]- }-}-------To prevent stale or incorrect results in the event that the request-serving node can't communicate with ZooKeeper, set the <<shards-tolerant-parameter,`shards.tolerant`>> parameter to `requireZkConnected`. This will cause requests to fail rather than setting a `zkConnected` header to `false`.--=== shards Parameter--By default, SolrCloud will run searches on all shards and combine the results if the shards parameter is not specified. You can specify one or more shard names as the value of the `shards` parameter to limit the shards that you want to search against.--[source,plain]------http://localhost:8983/solr/collection1/select?q=*:*&shards=shard1-http://localhost:8983/solr/collection1/select?q=*:*&shards=shard2,shard3-------=== shards.tolerant Parameter--In the event that one or more shards queried are unavailable, then Solr's default behavior is to fail the request. However, there are many use-cases where partial results are acceptable and so Solr provides a boolean `shards.tolerant` parameter (default `false`). In addition to `true` and `false`, `shards.tolerant` may also be set to `requireZkConnected` - see below.--If `shards.tolerant=true` then partial results may be returned. If the returned response does not contain results from all the appropriate shards then the response header contains a special flag called `partialResults`.--If `shards.tolerant=requireZkConnected` and the node serving the search request cannot communicate with ZooKeeper, the request will fail, rather than returning potentially stale or incorrect results. This will also cause requests to fail when one or more queried shards are completely unavailable, just like when `shards.tolerant=false`.--The client can specify '<<distributed-search-with-index-sharding.adoc#,`shards.info`>>' along with the `shards.tolerant` parameter to retrieve more fine-grained details.--Example response with `partialResults` flag set to `true`:--.Solr Response with partialResults-[source,json]------{- "responseHeader": {- "status": 0,- "zkConnected": true,- "partialResults": true,- "QTime": 20,- "params": {- "q": "*:*"- }- },- "response": {- "numFound": 77,- "start": 0,- "docs": [ "..." ]- }-}-------=== collection Parameter--The `collection` parameter allows you to specify a collection or a number of collections on which the query should be executed. This allows you to query multiple collections at once and all the feature of Solr which work in a distributed manner can work across collections.--[source,plain]------http://localhost:8983/solr/collection1/select?collection=collection1,collection2,collection3-------=== \_route_ Parameter--The `\_route_` parameter can be used to specify a route key which is used to figure out the corresponding shards. For example, if you have a document with a unique key "user1!123", then specifying the route key as "_route_=user1!" (notice the trailing '!' character) will route the request to the shard which hosts that user. You can specify multiple route keys separated by comma.-This parameter can be leveraged when we have shard data by users. See '<<shards-and-indexing-data-in-solrcloud.adoc#document-routing,`Document Routing`>>' for more information--[source,plain]------http://localhost:8983/solr/collection1/select?q=*:*&_route_=user1!-http://localhost:8983/solr/collection1/select?q=*:*&_route_=user1!,user2!-------== Distributed Tracing and Debugging--The `debug` parameter with a value of `track` can be used to trace the request as well as find timing information for each phase of a distributed request.---== Optimization--=== distrib.singlePass Parameter--If set to `true`, the `distrib.singlePass` parameter changes the distributed search algorithm to fetch all requested stored fields from each shard in the first phase itself. This eliminates the need for making a second request to fetch the stored fields.--This can be faster when requesting a very small number of fields containing small values. However, if large fields are requested or if a lot of fields are requested then the overhead of fetching them over the network from all shards can make the request slower as compared to the normal distributed search path.--Note that this optimization only applies to distributed search. Certain features such as faceting may make additional network requests for refinements, etc.diff --git a/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc b/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adocindex f515f870ff2..94ab897427e 100644--- a/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc+++ b/solr/solr-ref-guide/src/solrcloud-recoveries-and-write-tolerance.adoc@@ -20,7 +20,7 @@ SolrCloud is highly available and fault tolerant in reads and writes.== Write Side Fault Tolerance-SolrCloud is designed to replicate documents to ensure redundancy for your data, and enable you to send update requests to any node in the cluster. That node will determine if it hosts the leader for the appropriate shard, and if not it will forward the request to the the leader, which will then forward it to all existing replicas, using versioning to make sure every replica has the most up-to-date version. If the leader goes down, another replica can take its place. This architecture enables you to be certain that your data can be recovered in the event of a disaster, even if you are using <<near-real-time-searching.adoc#,Near Real Time Searching>>.+SolrCloud is designed to replicate documents to ensure redundancy for your data, and enable you to send update requests to any node in the cluster. That node will determine if it hosts the leader for the appropriate shard, and if not it will forward the request to the the leader, which will then forward it to all existing replicas, using versioning to make sure every replica has the most up-to-date version. If the leader goes down, another replica can take its place. This architecture enables you to be certain that your data can be recovered in the event of a disaster.=== Recoverydiff --git a/solr/solr-ref-guide/src/solrcloud-resilience.adoc b/solr/solr-ref-guide/src/solrcloud-resilience.adocdeleted file mode 100644index 4010a53dbd2..00000000000--- a/solr/solr-ref-guide/src/solrcloud-resilience.adoc+++ /dev/null@@ -1,26 +0,0 @@-= SolrCloud Resilience-:page-children: solrcloud-recoveries-and-write-tolerance, solrcloud-query-routing-and-read-tolerance-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--In this section, we'll cover how does Solr handle reads and writes when all the nodes in the cluster are not healthy--The following sections cover these topics:--* <<solrcloud-recoveries-and-write-tolerance.adoc#,SolrCloud Recoveries and Write Tolerance>>-* <<solrcloud-query-routing-and-read-tolerance.adoc#,SolrCloud Query Routing And Read Tolerance>>-diff --git a/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc b/solr/solr-ref-guide/src/solrcloud-shards-indexing.adocsimilarity index 98%rename from solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adocrename to solr/solr-ref-guide/src/solrcloud-shards-indexing.adocindex c76429a3ebf..2f360bf2f99 100644--- a/solr/solr-ref-guide/src/shards-and-indexing-data-in-solrcloud.adoc+++ b/solr/solr-ref-guide/src/solrcloud-shards-indexing.adoc@@ -1,4 +1,4 @@-= Shards and Indexing Data in SolrCloud+= SolrCloud Shards and Indexing// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -88,7 +88,7 @@ If the PULL replica dies or is unreachable for any other reason, it won't be que=== Queries with Preferred Replica Types-By default all replicas serve queries. See the section <<distributed-requests.adoc#shards-preference-parameter,shards.preference Parameter>> for details on how to indicate preferred replica types for queries.+By default all replicas serve queries. See the section <<solrcloud-distributed-requests.adoc#shards-preference-parameter,shards.preference Parameter>> for details on how to indicate preferred replica types for queries.== Document Routingdiff --git a/solr/solr-ref-guide/src/solrcloud-with-legacy-configuration-files.adoc b/solr/solr-ref-guide/src/solrcloud-with-legacy-configuration-files.adocindex 743814405fa..1f765e6713b 100644--- a/solr/solr-ref-guide/src/solrcloud-with-legacy-configuration-files.adoc+++ b/solr/solr-ref-guide/src/solrcloud-with-legacy-configuration-files.adoc@@ -16,7 +16,7 @@// specific language governing permissions and limitations// under the License.-If you are migrating from a non-SolrCloud environment to SolrCloud, this information may be helpful.+If you are migrating from a user-managed cluster to SolrCloud, this information may be helpful.All of the required configuration is already set up in the sample configurations shipped with Solr. You only need to add the following if you are migrating old configuration files. Do not remove these files and parameters from a new Solr instance if you intend to use Solr in SolrCloud mode.diff --git a/solr/solr-ref-guide/src/solrcloud.adoc b/solr/solr-ref-guide/src/solrcloud.adocdeleted file mode 100644index 3327744e152..00000000000--- a/solr/solr-ref-guide/src/solrcloud.adoc+++ /dev/null@@ -1,45 +0,0 @@-= SolrCloud-:page-children: getting-started-with-solrcloud, how-solrcloud-works, solrcloud-resilience, solrcloud-configuration-and-parameters-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--Apache Solr includes the ability to set up a cluster of Solr servers that combines fault tolerance and high availability. Called *SolrCloud*, these capabilities provide distributed indexing and search capabilities, supporting the following features:--* Central configuration for the entire cluster-* Automatic load balancing and fail-over for queries-* ZooKeeper integration for cluster coordination and configuration.--SolrCloud is flexible distributed search and indexing, without a leader node to allocate nodes, shards and replicas. Instead, Solr uses ZooKeeper to manage these locations, depending on configuration files and schemas. Queries and updates can be sent to any server. Solr will use the information in the ZooKeeper database to figure out which servers need to handle the request.--In this section, we'll cover everything you need to know about using Solr in SolrCloud mode. We've split up the details into the following topics:--* <<getting-started-with-solrcloud.adoc#,Getting Started with SolrCloud>>-* <<how-solrcloud-works.adoc#,How SolrCloud Works>>-** <<shards-and-indexing-data-in-solrcloud.adoc#,Shards and Indexing Data in SolrCloud>>-** <<distributed-requests.adoc#,Distributed Requests>>-* <<solrcloud-resilience.adoc#,SolrCloud Resilience>>-** <<solrcloud-recoveries-and-write-tolerance.adoc#,SolrCloud Recoveries and Write Tolerance>>-** <<solrcloud-query-routing-and-read-tolerance.adoc#,SolrCloud Query Routing And Read Tolerance>>-* <<solrcloud-configuration-and-parameters.adoc#,SolrCloud Configuration and Parameters>>-** <<setting-up-an-external-zookeeper-ensemble.adoc#,Setting Up an External ZooKeeper Ensemble>>-** <<using-zookeeper-to-manage-configuration-files.adoc#,Using ZooKeeper to Manage Configuration Files>>-** <<zookeeper-access-control.adoc#,ZooKeeper Access Control>>-** <<collections-api.adoc#,Collections API>>-** <<parameter-reference.adoc#,Parameter Reference>>-** <<command-line-utilities.adoc#,Command Line Utilities>>-** <<solrcloud-with-legacy-configuration-files.adoc#,SolrCloud with Legacy Configuration Files>>-** <<configsets-api.adoc#,Configsets API>>diff --git a/solr/solr-ref-guide/src/using-solrj.adoc b/solr/solr-ref-guide/src/solrj.adocsimilarity index 70%rename from solr/solr-ref-guide/src/using-solrj.adocrename to solr/solr-ref-guide/src/solrj.adocindex 620123210fe..6098a28fb41 100644--- a/solr/solr-ref-guide/src/using-solrj.adoc+++ b/solr/solr-ref-guide/src/solrj.adoc@@ -1,4 +1,4 @@-= Using SolrJ+= SolrJ:solr-root-path: ../../:example-source-dir: {solr-root-path}solrj/src/test/org/apache/solr/client/ref_guide_examples/// Licensed to the Apache Software Foundation (ASF) under one@@ -18,11 +18,14 @@// specific language governing permissions and limitations// under the License.-{solr-javadocs}/solrj/[SolrJ] is an API that makes it easy for applications written in Java (or any language based on the JVM) to talk to Solr. SolrJ hides a lot of the details of connecting to Solr and allows your application to interact with Solr with simple high-level methods. SolrJ supports most Solr APIs, and is highly configurable.+{solr-javadocs}/solrj/[SolrJ] is an API that makes it easy for applications written in Java (or any language based on the JVM) to talk to Solr.+SolrJ hides a lot of the details of connecting to Solr and allows your application to interact with Solr with simple high-level methods.+SolrJ supports most Solr APIs, and is highly configurable.== Building and Running SolrJ Applications-The SolrJ API ships with Solr, so you do not have to download or install anything else. But you will need to configure your build to include SolrJ and its dependencies.+The SolrJ API ships with Solr, so you do not have to download or install anything else.+But you will need to configure your build to include SolrJ and its dependencies.=== Common Build Systems@@ -64,7 +67,9 @@ At build time, all that is required is the SolrJ jar itself: `solr-solrj-{solr-djavac -cp .:$SOLR_HOME/dist/solr-solrj-{solr-docs-version}.0.jar ...-----At runtime, you need a few of SolrJ's dependencies, in addition to SolrJ itself. For convenience, these dependencies are made available in the `dist/solrj-lib` directory. Run your project with a classpath like:+At runtime, you need a few of SolrJ's dependencies, in addition to SolrJ itself.+For convenience, these dependencies are made available in the `dist/solrj-lib` directory.+Run your project with a classpath like:[source,bash,subs="attributes"]----@@ -77,7 +82,8 @@ If you are worried about the SolrJ libraries expanding the size of your client aFor all its flexibility, SolrJ is built around a few simple interfaces.-All requests to Solr are sent by a {solr-javadocs}/solrj/org/apache/solr/client/solrj/SolrClient.html[`SolrClient`]. SolrClient's are the main workhorses at the core of SolrJ. They handle the work of connecting to and communicating with Solr, and are where most of the user configuration happens.+All requests to Solr are sent by a {solr-javadocs}/solrj/org/apache/solr/client/solrj/SolrClient.html[`SolrClient`]. SolrClient's are the main workhorses at the core of SolrJ.+They handle the work of connecting to and communicating with Solr, and are where most of the user configuration happens.Requests are sent in the form of {solr-javadocs}/solrj/org/apache/solr/client/solrj/SolrRequest.html[`SolrRequests`], and are returned as {solr-javadocs}/solrj/org/apache/solr/client/solrj/SolrResponse.html[`SolrResponses`].@@ -85,30 +91,43 @@ Requests are sent in the form of {solr-javadocs}/solrj/org/apache/solr/client/so`SolrClient` has a few concrete implementations, each geared towards a different usage-pattern or resiliency model:-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html[`HttpSolrClient`] - geared towards query-centric workloads, though also a good general-purpose client. Communicates directly with a single Solr node.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/HttpSolrClient.html[`HttpSolrClient`] - geared towards query-centric workloads, though also a good general-purpose client.+Communicates directly with a single Solr node.- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/Http2SolrClient.html[`Http2SolrClient`] - async, non-blocking and general-purpose client that leverage HTTP/2. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/LBHttpSolrClient.html[`LBHttpSolrClient`] - balances request load across a list of Solr nodes. Adjusts the list of "in-service" nodes based on node health.-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.html[`LBHttp2SolrClient`] - just like `LBHttpSolrClient` but using `Http2SolrClient` instead. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[`CloudSolrClient`] - geared towards communicating with SolrCloud deployments. Uses already-recorded ZooKeeper state to discover and route requests to healthy Solr nodes.-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.html[`ConcurrentUpdateSolrClient`] - geared towards indexing-centric workloads. Buffers documents internally before sending larger batches to Solr.-- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.html[`ConcurrentUpdateHttp2SolrClient`] - just like `ConcurrentUpdateSolrClient` but using `Http2SolrClient` instead. This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/LBHttpSolrClient.html[`LBHttpSolrClient`] - balances request load across a list of Solr nodes.+Adjusts the list of "in-service" nodes based on node health.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.html[`LBHttp2SolrClient`] - just like `LBHttpSolrClient` but using `Http2SolrClient` instead.+This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/CloudSolrClient.html[`CloudSolrClient`] - geared towards communicating with SolrCloud deployments.+Uses already-recorded ZooKeeper state to discover and route requests to healthy Solr nodes.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.html[`ConcurrentUpdateSolrClient`] - geared towards indexing-centric workloads.+Buffers documents internally before sending larger batches to Solr.+- {solr-javadocs}/solrj/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.html[`ConcurrentUpdateHttp2SolrClient`] - just like `ConcurrentUpdateSolrClient` but using `Http2SolrClient` instead.+This class is experimental therefore its API's might change or be removed in minor versions of SolrJ.=== Common Configuration Options-Most SolrJ configuration happens at the `SolrClient` level. The most common/important of these are discussed below. For comprehensive information on how to tweak your `SolrClient`, see the Javadocs for the involved client, and its corresponding builder object.+Most SolrJ configuration happens at the `SolrClient` level.+The most common/important of these are discussed below.+For comprehensive information on how to tweak your `SolrClient`, see the Javadocs for the involved client, and its corresponding builder object.==== Base URLs-Most `SolrClient` implementations (except for `CloudSolrClient` and `Http2SolrClient`) require users to specify one or more Solr base URLs, which the client then uses to send HTTP requests to Solr. The path users include on the base URL they provide has an effect on the behavior of the created client from that point on.+Most `SolrClient` implementations (except for `CloudSolrClient` and `Http2SolrClient`) require users to specify one or more Solr base URLs, which the client then uses to send HTTP requests to Solr.+The path users include on the base URL they provide has an effect on the behavior of the created client from that point on.-. A URL with a path pointing to a specific core or collection (e.g., `\http://hostname:8983/solr/core1`). When a core or collection is specified in the base URL, subsequent requests made with that client are not required to re-specify the affected collection. However, the client is limited to sending requests to that core/collection, and can not send requests to any others.+. A URL with a path pointing to a specific core or collection (e.g., `\http://hostname:8983/solr/core1`). When a core or collection is specified in the base URL, subsequent requests made with that client are not required to re-specify the affected collection.+However, the client is limited to sending requests to that core/collection, and can not send requests to any others.. A URL pointing to the root Solr path (e.g., `\http://hostname:8983/solr`). When no core or collection is specified in the base URL, requests can be made to any core/collection, but the affected core/collection must be specified on all requests.-Generally speaking, if your `SolrClient` will only be used on a single core/collection, including that entity in the path is the most convenient. Where more flexibility is required, the collection/core should be excluded.+Generally speaking, if your `SolrClient` will only be used on a single core/collection, including that entity in the path is the most convenient.+Where more flexibility is required, the collection/core should be excluded.==== Base URLs of Http2SolrClient-The `Http2SolrClient` manages connections to different nodes efficiently. `Http2SolrClient`-does not require a `baseUrl`. In case a `baseUrl` is not provided, then `SolrRequest.basePath` must be set, so-`Http2SolrClient` knows which nodes to send requests to. If not an `IllegalArgumentException` will be thrown.+The `Http2SolrClient` manages connections to different nodes efficiently.+`Http2SolrClient` does not require a `baseUrl`.+In case a `baseUrl` is not provided, then `SolrRequest.basePath` must be set, so+`Http2SolrClient` knows which nodes to send requests to.+If not an `IllegalArgumentException` will be thrown.==== Base URLs of CloudSolrClient@@ -134,7 +153,8 @@ include::{example-source-dir}UsingSolrJRefGuideExamplesTest.java[tag=solrj-cloud==== Timeouts-All `SolrClient` implementations allow users to specify the connection and read timeouts for communicating with Solr. These are provided at client creation time, as in the example below:+All `SolrClient` implementations allow users to specify the connection and read timeouts for communicating with Solr.+These are provided at client creation time, as in the example below:[source,java,indent=0]----@@ -151,14 +171,16 @@ processing time of the largest update request.=== Cloud Request Routing-The SolrJ `CloudSolrClient` implementations (`CloudSolrClient` and `CloudHttp2SolrClient`) respect the <<distributed-requests.adoc#shards-preference-parameter,shards.preference parameter>>.+The SolrJ `CloudSolrClient` implementations (`CloudSolrClient` and `CloudHttp2SolrClient`) respect the <<solrcloud-distributed-requests.adoc#shards-preference-parameter,shards.preference parameter>>.Therefore requests sent to single-sharded collections, using either of the above clients, will route requests the same way that distributed requests are routed to individual shards.If no `shards.preference` parameter is provided, the clients will default to sorting replicas randomly.For update requests, while the replicas are sorted in the order defined by the request, leader replicas will always be sorted first.== Querying in SolrJ-`SolrClient` has a number of `query()` methods for fetching results from Solr. Each of these methods takes in a `SolrParams`,an object encapsulating arbitrary query-parameters. And each method outputs a `QueryResponse`, a wrapper which can be used to access the result documents and other related metadata.+`SolrClient` has a number of `query()` methods for fetching results from Solr.+ Each of these methods takes in a `SolrParams`,an object encapsulating arbitrary query-parameters.+ And each method outputs a `QueryResponse`, a wrapper which can be used to access the result documents and other related metadata.The following snippet uses a SolrClient to query Solr's "techproducts" example collection, and iterate over the results.@@ -167,7 +189,8 @@ The following snippet uses a SolrClient to query Solr's "techproducts" example cinclude::{example-source-dir}UsingSolrJRefGuideExamplesTest.java[tag=solrj-query-with-raw-solrparams]-----`SolrParams` has a `SolrQuery` subclass, which provides some convenience methods that greatly simplifies query creation. The following snippet shows how the query from the previous example can be built using some of the convenience methods in `SolrQuery`:+`SolrParams` has a `SolrQuery` subclass, which provides some convenience methods that greatly simplifies query creation.+The following snippet shows how the query from the previous example can be built using some of the convenience methods in `SolrQuery`:[source,java,indent=0]----@@ -176,7 +199,8 @@ include::{example-source-dir}UsingSolrJRefGuideExamplesTest.java[tag=solrj-query== Indexing in SolrJ-Indexing is also simple using SolrJ. Users build the documents they want to index as instances of `SolrInputDocument`, and provide them as arguments to one of the `add()` methods on `SolrClient`.+Indexing is also simple using SolrJ.+Users build the documents they want to index as instances of `SolrInputDocument`, and provide them as arguments to one of the `add()` methods on `SolrClient`.The following example shows how to use SolrJ to add a document to Solr's "techproducts" example collection:@@ -185,12 +209,17 @@ The following example shows how to use SolrJ to add a document to Solr's "techprinclude::{example-source-dir}UsingSolrJRefGuideExamplesTest.java[tag=solrj-index-with-raw-solrinputdoc]-----CAUTION: The indexing examples above are intended to show syntax. For brevity, they break several Solr indexing best-practices. Under normal circumstances, documents should be indexed in larger batches, instead of one at a time. It is also suggested that Solr administrators commit documents using Solr's autocommit settings, and not using explicit `commit()` invocations.+CAUTION: The indexing examples above are intended to show syntax.+For brevity, they break several Solr indexing best-practices.+Under normal circumstances, documents should be indexed in larger batches, instead of one at a time.+It is also suggested that Solr administrators commit documents using Solr's autocommit settings, and not using explicit `commit()` invocations.== Java Object Binding-While the `UpdateResponse` and `QueryResponse` interfaces that SolrJ provides are useful, it is often more convenient to work with domain-specific objects that can more easily be understood by your application. Thankfully, SolrJ supports this by implicitly converting documents to and from any class that has been specially marked with {solr-javadocs}/solrj/org/apache/solr/client/solrj//beans/Field.html[`Field`] annotations.+While the `UpdateResponse` and `QueryResponse` interfaces that SolrJ provides are useful, it is often more convenient to work with domain-specific objects that can more easily be understood by your application.+Thankfully, SolrJ supports this by implicitly converting documents to and from any class that has been specially marked with {solr-javadocs}/solrj/org/apache/solr/client/solrj//beans/Field.html[`Field`] annotations.-Each instance variable in a Java object can be mapped to a corresponding Solr field, using the `Field` annotation. The Solr field shares the name of the annotated variable by default, however, this can be overridden by providing the annotation with an explicit field name.+Each instance variable in a Java object can be mapped to a corresponding Solr field, using the `Field` annotation.+The Solr field shares the name of the annotated variable by default, however, this can be overridden by providing the annotation with an explicit field name.The example snippet below shows an annotated `TechProduct` class that can be used to represent results from Solr's "techproducts" example collection.@@ -214,7 +243,9 @@ include::{example-source-dir}UsingSolrJRefGuideExamplesTest.java[tag=solrj-query----== Other APIs-SolrJ allows more than just querying and indexing. It supports all of Solr's APIs. Accessing Solr's other APIs is as easy as finding the appropriate request object, providing any necessary parameters, and passing it to the `request()` method of your `SolrClient`. `request()` will return a `NamedList`: a generic object which mirrors the hierarchical structure of the JSON or XML returned by their request.+SolrJ allows more than just querying and indexing.+It supports all of Solr's APIs.+Accessing Solr's other APIs is as easy as finding the appropriate request object, providing any necessary parameters, and passing it to the `request()` method of your `SolrClient`. `request()` will return a `NamedList`: a generic object which mirrors the hierarchical structure of the JSON or XML returned by their request.The example below shows how SolrJ users can call the CLUSTERSTATUS API of SolrCloud deployments, and manipulate the returned `NamedList`:diff --git a/solr/solr-ref-guide/src/spatial-search.adoc b/solr/solr-ref-guide/src/spatial-search.adocindex 0ae872d97f3..1a87e1d3e86 100644--- a/solr/solr-ref-guide/src/spatial-search.adoc+++ b/solr/solr-ref-guide/src/spatial-search.adoc@@ -32,9 +32,12 @@ There are four main field types available for spatial search:* `SpatialRecursivePrefixTreeFieldType` (RPT for short), including `RptWithGeometrySpatialField`, a derivative* `BBoxField`-`LatLonPointSpatialField` is the ideal field type for the most common use-cases for lat-lon point data. It replaces LatLonType which still exists for backwards compatibility. RPT offers some more features for more advanced/custom use cases and options like polygons and heatmaps.+`LatLonPointSpatialField` is the ideal field type for the most common use-cases for lat-lon point data.+It replaces LatLonType which still exists for backwards compatibility.+RPT offers some more features for more advanced/custom use cases and options like polygons and heatmaps.-`RptWithGeometrySpatialField` is for indexing and searching non-point data though it can do points too. It can't do sorting/boosting.+`RptWithGeometrySpatialField` is for indexing and searching non-point data though it can do points too.+It can't do sorting/boosting.`BBoxField` is for indexing bounding boxes, querying by a box, specifying a search predicate (Intersects,Within,Contains,Disjoint,Equals), and a relevancy sort/boost like overlapRatio or simply the area.@@ -47,15 +50,19 @@ Here's how `LatLonPointSpatialField` (LLPSF) should usually be configured in the[source,xml]<fieldType name="location" class="solr.LatLonPointSpatialField" docValues="true"/>-LLPSF supports toggling `indexed`, `stored`, `docValues`, and `multiValued`. LLPSF internally uses a 2-dimensional Lucene "Points" (BDK tree) index when "indexed" is enabled (the default). When "docValues" is enabled, a latitude and longitudes pair are bit-interleaved into 64 bits and put into Lucene DocValues. The accuracy of the docValues data is about a centimeter.+LLPSF supports toggling `indexed`, `stored`, `docValues`, and `multiValued`. LLPSF internally uses a 2-dimensional Lucene "Points" (BDK tree) index when "indexed" is enabled (the default). When "docValues" is enabled, a latitude and longitudes pair are bit-interleaved into 64 bits and put into Lucene DocValues.+The accuracy of the docValues data is about a centimeter.== Indexing PointsFor indexing geodetic points (latitude and longitude), supply it in "lat,lon" order (comma separated).-For indexing non-geodetic points, it depends. Use `x y` (a space) if RPT. For PointType however, use `x,y` (a comma).+For indexing non-geodetic points, it depends.+Use `x y` (a space) if RPT.+For PointType however, use `x,y` (a comma).-If you'd rather use a standard industry format, Solr supports https://en.wikipedia.org/wiki/Well-known_text[WKT] and http://geojson.org/[GeoJSON]. However it's much bulkier than the raw coordinates for such simple data. (Not supported by the deprecated LatLonType or PointType)+If you'd rather use a standard industry format, Solr supports https://en.wikipedia.org/wiki/Well-known_text[WKT] and http://geojson.org/[GeoJSON]. However it's much bulkier than the raw coordinates for such simple data.+(Not supported by the deprecated LatLonType or PointType)=== Indexing GeoJSON and WKT@@ -67,23 +74,27 @@ bin/post -type "application/json" -url "http://localhost:8983/solr/mycollection/The key parameter to pass in with your request is:`format`::-The format of the file to pass in. Accepted values: `WKT` or `geojson`.+The format of the file to pass in.+ Accepted values: `WKT` or `geojson`.== Searching with Query ParsersThere are two spatial Solr "query parsers" for geospatial search: `geofilt` and `bbox`. They take the following parameters:`d`::-The radial distance, usually in kilometers. RPT & BBoxField can set other units via the setting `distanceUnits`.+The radial distance, usually in kilometers.+RPT & BBoxField can set other units via the setting `distanceUnits`.`pt`::-The center point using the format "lat,lon" if latitude & longitude. Otherwise, "x,y" for PointType or "x y" for RPT field types.+The center point using the format "lat,lon" if latitude & longitude.+Otherwise, "x,y" for PointType or "x y" for RPT field types.`sfield`::A spatial indexed field.`score`::-(Advanced option; not supported by LatLonType (deprecated) or PointType) If the query is used in a scoring context (e.g., as the main query in `q`), this _<<local-parameters-in-queries.adoc#,local parameter>>_ determines what scores will be produced. Valid values are:+(Advanced option; not supported by LatLonType (deprecated) or PointType) If the query is used in a scoring context (e.g., as the main query in `q`), this _<<local-params.adoc#,local param>>_ determines what scores will be produced.+Valid values are:* `none`: A fixed score of 1.0. (the default)* `kilometers`: distance in kilometers between the field value and the specified center point@@ -94,7 +105,8 @@ A spatial indexed field.+[WARNING]====-Don't use this for indexed non-point shapes (e.g., polygons). The results will be erroneous. And with RPT, it's only recommended for multi-valued point data, as the implementation doesn't scale very well and for single-valued fields, you should instead use a separate non-RPT field purely for distance sorting.+Don't use this for indexed non-point shapes (e.g., polygons). The results will be erroneous.+And with RPT, it's only recommended for multi-valued point data, as the implementation doesn't scale very well and for single-valued fields, you should instead use a separate non-RPT field purely for distance sorting.====+When used with `BBoxField`, additional options are supported:@@ -109,7 +121,9 @@ When used with `BBoxField`, additional options are supported:=== geofilt-The `geofilt` filter allows you to retrieve results based on the geospatial distance (AKA the "great circle distance") from a given point. Another way of looking at it is that it creates a circular shape filter. For example, to find all documents within five kilometers of a given lat/lon point, you could enter:+The `geofilt` filter allows you to retrieve results based on the geospatial distance (AKA the "great circle distance") from a given point.+Another way of looking at it is that it creates a circular shape filter.+For example, to find all documents within five kilometers of a given lat/lon point, you could enter:[source,text]&q=*:*&fq={!geofilt sfield=store}&pt=45.15,-93.85&d=5@@ -121,14 +135,17 @@ image::images/spatial-search/circle.png[5KM radius]=== bbox-The `bbox` filter is very similar to `geofilt` except it uses the _bounding box_ of the calculated circle. See the blue box in the diagram below. It takes the same parameters as geofilt.+The `bbox` filter is very similar to `geofilt` except it uses the _bounding box_ of the calculated circle.+See the blue box in the diagram below.+It takes the same parameters as geofilt.Here's a sample query:[source,text]&q=*:*&fq={!bbox sfield=store}&pt=45.15,-93.85&d=5-The rectangular shape is faster to compute and so it's sometimes used as an alternative to `geofilt` when it's acceptable to return points outside of the radius. However, if the ideal goal is a circle but you want it to run faster, then instead consider using the RPT field and try a large `distErrPct` value like `0.1` (10% radius). This will return results outside the radius but it will do so somewhat uniformly around the shape.+The rectangular shape is faster to compute and so it's sometimes used as an alternative to `geofilt` when it's acceptable to return points outside of the radius.+However, if the ideal goal is a circle but you want it to run faster, then instead consider using the RPT field and try a large `distErrPct` value like `0.1` (10% radius). This will return results outside the radius but it will do so somewhat uniformly around the shape.image::images/spatial-search/bbox.png[Bounding box]@@ -139,21 +156,26 @@ When a bounding box includes a pole, the bounding box ends up being a "bounding=== Filtering by an Arbitrary Rectangle-Sometimes the spatial search requirement calls for finding everything in a rectangular area, such as the area covered by a map the user is looking at. For this case, geofilt and bbox won't cut it. This is somewhat of a trick, but you can use Solr's range query syntax for this by supplying the lower-left corner as the start of the range and the upper-right corner as the end of the range.+Sometimes the spatial search requirement calls for finding everything in a rectangular area, such as the area covered by a map the user is looking at.+For this case, geofilt and bbox won't cut it.+This is somewhat of a trick, but you can use Solr's range query syntax for this by supplying the lower-left corner as the start of the range and the upper-right corner as the end of the range.Here's an example:[source,text]&q=*:*&fq=store:[45,-94 TO 46,-93]-LatLonType (deprecated) does *not* support rectangles that cross the dateline. For RPT and BBoxField, if you are non-geospatial coordinates (`geo="false"`) then you must quote the points due to the space, e.g., `"x y"`.+LatLonType (deprecated) does *not* support rectangles that cross the dateline.+For RPT and BBoxField, if you are non-geospatial coordinates (`geo="false"`) then you must quote the points due to the space, e.g., `"x y"`.=== Optimizing: Cache or Not-It's most common to put a spatial query into an "fq" parameter – a filter query. By default, Solr will cache the query in the filter cache.+It's most common to put a spatial query into an "fq" parameter – a filter query.+By default, Solr will cache the query in the filter cache.-If you know the filter query (be it spatial or not) is fairly unique and not likely to get a cache hit then specify `cache="false"` as a local-param as seen in the following example. The only spatial types which stand to benefit from this technique are LatLonPointSpatialField and LatLonType (deprecated). Enable docValues on the field (if it isn't already). LatLonType (deprecated) additionally requires a `cost="100"` (or more) local-param.+If you know the filter query (be it spatial or not) is fairly unique and not likely to get a cache hit then specify `cache="false"` as a local-param as seen in the following example.+The only spatial types which stand to benefit from this technique are LatLonPointSpatialField and LatLonType (deprecated). Enable docValues on the field (if it isn't already). LatLonType (deprecated) additionally requires a `cost="100"` (or more) local-param.[source,text]&q=...mykeywords...&fq=...someotherfilters...&fq={!geofilt cache=false}&sfield=store&pt=45.15,-93.85&d=5@@ -205,7 +227,7 @@ There are other ways to do it too, like using a `{!geofilt}` in each facet.query=== Boost Nearest Results-Using the <<the-dismax-query-parser.adoc#,DisMax>> or <<the-extended-dismax-query-parser.adoc#,Extended DisMax>>, you can combine spatial search with the boost function to boost the nearest results:+Using the <<dismax-query-parser.adoc#,DisMax>> or <<edismax-query-parser.adoc#,Extended DisMax>>, you can combine spatial search with the boost function to boost the nearest results:[source,text]&q.alt=*:*&fq={!geofilt}&sfield=store&pt=45.15,-93.85&d=50&bf=recip(geodist(),2,200,20)&sort=score desc@@ -238,9 +260,9 @@ RPT _shares_ various features in common with `LatLonPointSpatialField`. Some are[IMPORTANT]====-Although RPT supports distance sorting/boosting, it is so inefficient at doing this that it might be- removed in the future. Fortunately, you can use LatLonPointSpatialField _as well_ as RPT. Use LLPSF for the distance- sorting/boosting; it only needs to have docValues for this; the index attribute can be disabled as it won't be used.+Although RPT supports distance sorting/boosting, it is so inefficient at doing this that it might be removed in the future.+Fortunately, you can use LatLonPointSpatialField _as well_ as RPT.+Use LLPSF for the distance sorting/boosting; it only needs to have docValues for this; the index attribute can be disabled as it won't be used.======= Schema Configuration for RPT@@ -251,7 +273,9 @@ To use RPT, the field type must be registered and configured in `schema.xml`. ThThe name of the field type.`class`::-This should be `solr.SpatialRecursivePrefixTreeFieldType`. But be aware that the Lucene spatial module includes some other so-called "spatial strategies" other than RPT, notably TermQueryPT*, BBox, PointVector*, and SerializedDV. Solr requires a field type to parallel these in order to use them. The asterisked ones have them.+This should be `solr.SpatialRecursivePrefixTreeFieldType`. But be aware that the Lucene spatial module includes some other so-called "spatial strategies" other than RPT, notably TermQueryPT*, BBox, PointVector*, and SerializedDV.+Solr requires a field type to parallel these in order to use them.+The asterisked ones have them.`spatialContextFactory`::This is a Java class name to an internal extension point governing support for shape definitions & parsing.@@ -259,49 +283,66 @@ There are two built-in aliases for known implementations: `Geo3D` and `JTS`.The default blank value does not support polygons.`geo`::-If `true`, the default, latitude and longitude coordinates will be used and the mathematical model will generally be a sphere. If `false`, the coordinates will be generic X & Y on a 2D plane using Euclidean/Cartesian geometry.+If `true`, the default, latitude and longitude coordinates will be used and the mathematical model will generally be a sphere.+If `false`, the coordinates will be generic X & Y on a 2D plane using Euclidean/Cartesian geometry.-`format`:: Defines the shape syntax/format to be used. Defaults to `WKT` but `GeoJSON` is another popular format. Spatial4j governs this feature and supports https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/io/package-frame.html[other formats]. If a given shape is parseable as "lat,lon" or "x y" then that is always supported.+`format`:: Defines the shape syntax/format to be used.+Defaults to `WKT` but `GeoJSON` is another popular format.+Spatial4j governs this feature and supports https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/io/package-frame.html[other formats]. If a given shape is parseable as "lat,lon" or "x y" then that is always supported.`distanceUnits`::-This is used to specify the units for distance measurements used throughout the use of this field. This can be `degrees`, `kilometers` or `miles`. It is applied to nearly all distance measurements involving the field: `maxDistErr`, `distErr`, `d`, `geodist` and the `score` when score is `distance`, `area`, or `area2d`. However, it doesn't affect distances embedded in WKT strings, (e.g., `BUFFER(POINT(200 10),0.2)`), which are still in degrees.+This is used to specify the units for distance measurements used throughout the use of this field.+This can be `degrees`, `kilometers` or `miles`. It is applied to nearly all distance measurements involving the field: `maxDistErr`, `distErr`, `d`, `geodist` and the `score` when score is `distance`, `area`, or `area2d`. However, it doesn't affect distances embedded in WKT strings, (e.g., `BUFFER(POINT(200 10),0.2)`), which are still in degrees.+`distanceUnits` defaults to either `kilometers` if `geo` is `true`, or `degrees` if `geo` is `false`.+`distanceUnits` replaces the `units` attribute; which is now deprecated and mutually exclusive with this attribute.`distErrPct`::-Defines the default precision of non-point shapes (both index & query), as a fraction between `0.0` (fully precise) to `0.5`. The closer this number is to zero, the more accurate the shape will be. However, more precise indexed shapes use more disk space and take longer to index.+Defines the default precision of non-point shapes (both index & query), as a fraction between `0.0` (fully precise) to `0.5`. The closer this number is to zero, the more accurate the shape will be.+However, more precise indexed shapes use more disk space and take longer to index.+-Bigger `distErrPct` values will make queries faster but less accurate. At query time this can be overridden in the query syntax, such as to `0.0` so as to not approximate the search shape. The default for the RPT field is `0.025`.+Bigger `distErrPct` values will make queries faster but less accurate.+At query time this can be overridden in the query syntax, such as to `0.0` so as to not approximate the search shape.+The default for the RPT field is `0.025`.+-NOTE: For RPTWithGeometrySpatialField (see below), there's always complete accuracy with the serialized geometry and so this doesn't control accuracy so much as it controls the trade-off of how big the index should be. distErrPct defaults to 0.15 for that field.+NOTE: For RPTWithGeometrySpatialField (see below), there's always complete accuracy with the serialized geometry and so this doesn't control accuracy so much as it controls the trade-off of how big the index should be.+`distErrPct` defaults to `0.15` for that field.-`maxDistErr`:: Defines the highest level of detail required for indexed data. If left blank, the default is one meter – just a bit less than 0.000009 degrees. This setting is used internally to compute an appropriate maxLevels (see below).+`maxDistErr`:: Defines the highest level of detail required for indexed data.+If left blank, the default is one meter – just a bit less than 0.000009 degrees.+This setting is used internally to compute an appropriate maxLevels (see below).`worldBounds`::-Defines the valid numerical ranges for x and y, in the format of `ENVELOPE(minX, maxX, maxY, minY)`. If `geo="true"`, the standard lat-lon world boundaries are assumed. If `geo=false`, you should define your boundaries.+Defines the valid numerical ranges for x and y, in the format of `ENVELOPE(minX, maxX, maxY, minY)`. If `geo="true"`, the standard lat-lon world boundaries are assumed.+If `geo=false`, you should define your boundaries.`distCalculator`::-Defines the distance calculation algorithm. If `geo=true`, `haversine` is the default. If `geo=false`, `cartesian` will be the default. Other possible values are `lawOfCosines`, `vincentySphere` and `cartesian^2`.+Defines the distance calculation algorithm.+If `geo=true`, `haversine` is the default.+If `geo=false`, `cartesian` will be the default.+Other possible values are `lawOfCosines`, `vincentySphere` and `cartesian^2`.-`prefixTree`:: Defines the spatial grid implementation. Since a PrefixTree (such as RecursivePrefixTree) maps the world as a grid, each grid cell is decomposed to another set of grid cells at the next level.+`prefixTree`:: Defines the spatial grid implementation.+Since a PrefixTree (such as RecursivePrefixTree) maps the world as a grid, each grid cell is decomposed to another set of grid cells at the next level.+If `geo=true` then the default prefix tree is `geohash`, otherwise it's `quad`. Geohash has 32 children at each level, quad has 4. Geohash can only be used for `geo=true` as it's strictly geospatial.+A third choice is `packedQuad`, which is generally more efficient than `quad`, provided there are many levels -- perhaps 20 or more.-`maxLevels`:: Sets the maximum grid depth for indexed data. Instead, it's usually more intuitive to compute an appropriate maxLevels by specifying `maxDistErr`.+`maxLevels`:: Sets the maximum grid depth for indexed data.+Instead, it's usually more intuitive to compute an appropriate maxLevels by specifying `maxDistErr`.*_And there are others:_* `normWrapLongitude`, `datelineRule`, `validationRule`, `autoIndex`, `allowMultiOverlap`, `precisionModel`. For further info, see notes below about `spatialContextFactory` implementations referenced above, especially the link to the JTS based one.=== Standard ShapesThe RPT field types support a set of standard shapes:-points, circles (aka buffered points), envelopes (aka rectangles or bounding boxes), line strings,-polygons, and "multi" variants of these. The envelopes and line strings are Euclidean/cartesian (flat 2D) shapes.-Underlying Solr is the Spatial4j library which implements them. To support other shapes, you can configure the-`spatialContextFactory` attribute on the field type to reference other options. Two are available: JTS and Geo3D.+points, circles (aka buffered points), envelopes (aka rectangles or bounding boxes), line strings, polygons, and "multi" variants of these.+The envelopes and line strings are Euclidean/cartesian (flat 2D) shapes.+Underlying Solr is the Spatial4j library which implements them.+To support other shapes, you can configure the `spatialContextFactory` attribute on the field type to reference other options.+Two are available: JTS and Geo3D.=== JTS and Polygons (flat)@@ -337,7 +378,9 @@ Here's an example polygon query for a field "geo" that can be either solr.Spatia[source,plain]&q=*:*&fq={!field f=geo}Intersects(POLYGON((-10 30, -40 40, -10 -20, 40 20, 0 0, -10 30)))-Inside the parenthesis following the search predicate is the shape definition. The format of that shape is governed by the `format` attribute on the field type, defaulting to WKT. If you prefer GeoJSON, you can specify that instead.+Inside the parenthesis following the search predicate is the shape definition.+The format of that shape is governed by the `format` attribute on the field type, defaulting to WKT.+If you prefer GeoJSON, you can specify that instead.Beyond this Reference Guide and Spatila4j's docs, there are some details that remain at the Solr Wiki at https://cwiki.apache.org/confluence/display/solr/SolrAdaptersForLuceneSpatial4.@@ -372,9 +415,13 @@ If the order is wrong then the interpretation is inverted, thus the polygon will=== RptWithGeometrySpatialField-The `RptWithGeometrySpatialField` field type is a derivative of `SpatialRecursivePrefixTreeFieldType` that also stores the original geometry internally in Lucene DocValues, which it uses to achieve accurate search. It can also be used for indexed point fields. The Intersects predicate (the default) is particularly fast, since many search results can be returned as an accurate hit without requiring a geometry check. This field type is configured just like RPT except that the default `distErrPct` is 0.15 (higher than 0.025) because the grid squares are purely for performance and not to fundamentally represent the shape.+The `RptWithGeometrySpatialField` field type is a derivative of `SpatialRecursivePrefixTreeFieldType` that also stores the original geometry internally in Lucene DocValues, which it uses to achieve accurate search.+It can also be used for indexed point fields.+The Intersects predicate (the default) is particularly fast, since many search results can be returned as an accurate hit without requiring a geometry check.+This field type is configured just like RPT except that the default `distErrPct` is 0.15 (higher than 0.025) because the grid squares are purely for performance and not to fundamentally represent the shape.-An optional in-memory cache can be defined in `solrconfig.xml`, which should be done when the data tends to have shapes with many vertices. Assuming you name your field "geom", you can configure an optional cache in `solrconfig.xml` by adding the following – notice the suffix of the cache name:+An optional in-memory cache can be defined in `solrconfig.xml`, which should be done when the data tends to have shapes with many vertices.+Assuming you name your field "geom", you can configure an optional cache in `solrconfig.xml` by adding the following – notice the suffix of the cache name:[source,xml]----@@ -386,11 +433,16 @@ An optional in-memory cache can be defined in `solrconfig.xml`, which should beregenerator="solr.NoOpRegenerator"/>-----When using this field type, you will likely _not_ want to mark the field as stored because it's redundant with the DocValues data and surely larger because of the formatting (be it WKT or GeoJSON). To retrieve the spatial data in search results from DocValues, use the `[geo]` transformer -- <<transforming-result-documents.adoc#,Transforming Result Documents>>.+When using this field type, you will likely _not_ want to mark the field as stored because it's redundant with the DocValues data and surely larger because of the formatting (be it WKT or GeoJSON).+To retrieve the spatial data in search results from DocValues, use the <<document-transformers.adoc#geo-geospatial-formatter,`[geo]` transformer>>.=== Heatmap Faceting-The RPT field supports generating a 2D grid of facet counts for documents having spatial data in each grid cell. For high-detail grids, this can be used to plot points, and for lesser detail it can be used for heatmap generation. The grid cells are determined at index-time based on RPT's configuration. At facet counting time, the indexed cells in the region of interest are traversed and a grid of counters corresponding to each cell are incremented. Solr can return the data in a straight-forward 2D array of integers or in a PNG which compresses better for larger data sets but must be decoded.+The RPT field supports generating a 2D grid of facet counts for documents having spatial data in each grid cell.+For high-detail grids, this can be used to plot points, and for lesser detail it can be used for heatmap generation.+The grid cells are determined at index-time based on RPT's configuration.+At facet counting time, the indexed cells in the region of interest are traversed and a grid of counters corresponding to each cell are incremented.+Solr can return the data in a straight-forward 2D array of integers or in a PNG which compresses better for larger data sets but must be decoded.The heatmap feature is accessible both from Solr's standard faceting feature, plus the newer more flexible <<json-facet-api.adoc#heatmap-facet,JSON Facet API>>.We'll proceed now with standard faceting.@@ -404,23 +456,32 @@ Set to `true` to enable standard faceting.The field name of type RPT.`facet.heatmap.geom`::-The region to compute the heatmap on, specified using the rectangle-range syntax or WKT. It defaults to the world. ex: `["-180 -90" TO "180 90"]`.+The region to compute the heatmap on, specified using the rectangle-range syntax or WKT.+It defaults to the world.+ex: `["-180 -90" TO "180 90"]`.`facet.heatmap.gridLevel`::-A specific grid level, which determines how big each grid cell is. Defaults to being computed via `distErrPct` (or `distErr`).+A specific grid level, which determines how big each grid cell is.+Defaults to being computed via `distErrPct` (or `distErr`).`facet.heatmap.distErrPct`::-A fraction of the size of geom used to compute gridLevel. Defaults to 0.15. It's computed the same as a similarly named parameter for RPT.+A fraction of the size of geom used to compute gridLevel.+Defaults to 0.15. It's computed the same as a similarly named parameter for RPT.`facet.heatmap.distErr`::-A cell error distance used to pick the grid level indirectly. It's computed the same as a similarly named parameter for RPT.+A cell error distance used to pick the grid level indirectly.+It's computed the same as a similarly named parameter for RPT.`facet.heatmap.format`::The format, either `ints2D` (default) or `png`.[TIP]====-You'll experiment with different `distErrPct` values (probably 0.10 - 0.20) with various input geometries till the default size is what you're looking for. The specific details of how it's computed aren't important. For high-detail grids used in point-plotting (loosely one cell per pixel), set `distErr` to be the number of decimal-degrees of several pixels or so of the map being displayed. Also, you probably don't want to use a geohash-based grid because the cell orientation between grid levels flip-flops between being square and rectangle. Quad is consistent and has more levels, albeit at the expense of a larger index.+You'll experiment with different `distErrPct` values (probably 0.10 - 0.20) with various input geometries till the default size is what you're looking for.+The specific details of how it's computed aren't important.+For high-detail grids used in point-plotting (loosely one cell per pixel), set `distErr` to be the number of decimal-degrees of several pixels or so of the map being displayed.+Also, you probably don't want to use a geohash-based grid because the cell orientation between grid levels flip-flops between being square and rectangle.+Quad is consistent and has more levels, albeit at the expense of a larger index.====Here's some sample output in JSON (with "..." inserted for brevity):@@ -431,17 +492,31 @@ Here's some sample output in JSON (with "..." inserted for brevity):counts_ints2D=[[0, 0, 2, 1, ....],[1, 1, 3, 2, ...],...]}-----The output shows the gridLevel which is interesting since it's often computed from other parameters. If an interface being developed allows an explicit resolution increase/decrease feature then subsequent requests can specify the gridLevel explicitly.+The output shows the gridLevel which is interesting since it's often computed from other parameters.+If an interface being developed allows an explicit resolution increase/decrease feature then subsequent requests can specify the gridLevel explicitly.-The `minX`, `maxX`, `minY`, `maxY` reports the region where the counts are. This is the minimally enclosing bounding rectangle of the input `geom` at the target grid level. This may wrap the dateline. The `columns` and `rows` values are how many columns and rows that the output rectangle is to be divided by evenly. Note: Don't divide an on-screen projected map rectangle evenly to plot these rectangles/points since the cell data is in the coordinate space of decimal degrees if geo=true or whatever units were given if geo=false. This could be arranged to be the same as an on-screen map but won't necessarily be.+The `minX`, `maxX`, `minY`, `maxY` reports the region where the counts are.+This is the minimally enclosing bounding rectangle of the input `geom` at the target grid level.+This may wrap the dateline.+The `columns` and `rows` values are how many columns and rows that the output rectangle is to be divided by evenly.+Note: Don't divide an on-screen projected map rectangle evenly to plot these rectangles/points since the cell data is in the coordinate space of decimal degrees if geo=true or whatever units were given if geo=false.+This could be arranged to be the same as an on-screen map but won't necessarily be.-The `counts_ints2D` key has a 2D array of integers. The initial outer level is in row order (top-down), then the inner arrays are the columns (left-right). If any array would be all zeros, a null is returned instead for efficiency reasons. The entire value is null if there is no matching spatial data.+The `counts_ints2D` key has a 2D array of integers.+The initial outer level is in row order (top-down), then the inner arrays are the columns (left-right). If any array would be all zeros, a null is returned instead for efficiency reasons.+The entire value is null if there is no matching spatial data.-If `format=png` then the output key is `counts_png`. It's a base-64 encoded string of a 4-byte PNG. The PNG logically holds exactly the same data that the ints2D format does. Note that the alpha channel byte is flipped to make it easier to view the PNG for diagnostic purposes, since otherwise counts would have to exceed 2^24 before it becomes non-opague. Thus counts greater than this value will become opaque.+If `format=png` then the output key is `counts_png`. It's a base-64 encoded string of a 4-byte PNG.+The PNG logically holds exactly the same data that the ints2D format does.+Note that the alpha channel byte is flipped to make it easier to view the PNG for diagnostic purposes, since otherwise counts would have to exceed 2^24 before it becomes non-opague.+Thus counts greater than this value will become opaque.== BBoxField-The `BBoxField` field type indexes a single rectangle (bounding box) per document field and supports searching via a bounding box. It supports most spatial search predicates, it has enhanced relevancy modes based on the overlap or area between the search rectangle and the indexed rectangle. It's particularly useful for its relevancy modes. To configure it in the schema, use a configuration like this:+The `BBoxField` field type indexes a single rectangle (bounding box) per document field and supports searching via a bounding box.+It supports most spatial search predicates, it has enhanced relevancy modes based on the overlap or area between the search rectangle and the indexed rectangle.+It's particularly useful for its relevancy modes.+To configure it in the schema, use a configuration like this:[source,xml]----@@ -452,11 +527,19 @@ The `BBoxField` field type indexes a single rectangle (bounding box) per documen<fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>-----BBoxField is actually based off of 4 instances of another field type referred to by numberType. It also uses a boolean to flag a dateline cross. Assuming you want to use the relevancy feature, docValues is required. Some of the attributes are in common with the RPT field like geo, units, worldBounds, and spatialContextFactory because they share some of the same spatial infrastructure.+BBoxField is actually based off of 4 instances of another field type referred to by numberType.+It also uses a boolean to flag a dateline cross.+Assuming you want to use the relevancy feature, docValues is required.+Some of the attributes are in common with the RPT field like geo, units, worldBounds, and spatialContextFactory because they share some of the same spatial infrastructure.-To index a box, add a field value to a bbox field that's a string in the WKT/CQL ENVELOPE syntax. Example: `ENVELOPE(-10, 20, 15, 10)` which is minX, maxX, maxY, minY order. The parameter ordering is unintuitive but that's what the spec calls for. Alternatively, you could provide a rectangular polygon in WKT (or GeoJSON if you set set `format="GeoJSON"`).+To index a box, add a field value to a bbox field that's a string in the WKT/CQL ENVELOPE syntax.+Example: `ENVELOPE(-10, 20, 15, 10)` which is minX, maxX, maxY, minY order.+The parameter ordering is unintuitive but that's what the spec calls for.+Alternatively, you could provide a rectangular polygon in WKT (or GeoJSON if you set set `format="GeoJSON"`).-To search, you can use the `{!bbox}` query parser, or the range syntax e.g., `[10,-10 TO 15,20]`, or the ENVELOPE syntax wrapped in parenthesis with a leading search predicate. The latter is the only way to choose a predicate other than Intersects. For example:+To search, you can use the `{!bbox}` query parser, or the range syntax e.g., `[10,-10 TO 15,20]`, or the ENVELOPE syntax wrapped in parenthesis with a leading search predicate.+The latter is the only way to choose a predicate other than Intersects.+For example:[source,plain]&q={!field f=bbox}Contains(ENVELOPE(-10, 20, 15, 10))@@ -468,4 +551,8 @@ Now to sort the results by one of the relevancy modes, use it like this:&q={!field f=bbox score=overlapRatio}Intersects(ENVELOPE(-10, 20, 15, 10))-The `score` local parameter can be one of `overlapRatio`, `area`, and `area2D`. `area` scores by the document area using surface-of-a-sphere (assuming `geo=true`) math, while `area2D` uses simple width * height. `overlapRatio` computes a [0-1] ranged score based on how much overlap exists relative to the document's area and the query area. The javadocs of {lucene-javadocs}/spatial-extras/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.html[BBoxOverlapRatioValueSource] have more info on the formula. There is an additional parameter `queryTargetProportion` that allows you to weight the query side of the formula to the index (target) side of the formula. You can also use `&debug=results` to see useful score computation info.+The `score` local parameter can be one of `overlapRatio`, `area`, and `area2D`. `area` scores by the document area using surface-of-a-sphere (assuming `geo=true`) math, while `area2D` uses simple width * height.+`overlapRatio` computes a [0-1] ranged score based on how much overlap exists relative to the document's area and the query area.+The javadocs of {lucene-javadocs}/spatial-extras/org/apache/lucene/spatial/bbox/BBoxOverlapRatioValueSource.html[BBoxOverlapRatioValueSource] have more info on the formula.+There is an additional parameter `queryTargetProportion` that allows you to weight the query side of the formula to the index (target) side of the formula.+You can also use `&debug=results` to see useful score computation info.diff --git a/solr/solr-ref-guide/src/spell-checking.adoc b/solr/solr-ref-guide/src/spell-checking.adocindex 1f2250e3914..f24eae64b23 100644--- a/solr/solr-ref-guide/src/spell-checking.adoc+++ b/solr/solr-ref-guide/src/spell-checking.adoc@@ -140,7 +140,7 @@ The spellchecker can be configured with a traditional checker (i.e., `DirectSolr=== Add It to a Request Handler-Queries will be sent to a <<query-syntax-and-parsing.adoc#,RequestHandler>>. If every request should generate a suggestion, then you would add the following to the `requestHandler` that you are using:+Queries will be sent to a <<query-syntax-and-parsers.adoc#,RequestHandler>>. If every request should generate a suggestion, then you would add the following to the `requestHandler` that you are using:[source,xml]----@@ -303,10 +303,12 @@ Results:The `SpellCheckComponent` also supports spellchecking on distributed indexes. If you are using the SpellCheckComponent on a request handler other than "/select", you must provide the following two parameters:`shards`::-Specifies the shards in your distributed indexing configuration. For more information about distributed indexing, see <<distributed-search-with-index-sharding.adoc#,Distributed Search with Index Sharding>>+Specifies the shards in your distributed indexing configuration.+For more information about distributed indexing, see <<user-managed-distributed-search.adoc#,User-Managed Distributed Search>>.`shards.qt`::-Specifies the request handler Solr uses for requests to shards. This parameter is not required for the `/select` request handler.+Specifies the request handler Solr uses for requests to shards.+This parameter is not required for the `/select` request handler.For example:diff --git a/solr/solr-ref-guide/src/the-standard-query-parser.adoc b/solr/solr-ref-guide/src/standard-query-parser.adocsimilarity index 93%rename from solr/solr-ref-guide/src/the-standard-query-parser.adocrename to solr/solr-ref-guide/src/standard-query-parser.adocindex 446026fd8a5..5e810cad1e8 100644--- a/solr/solr-ref-guide/src/the-standard-query-parser.adoc+++ b/solr/solr-ref-guide/src/standard-query-parser.adoc@@ -1,4 +1,4 @@-= The Standard Query Parser+= Standard Query Parser// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,7 +18,7 @@Solr's default Query Parser is also known as the "```lucene```" parser.-The key advantage of the standard query parser is that it supports a robust and fairly intuitive syntax allowing you to create a variety of structured queries. The largest disadvantage is that it's very intolerant of syntax errors, as compared with something like the <<the-dismax-query-parser.adoc#,DisMax>> query parser which is designed to throw as few errors as possible.+The key advantage of the standard query parser is that it supports a robust and fairly intuitive syntax allowing you to create a variety of structured queries. The largest disadvantage is that it's very intolerant of syntax errors, as compared with something like the <<dismax-query-parser.adoc#,DisMax>> query parser which is designed to throw as few errors as possible.== Standard Query Parser Parameters@@ -111,13 +111,11 @@ Solr supports a variety of term modifiers that add flexibility or precision, asSolr's standard query parser supports single and multiple character wildcard searches within single terms. Wildcard characters can be applied to single terms, but not to search phrases.-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="30,20,50",options="header"]+[%autowidth.stretch,options="header"]|===|Wildcard Search Type |Special Character |Example-|Single character (matches a single character) |? |The search string `te?t` would match both test and text.-|Multiple characters (matches zero or more sequential characters) |* |The wildcard search: `tes*` would match test, testing, and tester. You can also use wildcard characters in the middle of a term. For example: `te*t` would match test and text. `*est` would match pest and test.+|Single character (matches a single character) | `?` |The search string `te?t` would match both test and text.+|Multiple characters (matches zero or more sequential characters) | `*` |The wildcard search: `tes*` would match test, testing, and tester. You can also use wildcard characters in the middle of a term. For example: `te*t` would match test and text. `*est` would match pest and test.|====== Fuzzy Searches@@ -220,9 +218,13 @@ Example:== Querying Specific Fields-Data indexed in Solr is organized in fields, which are <<defining-fields.adoc#,defined in the Solr Schema>>. Searches can take advantage of fields to add precision to queries. For example, you can search for a term only in a specific field, such as a title field.+Data indexed in Solr is organized in fields, which are <<fields.adoc#,defined in the Solr Schema>>.+Searches can take advantage of fields to add precision to queries.+For example, you can search for a term only in a specific field, such as a title field.-The Schema defines one field as a default field. If you do not specify a field in a query, Solr searches only the default field. Alternatively, you can specify a different field or a combination of fields in a query.+The Schema defines one field as a default field.+If you do not specify a field in a query, Solr searches only the default field.+Alternatively, you can specify a different field or a combination of fields in a query.To specify a field, type the field name followed by a colon ":" and then the term you are searching for within the field.@@ -240,9 +242,7 @@ The field is only valid for the term that it directly precedes, so the query `tiBoolean operators allow you to apply Boolean logic to queries, requiring the presence or absence of specific terms or conditions in fields in order to match documents. The table below summarizes the Boolean operators supported by the standard query parser.-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="20,40,40",options="header"]+[%autowidth.stretch,options="header"]|===|Boolean Operator |Alternative Symbol |Description|AND |`&&` |Requires both terms on either side of the Boolean operator to be present for a match.@@ -381,7 +381,7 @@ This can even be used to cache individual clauses of complex filter queries. In=== Specifying Dates and Times-Queries against date based fields must use the <<working-with-dates.adoc#,appropriate date formating>>. Queries for exact date values will require quoting or escaping since `:` is the parser syntax used to denote a field query:+Queries against date based fields must use the <<date-formatting-math.adoc#,appropriate date formating>>. Queries for exact date values will require quoting or escaping since `:` is the parser syntax used to denote a field query:* `createdate:1976-03-06T23\:59\:59.999Z`* `createdate:"1976-03-06T23:59:59.999Z"`diff --git a/solr/solr-ref-guide/src/the-stats-component.adoc b/solr/solr-ref-guide/src/stats-component.adocsimilarity index 74%rename from solr/solr-ref-guide/src/the-stats-component.adocrename to solr/solr-ref-guide/src/stats-component.adocindex 0db83a0908b..4b4ddd251d1 100644--- a/solr/solr-ref-guide/src/the-stats-component.adoc+++ b/solr/solr-ref-guide/src/stats-component.adoc@@ -1,4 +1,4 @@-= The Stats Component+= Stats Component// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -33,9 +33,11 @@ The Stats Component accepts the following parameters:If `true`, then invokes the Stats component.`stats.field`::-Specifies a field for which statistics should be generated. This parameter may be invoked multiple times in a query in order to request statistics on multiple fields.+Specifies a field for which statistics should be generated.+This parameter may be invoked multiple times in a query in order to request statistics on multiple fields.+-<<local-parameters-in-queries.adoc#,Local Parameters>> may be used to indicate which subset of the supported statistics should be computed, and/or that statistics should be computed over the results of an arbitrary numeric function (or query) instead of a simple field name. See the examples below.+<<local-params.adoc#,Local Params>> may be used to indicate which subset of the supported statistics should be computed, and/or that statistics should be computed over the results of an arbitrary numeric function (or query) instead of a simple field name.+See the examples below.=== Stats Component Example@@ -90,51 +92,65 @@ http://localhost:8983/solr/techproducts/select?q=*:*&wt=xml&stats=true&stats.fie== Statistics Supported-The table below explains the statistics supported by the Stats component. Not all statistics are supported for all field types, and not all statistics are computed by default (see <<Local Parameters with the Stats Component>> below for details)+The table below explains the statistics supported by the Stats component.+Not all statistics are supported for all field types, and not all statistics are computed by default (see <<Local Params with the Stats Component>> below for details)`min`::-The minimum value of the field/function in all documents in the set. This statistic is computed for all field types and is computed by default.+The minimum value of the field/function in all documents in the set.+This statistic is computed for all field types and is computed by default.`max`::-The maximum value of the field/function in all documents in the set. This statistic is computed for all field types and is computed by default.+The maximum value of the field/function in all documents in the set.+This statistic is computed for all field types and is computed by default.`sum`::-The sum of all values of the field/function in all documents in the set. This statistic is computed for numeric and date field types and is computed by default.+The sum of all values of the field/function in all documents in the set.+This statistic is computed for numeric and date field types and is computed by default.`count`::-The number of values found in all documents in the set for this field/function. This statistic is computed for all field types and is computed by default.+The number of values found in all documents in the set for this field/function.+This statistic is computed for all field types and is computed by default.`missing`::-The number of documents in the set which do not have a value for this field/function. This statistic is computed for all field types and is computed by default.+The number of documents in the set which do not have a value for this field/function.+This statistic is computed for all field types and is computed by default.`sumOfSquares`::Sum of all values squared (a by product of computing stddev). This statistic is computed for numeric and date field types and is computed by default.`mean`::-The average `(v1 + v2 .... + vN)/N`. This statistic is computed for numeric and date field types and is computed by default.+The average `(v1 + v2 .... + vN)/N`.+This statistic is computed for numeric and date field types and is computed by default.`stddev`::-Standard deviation, measuring how widely spread the values in the data set are. This statistic is computed for numeric and date field types and is computed by default.+Standard deviation, measuring how widely spread the values in the data set are.+This statistic is computed for numeric and date field types and is computed by default.`percentiles`::A list of percentile values based on cut-off points specified by the parameter value, such as `1,99,99.9`. These values are an approximation, using the https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf[t-digest algorithm]. This statistic is computed for numeric field types and is not computed by default.`distinctValues`::-The set of all distinct values for the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. This statistic is computed for all field types but is not computed by default.+The set of all distinct values for the field/function in all of the documents in the set.+This calculation can be very expensive for fields that do not have a tiny cardinality.+This statistic is computed for all field types but is not computed by default.`countDistinct`::-The exact number of distinct values in the field/function in all of the documents in the set. This calculation can be very expensive for fields that do not have a tiny cardinality. This statistic is computed for all field types but is not computed by default.+The exact number of distinct values in the field/function in all of the documents in the set.+This calculation can be very expensive for fields that do not have a tiny cardinality.+This statistic is computed for all field types but is not computed by default.`cardinality`::-A statistical approximation (currently using the https://en.wikipedia.org/wiki/HyperLogLog[HyperLogLog] algorithm) of the number of distinct values in the field/function in all of the documents in the set. This calculation is much more efficient then using the `countDistinct` option, but may not be 100% accurate.+A statistical approximation (currently using the https://en.wikipedia.org/wiki/HyperLogLog[HyperLogLog] algorithm) of the number of distinct values in the field/function in all of the documents in the set.+This calculation is much more efficient then using the `countDistinct` option, but may not be 100% accurate.+-Input for this option can be floating point number between `0.0` and `1.0` indicating how aggressively the algorithm should try to be accurate: `0.0` means use as little memory as possible; `1.0` means use as much memory as needed to be as accurate as possible. `true` is supported as an alias for `0.3`.+Input for this option can be floating point number between `0.0` and `1.0` indicating how aggressively the algorithm should try to be accurate: `0.0` means use as little memory as possible; `1.0` means use as much memory as needed to be as accurate as possible.+`true` is supported as an alias for `0.3`.+This statistic is computed for all field types but is not computed by default.-== Local Parameters with the Stats Component+== Local Params with the Stats Component-Similar to the <<faceting.adoc#,Facet Component>>, the `stats.field` parameter supports local parameters for:+Similar to the <<faceting.adoc#,Facet Component>>, the `stats.field` parameter supports local params for:* Tagging & Excluding Filters: `stats.field={!ex=filterA}price`* Changing the Output Key: `stats.field={!key=my_price_stats}price`@@ -144,21 +160,23 @@ Local parameters can also be used to specify individual statistics by name, over[IMPORTANT]====-If any supported statistics are specified via local parameters, then the entire set of default statistics is overridden and only the requested statistics are computed.+If any supported statistics are specified via local params, then the entire set of default statistics is overridden and only the requested statistics are computed.====Additional "Expert" local params are supported in some cases for affecting the behavior of some statistics:* `percentiles`-** `tdigestCompression` - a positive numeric value defaulting to `100.0` controlling the compression factor of the T-Digest. Larger values means more accuracy, but also uses more memory.+** `tdigestCompression` - a positive numeric value defaulting to `100.0` controlling the compression factor of the T-Digest.+Larger values means more accuracy, but also uses more memory.* `cardinality`** `hllPreHashed` - a boolean option indicating that the statistics are being computed over a "long" field that has already been hashed at index time – allowing the HLL computation to skip this step.** `hllLog2m` - an integer value specifying an explicit "log2m" value to use, overriding the heuristic value determined by the cardinality local param and the field type – see the https://github.com/aggregateknowledge/java-hll/[java-hll] documentation for more details** `hllRegwidth` - an integer value specifying an explicit "regwidth" value to use, overriding the heuristic value determined by the cardinality local param and the field type – see the https://github.com/aggregateknowledge/java-hll/[java-hll] documentation for more details-=== Examples with Local Parameters+=== Examples with Local Params-Here we compute some statistics for the price field. The min, max, mean, 90th, and 99th percentile price values are computed against all products that are in stock (`q=*:*` and `fq=inStock:true`), and independently all of the default statistics are computed against all products regardless of whether they are in stock or not (by excluding that filter).+Here we compute some statistics for the price field.+The min, max, mean, 90th, and 99th percentile price values are computed against all products that are in stock (`q=*:*` and `fq=inStock:true`), and independently all of the default statistics are computed against all products regardless of whether they are in stock or not (by excluding that filter).[source,text]http://localhost:8983/solr/techproducts/select?q=*:*&fq={!tag=stock_check}inStock:true&stats=true&stats.field={!ex=stock_check+key=instock_prices+min=true+max=true+mean=true+percentiles='90,99'}price&stats.field={!key=all_prices}price&rows=0&indent=true&wt=xmldiff --git a/solr/solr-ref-guide/src/stream-api.adoc b/solr/solr-ref-guide/src/stream-api.adocindex 18b2ad1445f..55cada03140 100644--- a/solr/solr-ref-guide/src/stream-api.adoc+++ b/solr/solr-ref-guide/src/stream-api.adoc@@ -22,19 +22,16 @@ The `/stream` request handler, beyond just running a streaming expression, alsolets you find out what expressions are available to use and lets you control thebehavior of any registered Daemon processes.-NOTE: This API doesn't follow the v2 API structure.---+NOTE: This API doesn't follow the v2 API structure.[[plugins]]== PLUGINS: List all expressions registered-Lists out all the streaming expressions that have been registered and are available for use. This includes both the default expressions registered by the StreamHandler and any that you have registered.+Lists out all the streaming expressions that have been registered and are available for use.+This includes both the default expressions registered by the StreamHandler and any that you have registered.`/stream?action=PLUGINS`-=== PLUGINS ResponseThe response will list each expression registered and it's implementing class.@@ -60,7 +57,7 @@ http://localhost:8983/solr/gettingstarted/stream?action=PLUGINS"upper":"org.apache.solr.client.solrj.io.eval.UpperEvaluator","commit":"org.apache.solr.client.solrj.io.stream.CommitStream","echo":"org.apache.solr.client.solrj.io.stream.EchoStream"-}+}}----[[list]]@@ -69,20 +66,18 @@ http://localhost:8983/solr/gettingstarted/stream?action=PLUGINSThe <<stream-decorator-reference.adoc#daemon,daemon>> function allows you to wrap a streaming expression and run it at intervals to provide both continuous push and pull streaming.This command lists out all the currently running daemon processes.-`/stream?action=LIST`IMPORTANT: This command lists out all the daemon processed that are registered to a specific core that you are interacting with, not across the collection as a whole.-If you have a collection that consists of multiple shards or multiple replicas of those shards, each LIST command will bounce between the cores, returning different lists of processes. It's recommended that for-managing daemon processes you create a collection with a single core and no replicas to host the daemon processes to ensure a single view.+If you have a collection that consists of multiple shards or multiple replicas of those shards, each LIST command will bounce between the cores, returning different lists of processes.+It's recommended that for managing daemon processes you create a collection with a single core and no replicas to host the daemon processes to ensure a single view.=== LIST ResponseThe response will describe each daemon process on that specific core.=== Examples using LIST-This assumes that you have registered a daemon process similar to the below simplistic example that reads a single-random document from the `gettingstarted` collection and then writes it back to the same collection every 10 seconds:+This assumes that you have registered a daemon process similar to the below simplistic example that reads a single random document from the `gettingstarted` collection and then writes it back to the same collection every 10 seconds:[source,text]----@@ -121,8 +116,8 @@ http://localhost:8983/solr/gettingstarted/stream?action=LIST"EOF":true}]}}-----This shows a single daemon process running under the id of 12345, and that it has been run 421 times. Each process is a {java-javadocs}java/lang/Thread.html,-and the states are those of a Thread.+This shows a single daemon process running under the id of 12345, and that it has been run 421 times.+Each process is a {java-javadocs}java/lang/Thread.html, and the states are those of a Thread.[[stop]]== STOP: Stop a Daemon processesdiff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adocindex 9c5063beae9..55e67a7d6e0 100644--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc@@ -19,7 +19,9 @@== cartesianProduct-The `cartesianProduct` function turns a single tuple with a multi-valued field (i.e., an array) into multiple tuples, one for each value in the array field. That is, given a single tuple containing an array of N values for fieldA, the `cartesianProduct` function will output N tuples, each with one value from the original tuple's array. In essence, you can flatten arrays for further processing.+The `cartesianProduct` function turns a single tuple with a multi-valued field (i.e., an array) into multiple tuples, one for each value in the array field.+That is, given a single tuple containing an array of N values for fieldA, the `cartesianProduct` function will output N tuples, each with one value from the original tuple's array.+In essence, you can flatten arrays for further processing.For example, using `cartesianProduct` you can turn this tuple:@@ -385,18 +387,28 @@ As you can see in the examples above, the `cartesianProduct` function does suppo== classify-The `classify` function classifies tuples using a logistic regression text classification model. It was designed specifically to work with models trained using the <<stream-source-reference.adoc#train,train function>>. The `classify` function uses the <<stream-source-reference.adoc#model,model function>> to retrieve a stored model and then scores a stream of tuples using the model. The tuples read by the classifier must contain a text field that can be used for classification. The classify function uses a Lucene analyzer to extract the features from the text so the model can be applied. By default the `classify` function looks for the analyzer using the name of text field in the tuple. If the Solr schema on the worker node does not contain this field, the analyzer can be looked up in another field by specifying the `analyzerField` parameter.+The `classify` function classifies tuples using a logistic regression text classification model.+It was designed specifically to work with models trained using the <<stream-source-reference.adoc#train,train function>>. The `classify` function uses the <<stream-source-reference.adoc#model,model function>> to retrieve a stored model and then scores a stream of tuples using the model.+The tuples read by the classifier must contain a text field that can be used for classification.+The classify function uses a Lucene analyzer to extract the features from the text so the model can be applied.+By default the `classify` function looks for the analyzer using the name of text field in the tuple.+If the Solr schema on the worker node does not contain this field, the analyzer can be looked up in another field by specifying the `analyzerField` parameter.Each tuple that is classified is assigned two scores:-* probability_d*: A float between 0 and 1 which describes the probability that the tuple belongs to the class. This is useful in the classification use case.+* probability_d*: A float between 0 and 1 which describes the probability that the tuple belongs to the class.+This is useful in the classification use case.-* score_d*: The score of the document that has not be squashed between 0 and 1. The score may be positive or negative. The higher the score the better the document fits the class. This un-squashed score will be useful in query re-ranking and recommendation use cases. This score is particularly useful when multiple high ranking documents have a probability_d score of 1, which won't provide a meaningful ranking between documents.+* score_d*: The score of the document that has not be squashed between 0 and 1. The score may be positive or negative.+The higher the score the better the document fits the class.+This un-squashed score will be useful in query re-ranking and recommendation use cases.+This score is particularly useful when multiple high ranking documents have a probability_d score of 1, which won't provide a meaningful ranking between documents.=== classify Parameters* `model expression`: (Mandatory) Retrieves the stored logistic regression model.-* `field`: (Mandatory) The field in the tuples to apply the classifier to. By default the analyzer for this field in the schema will be used extract the features.+* `field`: (Mandatory) The field in the tuples to apply the classifier to.+By default the analyzer for this field in the schema will be used extract the features.* `analyzerField`: (Optional) Specifies a different field to find the analyzer from in the schema.=== classify Syntax@@ -414,16 +426,21 @@ classify(model(modelCollection,field="text_t")-----In the example above the `classify expression` is retrieving the model using the `model` function. It is then classifying tuples returned by the `search` function. The `text_t` field is used for the text classification and the analyzer for the `text_t` field in the Solr schema is used to analyze the text and extract the features.+In the example above the `classify expression` is retrieving the model using the `model` function.+It is then classifying tuples returned by the `search` function.+The `text_t` field is used for the text classification and the analyzer for the `text_t` field in the Solr schema is used to analyze the text and extract the features.== commit-The `commit` function wraps a single stream (A) and given a collection and batch size will send commit messages to the collection when the batch size is fulfilled or the end of stream is reached. A commit stream is used most frequently with an update stream and as such the commit will take into account possible summary tuples coming from the update stream. All tuples coming into the commit stream will be returned out of the commit stream - no tuples will be dropped and no tuples will be added.+The `commit` function wraps a single stream (A) and given a collection and batch size will send commit messages to the collection when the batch size is fulfilled or the end of stream is reached.+A commit stream is used most frequently with an update stream and as such the commit will take into account possible summary tuples coming from the update stream.+All tuples coming into the commit stream will be returned out of the commit stream - no tuples will be dropped and no tuples will be added.=== commit Parameters* `collection`: The collection to send commit messages to (required)-* `batchSize`: The commit batch size, sends commit message when batch size is hit. If not provided (or provided as value 0) then a commit is only sent at the end of the incoming stream.+* `batchSize`: The commit batch size, sends commit message when batch size is hit.+If not provided (or provided as value 0) then a commit is only sent at the end of the incoming stream.* `waitFlush`: The value passed directly to the commit handler (true/false, default: false)* `waitSearcher`: The value passed directly to the commit handler (true/false, default: false)* `softCommit`: The value passed directly to the commit handler (true/false, default: false)@@ -446,13 +463,16 @@ commit(== complement-The `complement` function wraps two streams (A and B) and emits tuples from A which do not exist in B. The tuples are emitted in the order in which they appear in stream A. Both streams must be sorted by the fields being used to determine equality (using the `on` parameter).+The `complement` function wraps two streams (A and B) and emits tuples from A which do not exist in B.+The tuples are emitted in the order in which they appear in stream A.+Both streams must be sorted by the fields being used to determine equality (using the `on` parameter).=== complement Parameters* `StreamExpression for StreamA`* `StreamExpression for StreamB`-* `on`: Fields to be used for checking equality of tuples between A and B. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between A and B.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== complement Syntax@@ -473,13 +493,16 @@ complement(== daemon-The `daemon` function wraps another function and runs it at intervals using an internal thread. The `daemon` function can be used to provide both continuous push and pull streaming.+The `daemon` function wraps another function and runs it at intervals using an internal thread.+The `daemon` function can be used to provide both continuous push and pull streaming.=== Continuous Push Streaming-With continuous push streaming the `daemon` function wraps another function and is then sent to the `/stream` handler for execution. The `/stream` handler recognizes the `daemon` function and keeps it resident in memory, so it can run its internal function at intervals.+With continuous push streaming the `daemon` function wraps another function and is then sent to the `/stream` handler for execution.+The `/stream` handler recognizes the `daemon` function and keeps it resident in memory, so it can run its internal function at intervals.-In order to facilitate the pushing of tuples, the `daemon` function must wrap another stream decorator that pushes the tuples somewhere. One example of this is the `update` function, which wraps a stream and sends the tuples to another SolrCloud collection for indexing.+In order to facilitate the pushing of tuples, the `daemon` function must wrap another stream decorator that pushes the tuples somewhere.+One example of this is the `update` function, which wraps a stream and sends the tuples to another SolrCloud collection for indexing.=== daemon Syntax@@ -500,11 +523,19 @@ daemon(id="uniqueId",)-----The sample code above shows a `daemon` function wrapping an `update` function, which is wrapping a `topic` function. When this expression is sent to the `/stream` handler, the `/stream` hander sees the `daemon` function and keeps it in memory where it will run at intervals. In this particular example, the `daemon` function will run the `update` function every second. The `update` function is wrapping a <<stream-source-reference.adoc#topic,`topic` function>>, which will stream tuples that match the `topic` function query in batches. Each subsequent call to the topic will return the next batch of tuples for the topic. The `update` function will send all the tuples matching the topic to another collection to be indexed. The `terminate` parameter tells the daemon to terminate when the `topic` function stops sending tuples.+The sample code above shows a `daemon` function wrapping an `update` function, which is wrapping a `topic` function.+When this expression is sent to the `/stream` handler, the `/stream` hander sees the `daemon` function and keeps it in memory where it will run at intervals.+In this particular example, the `daemon` function will run the `update` function every second.+The `update` function is wrapping a <<stream-source-reference.adoc#topic,`topic` function>>, which will stream tuples that match the `topic` function query in batches.+Each subsequent call to the topic will return the next batch of tuples for the topic.+The `update` function will send all the tuples matching the topic to another collection to be indexed.+The `terminate` parameter tells the daemon to terminate when the `topic` function stops sending tuples.-The effect of this is to push documents that match a specific query into another collection. Custom push functions can be plugged in that push documents out of Solr and into other systems, such as Kafka or an email system.+The effect of this is to push documents that match a specific query into another collection.+Custom push functions can be plugged in that push documents out of Solr and into other systems, such as Kafka or an email system.-Push streaming can also be used for continuous background aggregation scenarios where aggregates are rolled up in the background at intervals and pushed to other Solr collections. Another use case is continuous background machine learning model optimization, where the optimized model is pushed to another Solr collection where it can be integrated into queries.+Push streaming can also be used for continuous background aggregation scenarios where aggregates are rolled up in the background at intervals and pushed to other Solr collections.+Another use case is continuous background machine learning model optimization, where the optimized model is pushed to another Solr collection where it can be integrated into queries.The `/stream` handler supports a small <<stream-api.adoc#plugins,set of commands>> for listing and controlling daemon functions:@@ -538,7 +569,8 @@ This command will stop a specific daemon function and remove it from memory.=== Continuous Pull Streaming-The {solr-javadocs}/solrj/org/apache/solr/client/solrj/io/stream/DaemonStream.html[DaemonStream] java class (part of the SolrJ libraries) can also be embedded in a java application to provide continuous pull streaming. Sample code:+The {solr-javadocs}/solrj/org/apache/solr/client/solrj/io/stream/DaemonStream.html[DaemonStream] java class (part of the SolrJ libraries) can also be embedded in a java application to provide continuous pull streaming.+Sample code:[source,java]----@@ -596,7 +628,7 @@ daemonStream.close();== delete-The `delete` function wraps other functions and uses the `id` and `\_version_` values found to send the tuples to a SolrCloud collection as <<uploading-data-with-index-handlers.adoc#delete-operations,Delete By Id>> commands.+The `delete` function wraps other functions and uses the `id` and `\_version_` values found to send the tuples to a SolrCloud collection as <<indexing-with-update-handlers.adoc#delete-operations,Delete By Id>> commands.This is similar to the `<<#update,update()>>` function described below.@@ -625,7 +657,8 @@ The example above consumes the tuples returned by the `search` function against[NOTE]====-Unlike the `update()` function, `delete()` defaults to `pruneVersionField=false` -- preserving any `\_version_` values found in the inner stream when converting the tuples to "Delete By ID" requests. This ensures that using this stream will not (by default) result in deleting any documents that were updated _after_ the `search(...)` was executed, but _before_ the `delete(...)` processed that tuple (leveraging <<updating-parts-of-documents.adoc#optimistic-concurrency,Optimistic concurrency>> constraints).+Unlike the `update()` function, `delete()` defaults to `pruneVersionField=false` -- preserving any `\_version_` values found in the inner stream when converting the tuples to "Delete By ID" requests.+This ensures that using this stream will not (by default) result in deleting any documents that were updated _after_ the `search(...)` was executed, but _before_ the `delete(...)` processed that tuple (leveraging <<partial-document-updates.adoc#optimistic-concurrency,Optimistic concurrency>> constraints).Users who wish to ignore concurrent updates and delete all matched documents should set `pruneVersionField=true` (or ensure that the inner stream tuples do not include any `\_version_` values).@@ -651,14 +684,20 @@ The `eval` function then compiles the string streaming expression and emits theeval(expr)-----In the example above the `eval` expression reads the first tuple from the underlying expression. It then compiles and+In the example above the `eval` expression reads the first tuple from the underlying expression.+It then compiles andexecutes the string streaming expression in the `expr_s` field.== executor-The `executor` function wraps a stream source that contains streaming expressions, and executes the expressions in parallel. The `executor` function looks for the expression in the `expr_s` field in each tuple. The `executor` function has an internal thread pool that runs tasks that compile and run expressions in parallel on the same worker node. This function can also be parallelized across worker nodes by wrapping it in the <<parallel,`parallel`>> function to provide parallel execution of expressions across a cluster.+The `executor` function wraps a stream source that contains streaming expressions, and executes the expressions in parallel.+The `executor` function looks for the expression in the `expr_s` field in each tuple.+The `executor` function has an internal thread pool that runs tasks that compile and run expressions in parallel on the same worker node.+This function can also be parallelized across worker nodes by wrapping it in the <<parallel,`parallel`>> function to provide parallel execution of expressions across a cluster.-The `executor` function does not do anything specific with the output of the expressions that it runs. Therefore the expressions that are executed must contain the logic for pushing tuples to their destination. The <<update,update function>> can be included in the expression being executed to send the tuples to a SolrCloud collection for storage.+The `executor` function does not do anything specific with the output of the expressions that it runs.+Therefore the expressions that are executed must contain the logic for pushing tuples to their destination.+The <<update,update function>> can be included in the expression being executed to send the tuples to a SolrCloud collection for storage.This model allows for asynchronous execution of jobs where the output is stored in a SolrCloud collection where it can be accessed as the job progresses.@@ -682,18 +721,26 @@ daemon(id="myDaemon",id="myTopic")))-----In the example above a <<daemon,daemon>> wraps an executor, which wraps a <<stream-source-reference.adoc#topic,topic>> that is returning tuples with expressions to execute. When sent to the stream handler, the daemon will call the executor at intervals which will cause the executor to read from the topic and execute the expressions found in the `expr_s` field. The daemon will repeatedly call the executor until all the tuples that match the topic have been iterated, then it will terminate. This is the approach for executing batches of streaming expressions from a `topic` queue.+In the example above a <<daemon,daemon>> wraps an executor, which wraps a <<stream-source-reference.adoc#topic,topic>> that is returning tuples with expressions to execute.+When sent to the stream handler, the daemon will call the executor at intervals which will cause the executor to read from the topic and execute the expressions found in the `expr_s` field.+The daemon will repeatedly call the executor until all the tuples that match the topic have been iterated, then it will terminate.+This is the approach for executing batches of streaming expressions from a `topic` queue.== fetch-The `fetch` function iterates a stream and fetches additional fields and adds them to the tuples. The `fetch` function fetches in batches to limit the number of calls back to Solr. Tuples streamed from the `fetch` function will contain the original fields and the additional fields that were fetched. The `fetch` function supports one-to-one fetches. Many-to-one fetches, where the stream source contains duplicate keys, will also work, but one-to-many fetches are currently not supported by this function.+The `fetch` function iterates a stream and fetches additional fields and adds them to the tuples.+The `fetch` function fetches in batches to limit the number of calls back to Solr.+Tuples streamed from the `fetch` function will contain the original fields and the additional fields that were fetched.+The `fetch` function supports one-to-one fetches.+Many-to-one fetches, where the stream source contains duplicate keys, will also work, but one-to-many fetches are currently not supported by this function.=== fetch Parameters* `Collection`: (Mandatory) The collection to fetch the fields from.* `StreamExpression`: (Mandatory) The stream source for the fetch function.* `fl`: (Mandatory) The fields to be fetched.-* `on`: Fields to be used for checking equality of tuples between stream source and fetched records. Formatted as `on="fieldNameInTuple=fieldNameInCollection"`.+* `on`: Fields to be used for checking equality of tuples between stream source and fetched records.+Formatted as `on="fieldNameInTuple=fieldNameInCollection"`.* `batchSize`: (Optional) The batch fetch size.=== fetch Syntax@@ -710,14 +757,17 @@ The example above fetches addresses for users by matching the username in the tu== having-The `having` expression wraps a stream and applies a boolean operation to each tuple. It emits only tuples for which the boolean operation returns *true*.+The `having` expression wraps a stream and applies a boolean operation to each tuple.+It emits only tuples for which the boolean operation returns *true*.=== having Parameters* `StreamExpression`: (Mandatory) The stream source for the having function.-* `booleanEvaluator`: (Mandatory) The following boolean operations are supported: `eq` (equals), `gt` (greater than), `lt` (less than), `gteq` (greater than or equal to), `lteq` (less than or equal to), `and`, `or`, `eor` (exclusive or), and `not`. Boolean evaluators can be nested with other evaluators to form complex boolean logic.+* `booleanEvaluator`: (Mandatory) The following boolean operations are supported: `eq` (equals), `gt` (greater than), `lt` (less than), `gteq` (greater than or equal to), `lteq` (less than or equal to), `and`, `or`, `eor` (exclusive or), and `not`.+Boolean evaluators can be nested with other evaluators to form complex boolean logic.-The comparison evaluators compare the value in a specific field with a value, whether a string, number, or boolean. For example: `eq(field1, 10)`, returns `true` if `field1` is equal to 10.+The comparison evaluators compare the value in a specific field with a value, whether a string, number, or boolean.+For example: `eq(field1, 10)`, returns `true` if `field1` is equal to 10.=== having Syntax@@ -738,7 +788,13 @@ In this example, the `having` expression iterates the aggregated tuples from the== leftOuterJoin-The `leftOuterJoin` function wraps two streams, Left and Right, and emits tuples from Left. If there is a tuple in Right equal (as defined by `on`) then the values in that tuple will be included in the emitted tuple. An equal tuple in Right *need not* exist for the Left tuple to be emitted. This supports one-to-one, one-to-many, many-to-one, and many-to-many left outer join scenarios. The tuples are emitted in the order in which they appear in the Left stream. Both streams must be sorted by the fields being used to determine equality (using the `on` parameter). If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.+The `leftOuterJoin` function wraps two streams, Left and Right, and emits tuples from Left.+If there is a tuple in Right equal (as defined by `on`) then the values in that tuple will be included in the emitted tuple.+An equal tuple in Right *need not* exist for the Left tuple to be emitted.+This supports one-to-one, one-to-many, many-to-one, and many-to-many left outer join scenarios.+The tuples are emitted in the order in which they appear in the Left stream.+Both streams must be sorted by the fields being used to determine equality (using the `on` parameter).+If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.You can wrap the incoming streams with a `select` function to be specific about which field values are included in the emitted tuple.@@ -746,7 +802,8 @@ You can wrap the incoming streams with a `select` function to be specific about* `StreamExpression for StreamLeft`* `StreamExpression for StreamRight`-* `on`: Fields to be used for checking equality of tuples between Left and Right. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between Left and Right.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== leftOuterJoin Syntax@@ -777,17 +834,24 @@ leftOuterJoin(== hashJoin-The `hashJoin` function wraps two streams, Left and Right, and for every tuple in Left which exists in Right will emit a tuple containing the fields of both tuples. This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. The order of the streams does not matter. If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.+The `hashJoin` function wraps two streams, Left and Right, and for every tuple in Left which exists in Right will emit a tuple containing the fields of both tuples.+This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios.+The tuples are emitted in the order in which they appear in the Left stream.+The order of the streams does not matter.+If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.You can wrap the incoming streams with a `select` function to be specific about which field values are included in the emitted tuple.-The hashJoin function can be used when the tuples of Left and Right cannot be put in the same order. Because the tuples are out of order this stream functions by reading all values from the Right stream during the open operation and will store all tuples in memory. The result of this is a memory footprint equal to the size of the Right stream.+The hashJoin function can be used when the tuples of Left and Right cannot be put in the same order.+Because the tuples are out of order this stream functions by reading all values from the Right stream during the open operation and will store all tuples in memory.+The result of this is a memory footprint equal to the size of the Right stream.=== hashJoin Parameters* `StreamExpression for StreamLeft`* `hashed=StreamExpression for StreamRight`-* `on`: Fields to be used for checking equality of tuples between Left and Right. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between Left and Right.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== hashJoin Syntax@@ -818,13 +882,20 @@ hashJoin(== innerJoin-Wraps two streams, Left and Right. For every tuple in Left which exists in Right a tuple containing the fields of both tuples will be emitted. This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. Both streams must be sorted by the fields being used to determine equality (the 'on' parameter). If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple. You can wrap the incoming streams with a `select(...)` expression to be specific about which field values are included in the emitted tuple.+Wraps two streams, Left and Right.+For every tuple in Left which exists in Right a tuple containing the fields of both tuples will be emitted.+This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios.+The tuples are emitted in the order in which they appear in the Left stream.+Both streams must be sorted by the fields being used to determine equality (the 'on' parameter).+If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.+You can wrap the incoming streams with a `select(...)` expression to be specific about which field values are included in the emitted tuple.=== innerJoin Parameters* `StreamExpression for StreamLeft`* `StreamExpression for StreamRight`-* `on`: Fields to be used for checking equality of tuples between Left and Right. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between Left and Right.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== innerJoin Syntax@@ -855,13 +926,17 @@ innerJoin(== intersect-The `intersect` function wraps two streams, A and B, and emits tuples from A which *DO* exist in B. The tuples are emitted in the order in which they appear in stream A. Both streams must be sorted by the fields being used to determine equality (the `on` parameter). Only tuples from A are emitted.+The `intersect` function wraps two streams, A and B, and emits tuples from A which *DO* exist in B.+The tuples are emitted in the order in which they appear in stream A.+Both streams must be sorted by the fields being used to determine equality (the `on` parameter).+Only tuples from A are emitted.=== intersect Parameters* `StreamExpression for StreamA`* `StreamExpression for StreamB`-* `on`: Fields to be used for checking equality of tuples between A and B. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between A and B.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== intersect Syntax@@ -905,14 +980,17 @@ list(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),== merge-The `merge` function merges two or more streaming expressions and maintains the ordering of the underlying streams. Because the order is maintained, the sorts of the underlying streams must line up with the on parameter provided to the merge function.+The `merge` function merges two or more streaming expressions and maintains the ordering of the underlying streams.+Because the order is maintained, the sorts of the underlying streams must line up with the on parameter provided to the merge function.=== merge Parameters* `StreamExpression A`* `StreamExpression B`* `Optional StreamExpression C,D,....Z`-* `on`: Sort criteria for performing the merge. Of the form `fieldName order` where order is `asc` or `desc`. Multiple fields can be provided in the form `fieldA order, fieldB order`.+* `on`: Sort criteria for performing the merge.+Of the form `fieldName order` where order is `asc` or `desc`.+Multiple fields can be provided in the form `fieldA order, fieldB order`.=== merge Syntax@@ -963,9 +1041,13 @@ merge(== null-The null expression is a useful utility function for understanding bottlenecks when performing parallel relational algebra (joins, intersections, rollups etc.). The null function reads all the tuples from an underlying stream and returns a single tuple with the count and processing time. Because the null stream adds minimal overhead of its own, it can be used to isolate the performance of Solr's /export handler. If the /export handlers performance is not the bottleneck, then the bottleneck is likely occurring in the workers where the stream decorators are running.+The null expression is a useful utility function for understanding bottlenecks when performing parallel relational algebra (joins, intersections, rollups etc.).+The null function reads all the tuples from an underlying stream and returns a single tuple with the count and processing time.+Because the null stream adds minimal overhead of its own, it can be used to isolate the performance of Solr's `/export` handler.+If the /export handlers performance is not the bottleneck, then the bottleneck is likely occurring in the workers where the stream decorators are running.-The null expression can be wrapped by the parallel function and sent to worker nodes. In this scenario each worker will return one tuple with the count of tuples processed on the worker and the timing information for that worker. This gives valuable information such as:+The null expression can be wrapped by the parallel function and sent to worker nodes.+In this scenario each worker will return one tuple with the count of tuples processed on the worker and the timing information for that worker.\This gives valuable information such as:1. As more workers are added does the performance of the /export handler improve or not.2. Are tuples being evenly distributed across the workers, or is the hash partitioning sending more documents to a single worker.@@ -986,21 +1068,32 @@ The null expression can be wrapped by the parallel function and sent to worker nsort="a_s desc")-----The expression above shows a parallel function wrapping a null function. This will cause the null function to be run in parallel across 20 worker nodes. Each worker will return a single tuple with number of tuples processed and time it took to iterate the tuples.+The expression above shows a parallel function wrapping a null function.+This will cause the null function to be run in parallel across 20 worker nodes.+Each worker will return a single tuple with number of tuples processed and time it took to iterate the tuples.== outerHashJoin-The `outerHashJoin` function wraps two streams, Left and Right, and emits tuples from Left. If there is a tuple in Right equal (as defined by the `on` parameter) then the values in that tuple will be included in the emitted tuple. An equal tuple in Right *need not* exist for the Left tuple to be emitted. This supports one-to-one, one-to-many, many-to-one, and many-to-many left outer join scenarios. The tuples are emitted in the order in which they appear in the Left stream. The order of the streams does not matter. If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.+The `outerHashJoin` function wraps two streams, Left and Right, and emits tuples from Left.+If there is a tuple in Right equal (as defined by the `on` parameter) then the values in that tuple will be included in the emitted tuple.+An equal tuple in Right *need not* exist for the Left tuple to be emitted.+This supports one-to-one, one-to-many, many-to-one, and many-to-many left outer join scenarios.+The tuples are emitted in the order in which they appear in the Left stream.+The order of the streams does not matter.+If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.You can wrap the incoming streams with a `select` function to be specific about which field values are included in the emitted tuple.-The outerHashJoin stream can be used when the tuples of Left and Right cannot be put in the same order. Because the tuples are out of order, this stream functions by reading all values from the Right stream during the open operation and will store all tuples in memory. The result of this is a memory footprint equal to the size of the Right stream.+The outerHashJoin stream can be used when the tuples of Left and Right cannot be put in the same order.+Because the tuples are out of order, this stream functions by reading all values from the Right stream during the open operation and will store all tuples in memory.+The result of this is a memory footprint equal to the size of the Right stream.=== outerHashJoin Parameters* `StreamExpression for StreamLeft`* `hashed=StreamExpression for StreamRight`-* `on`: Fields to be used for checking equality of tuples between Left and Right. Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.+* `on`: Fields to be used for checking equality of tuples between Left and Right.+Can be of the format `on="fieldName"`, `on="fieldNameInLeft=fieldNameInRight"`, or `on="fieldName, otherFieldName=rightOtherFieldName"`.=== outerHashJoin Syntax@@ -1033,20 +1126,27 @@ outerHashJoin(The `parallel` function wraps a streaming expression and sends it to N worker nodes to be processed in parallel.-The `parallel` function requires that the `partitionKeys` parameter be provided to the underlying searches. The `partitionKeys` parameter will partition the search results (tuples) across the worker nodes. Tuples with the same values as `partitionKeys` will be shuffled to the same worker nodes.+The `parallel` function requires that the `partitionKeys` parameter be provided to the underlying searches.+The `partitionKeys` parameter will partition the search results (tuples) across the worker nodes.+Tuples with the same values as `partitionKeys` will be shuffled to the same worker nodes.The `parallel` function maintains the sort order of the tuples returned by the worker nodes, so the sort criteria must incorporate the sort order of the tuples returned by the workers.For example if you sort on year, month and day you could partition on year only as long as there are enough different years to spread the tuples around the worker nodes.-Solr allows sorting on more than 4 fields, but you cannot specify more than 4 partitionKeys for speed considerations. Also it's overkill to specify many `partitionKeys` when we one or two keys could be enough to spread the tuples.+Solr allows sorting on more than 4 fields, but you cannot specify more than 4 partitionKeys for speed considerations.+Also it's overkill to specify many `partitionKeys` when we one or two keys could be enough to spread the tuples.-Parallel stream was designed when the underlying search stream will emit a lot of tuples from the collection. If the search stream only emits a small subset of the data from the collection using `parallel` could potentially be slower.+Parallel stream was designed when the underlying search stream will emit a lot of tuples from the collection.+If the search stream only emits a small subset of the data from the collection using `parallel` could potentially be slower..Worker Collections[TIP]====-The worker nodes can be from the same collection as the data, or they can be a different collection entirely, even one that only exists for `parallel` streaming expressions. A worker collection can be any SolrCloud collection that has the `/stream` handler configured. Unlike normal SolrCloud collections, worker collections don't have to hold any data. Worker collections can be empty collections that exist only to execute streaming expressions.+The worker nodes can be from the same collection as the data, or they can be a different collection entirely, even one that only exists for `parallel` streaming expressions.+A worker collection can be any SolrCloud collection that has the `/stream` handler configured.+Unlike normal SolrCloud collections, worker collections don't have to hold any data.+Worker collections can be empty collections that exist only to execute streaming expressions.======= parallel Parameters@@ -1069,12 +1169,14 @@ The worker nodes can be from the same collection as the data, or they can be a dsort="year_i desc")-----The expression above shows a `parallel` function wrapping a `rollup` function. This will cause the `rollup` function to be run in parallel across 20 worker nodes.+The expression above shows a `parallel` function wrapping a `rollup` function.+This will cause the `rollup` function to be run in parallel across 20 worker nodes..Warmup[TIP]====-The `parallel` function uses the hash query parser to split the data amongst the workers. It executes on all the documents and the result bitset is cached in the filterCache.+The `parallel` function uses the hash query parser to split the data amongst the workers.+It executes on all the documents and the result bitset is cached in the filterCache.+For a `parallel` stream with the same number of workers and `partitonKeys` the first query would be slower than subsequent queries.A trick to not pay the penalty for the first slow query would be to use a warmup query for every new searcher.@@ -1094,8 +1196,10 @@ The following is a `solrconfig.xml` snippet for 2 workers and "year_i" as the `p== plistThe `plist` function wraps N Stream Expressions and opens the streams in parallel-and iterates each stream sequentially. The difference between the `list` and `plist` is that-the streams are opened in parallel. Since many streams such as+and iterates each stream sequentially.+The difference between the `list` and `plist` is that+the streams are opened in parallel.+Since many streams such as`facet`, `stats` and `significantTerms` push down heavy operations to Solr when they are opened,the plist function can dramatically improve performance by doing these operations in parallel.@@ -1118,13 +1222,19 @@ plist(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),== priority-The `priority` function is a simple priority scheduler for the <<executor>> function. The `executor` function doesn't directly have a concept of task prioritization; instead it simply executes tasks in the order that they are read from its underlying stream. The `priority` function provides the ability to schedule a higher priority task ahead of lower priority tasks that were submitted earlier.+The `priority` function is a simple priority scheduler for the <<executor>> function.+The `executor` function doesn't directly have a concept of task prioritization; instead it simply executes tasks in the order that they are read from its underlying stream.+The `priority` function provides the ability to schedule a higher priority task ahead of lower priority tasks that were submitted earlier.-The `priority` function wraps two <<stream-source-reference.adoc#topic,topics>> that are both emitting tuples that contain streaming expressions to execute. The first topic is considered the higher priority task queue.+The `priority` function wraps two <<stream-source-reference.adoc#topic,topics>> that are both emitting tuples that contain streaming expressions to execute.+The first topic is considered the higher priority task queue.-Each time the `priority` function is called, it checks the higher priority task queue to see if there are any tasks to execute. If tasks are waiting in the higher priority queue then the priority function will emit the higher priority tasks. If there are no high priority tasks to run, the lower priority queue tasks are emitted.+Each time the `priority` function is called, it checks the higher priority task queue to see if there are any tasks to execute.+If tasks are waiting in the higher priority queue then the priority function will emit the higher priority tasks.+If there are no high priority tasks to run, the lower priority queue tasks are emitted.-The `priority` function will only emit a batch of tasks from one of the queues each time it is called. This ensures that no lower priority tasks are executed until the higher priority queue has no tasks to run.+The `priority` function will only emit a batch of tasks from one of the queues each time it is called.+This ensures that no lower priority tasks are executed until the higher priority queue has no tasks to run.=== priority Parameters@@ -1141,17 +1251,23 @@ daemon(id="myDaemon",topic(checkpointCollection, storedExpressions, q="priority:low", fl="id, expr_s", initialCheckPoint=0,id="lowPriorityTasks"))))-----In the example above the `daemon` function is calling the executor iteratively. Each time it's called, the `executor` function will execute the tasks emitted by the `priority` function. The `priority` function wraps two topics. The first topic is the higher priority task queue, the second topics is the lower priority topic.+In the example above the `daemon` function is calling the executor iteratively.+Each time it's called, the `executor` function will execute the tasks emitted by the `priority` function.+The `priority` function wraps two topics.+The first topic is the higher priority task queue, the second topics is the lower priority topic.== reduceThe `reduce` function wraps an internal stream and groups tuples by common fields.-Each tuple group is operated on as a single block by a pluggable reduce operation. The group operation provided with Solr implements distributed grouping functionality. The group operation also serves as an example reduce operation that can be referred to when building custom reduce operations.+Each tuple group is operated on as a single block by a pluggable reduce operation.+The group operation provided with Solr implements distributed grouping functionality.+The group operation also serves as an example reduce operation that can be referred to when building custom reduce operations.[IMPORTANT]====-The reduce function relies on the sort order of the underlying stream. Accordingly the sort order of the underlying stream must be aligned with the group by field.+The reduce function relies on the sort order of the underlying stream.+Accordingly the sort order of the underlying stream must be aligned with the group by field.======= reduce Parameters@@ -1172,15 +1288,22 @@ reduce(search(collection1, q="*:*", qt="/export", fl="id,a_s,a_i,a_f", sort="a_s== rollup-The `rollup` function wraps another stream function and rolls up aggregates over bucket fields. The rollup function relies on the sort order of the underlying stream to rollup aggregates one grouping at a time. Accordingly, the sort order of the underlying stream must match the fields in the `over` parameter of the rollup function.+The `rollup` function wraps another stream function and rolls up aggregates over bucket fields.+The rollup function relies on the sort order of the underlying stream to rollup aggregates one grouping at a time.+Accordingly, the sort order of the underlying stream must match the fields in the `over` parameter of the rollup function.-The rollup function also needs to process entire result sets in order to perform its aggregations. When the underlying stream is the `search` function, the `/export` handler can be used to provide full sorted result sets to the rollup function. This sorted approach allows the rollup function to perform aggregations over very high cardinality fields. The disadvantage of this approach is that the tuples must be sorted and streamed across the network to a worker node to be aggregated. For faster aggregation over low to moderate cardinality fields, the `facet` function can be used.+The rollup function also needs to process entire result sets in order to perform its aggregations.+When the underlying stream is the `search` function, the `/export` handler can be used to provide full sorted result sets to the rollup function.+This sorted approach allows the rollup function to perform aggregations over very high cardinality fields.+The disadvantage of this approach is that the tuples must be sorted and streamed across the network to a worker node to be aggregated.+For faster aggregation over low to moderate cardinality fields, the `facet` function can be used.=== rollup Parameters* `StreamExpression` (Mandatory)* `over`: (Mandatory) A list of fields to group by.-* `metrics`: (Mandatory) The list of metrics to compute. Currently supported metrics are `sum(col)`, `avg(col)`, `min(col)`, `max(col)`, `count(*)`.+* `metrics`: (Mandatory) The list of metrics to compute.+Currently supported metrics are `sum(col)`, `avg(col)`, `min(col)`, `max(col)`, `count(*)`.=== rollup Syntax@@ -1201,7 +1324,10 @@ rollup()-----The example about shows the rollup function wrapping the search function. Notice that search function is using the `/export` handler to provide the entire result set to the rollup stream. Also notice that the search function's `sort` parameter matches up with the rollup's `over` parameter. This allows the rollup function to rollup the over the `a_s` field, one group at a time.+The example about shows the rollup function wrapping the search function.+Notice that search function is using the `/export` handler to provide the entire result set to the rollup stream.+Also notice that the search function's `sort` parameter matches up with the rollup's `over` parameter.+This allows the rollup function to rollup the over the `a_s` field, one group at a time.== scoreNodes@@ -1209,14 +1335,18 @@ See section in <<graph-traversal.adoc#using-the-scorenodes-function-to-make-a-re== select-The `select` function wraps a streaming expression and outputs tuples containing a subset or modified set of fields from the incoming tuples. The list of fields included in the output tuple can contain aliases to effectively rename fields. The `select` stream supports both operations and evaluators. One can provide a list of operations and evaluators to perform on any fields, such as `replace, add, if`, etc.+The `select` function wraps a streaming expression and outputs tuples containing a subset or modified set of fields from the incoming tuples.+The list of fields included in the output tuple can contain aliases to effectively rename fields.+The `select` stream supports both operations and evaluators.+One can provide a list of operations and evaluators to perform on any fields, such as `replace, add, if`, etc.=== select Parameters* `StreamExpression`* `fieldName`: name of field to include in the output tuple (can include multiple of these), such as `outputTuple[fieldName] = inputTuple[fieldName]`* `fieldName as aliasFieldName`: aliased field name to include in the output tuple (can include multiple of these), such as `outputTuple[aliasFieldName] = incomingTuple[fieldName]`-* `replace(fieldName, value, withValue=replacementValue)`: if `incomingTuple[fieldName] == value` then `outgoingTuple[fieldName]` will be set to `replacementValue`. `value` can be the string "null" to replace a null value with some other value.+* `replace(fieldName, value, withValue=replacementValue)`: if `incomingTuple[fieldName] == value` then `outgoingTuple[fieldName]` will be set to `replacementValue`.+`value` can be the string "null" to replace a null value with some other value.* `replace(fieldName, value, withField=otherFieldName)`: if `incomingTuple[fieldName] == value` then `outgoingTuple[fieldName]` will be set to the value of `incomingTuple[otherFieldName]`. `value` can be the string "null" to replace a null value with some other value.=== select Syntax@@ -1237,7 +1367,10 @@ select(== sort-The `sort` function wraps a streaming expression and re-orders the tuples. The sort function emits all incoming tuples in the new sort order. The sort function reads all tuples from the incoming stream, re-orders them using an algorithm with `O(nlog(n))` performance characteristics, where n is the total number of tuples in the incoming stream, and then outputs the tuples in the new sort order. Because all tuples are read into memory, the memory consumption of this function grows linearly with the number of tuples in the incoming stream.+The `sort` function wraps a streaming expression and re-orders the tuples.+The sort function emits all incoming tuples in the new sort order.+The sort function reads all tuples from the incoming stream, re-orders them using an algorithm with `O(nlog(n))` performance characteristics, where n is the total number of tuples in the incoming stream, and then outputs the tuples in the new sort order.+Because all tuples are read into memory, the memory consumption of this function grows linearly with the number of tuples in the incoming stream.=== sort Parameters@@ -1246,7 +1379,8 @@ The `sort` function wraps a streaming expression and re-orders the tuples. The s=== sort Syntax-The expression below finds dog owners and orders the results by owner and pet name. Notice that it uses an efficient innerJoin by first ordering by the person/owner id and then re-orders the final output by the owner and pet names.+The expression below finds dog owners and orders the results by owner and pet name.+Notice that it uses an efficient innerJoin by first ordering by the person/owner id and then re-orders the final output by the owner and pet names.[source,text]----@@ -1262,7 +1396,9 @@ sort(== top-The `top` function wraps a streaming expression and re-orders the tuples. The top function emits only the top N tuples in the new sort order. The top function re-orders the underlying stream so the sort criteria *does not* have to match up with the underlying stream.+The `top` function wraps a streaming expression and re-orders the tuples.+The top function emits only the top N tuples in the new sort order.+The top function re-orders the underlying stream so the sort criteria *does not* have to match up with the underlying stream.=== top Parameters@@ -1272,7 +1408,9 @@ The `top` function wraps a streaming expression and re-orders the tuples. The to=== top Syntax-The expression below finds the top 3 results of the underlying search. Notice that it reverses the sort order. The top function re-orders the results of the underlying stream.+The expression below finds the top 3 results of the underlying search.+Notice that it reverses the sort order.+The top function re-orders the results of the underlying stream.[source,text]----@@ -1287,9 +1425,13 @@ top(n=3,== unique-The `unique` function wraps a streaming expression and emits a unique stream of tuples based on the `over` parameter. The unique function relies on the sort order of the underlying stream. The `over` parameter must match up with the sort order of the underlying stream.+The `unique` function wraps a streaming expression and emits a unique stream of tuples based on the `over` parameter.+The unique function relies on the sort order of the underlying stream.+The `over` parameter must match up with the sort order of the underlying stream.-The unique function implements a non-co-located unique algorithm. This means that records with the same unique `over` field do not need to be co-located on the same shard. When executed in the parallel, the `partitionKeys` parameter must be the same as the unique `over` field so that records with the same keys will be shuffled to the same worker.+The unique function implements a non-co-located unique algorithm.+This means that records with the same unique `over` field do not need to be co-located on the same shard.+When executed in the parallel, the `partitionKeys` parameter must be the same as the unique `over` field so that records with the same keys will be shuffled to the same worker.=== unique Parameters@@ -1336,4 +1478,5 @@ The `update` function wraps another functions and sends the tuples to a SolrClouThe example above sends the tuples returned by the `search` function to the `destinationCollection` to be indexed.-Wrapping `search(...)` as showing in this example is the common case usage of this decorator: to read documents from a collection as tuples, process or modify them in some way, and then add them back to a new collection. For this reason, `pruneVersionField=true` is the default behavior -- stripping any `\_version_` values found in the inner stream when converting the tuples to Solr documents to prevent any unexpected errors from <<updating-parts-of-documents.adoc#optimistic-concurrency,Optimistic concurrency>> constraints.+Wrapping `search(...)` as showing in this example is the common case usage of this decorator: to read documents from a collection as tuples, process or modify them in some way, and then add them back to a new collection.+For this reason, `pruneVersionField=true` is the default behavior -- stripping any `\_version_` values found in the inner stream when converting the tuples to Solr documents to prevent any unexpected errors from <<partial-document-updates.adoc#optimistic-concurrency,Optimistic concurrency>> constraints.diff --git a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc b/solr/solr-ref-guide/src/stream-evaluator-reference.adocindex 44efd5d6690..7cb6087dbb5 100644--- a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc+++ b/solr/solr-ref-guide/src/stream-evaluator-reference.adoc@@ -18,20 +18,21 @@// under the License.-Stream evaluators are different then stream sources or stream decorators. Both-stream sources and stream decorators return streams of tuples. Stream evaluators are more like-a traditional function that evaluates its parameters and-returns an result. That result can be a single value, array, map or other structure.+Stream evaluators are different then stream sources or stream decorators.+Both stream sources and stream decorators return streams of tuples.+Stream evaluators are more like a traditional function that evaluates its parameters and returns an result.+That result can be a single value, array, map or other structure.-Stream evaluators can be nested so that the output of an evaluator becomes the input-for another evaluator.+Stream evaluators can be nested so that the output of an evaluator becomes the input for another evaluator.-Stream evaluators can be called in different contexts. For example a stream evaluator-can be called on its own or it can be called within the context of a streaming expression.+Stream evaluators can be called in different contexts.+For example a stream evaluator can be called on its own or it can be called within the context of a streaming expression.== abs-The `abs` function will return the absolute value of the provided single parameter. The `abs` function will fail to execute if the value is non-numeric. If a null value is found then null will be returned as the result.+The `abs` function will return the absolute value of the provided single parameter.+The `abs` function will fail to execute if the value is non-numeric.+If a null value is found then null will be returned as the result.=== abs Parameters@@ -39,7 +40,9 @@ The `abs` function will return the absolute value of the provided single paramet=== abs Syntax-The expressions below show the various ways in which you can use the `abs` evaluator. Only one parameter is accepted. Returns a numeric value.+The expressions below show the various ways in which you can use the `abs` evaluator.+Only one parameter is accepted.+Returns a numeric value.[source,text]----@@ -68,7 +71,9 @@ if(gt(fieldA,fieldB),sin(fieldA),sin(fieldB)) // if fieldA > fieldB then return== add-The `add` function will take 2 or more numeric values and add them together. The `add` function will fail to execute if any of the values are non-numeric. If a null value is found then null will be returned as the result.+The `add` function will take 2 or more numeric values and add them together.+The `add` function will fail to execute if any of the values are non-numeric.+If a null value is found then null will be returned as the result.=== add Parameters@@ -79,7 +84,9 @@ The `add` function will take 2 or more numeric values and add them together. The=== add Syntax-The expressions below show the various ways in which you can use the `add` evaluator. The number and order of these parameters do not matter and is not limited except that at least two parameters are required. Returns a numeric value.+The expressions below show the various ways in which you can use the `add` evaluator.+The number and order of these parameters do not matter and is not limited except that at least two parameters are required.+Returns a numeric value.[source,text]----@@ -93,9 +100,8 @@ add(fieldA,if(gt(fieldA,fieldB),fieldA,fieldB)) // if fieldA > fieldB then field== analyze-The `analyze` function analyzes text using a Lucene/Solr analyzer and returns a list of tokens-emitted by the analyzer. The `analyze` function can be called on its own or within the-`<<stream-decorator-reference.adoc#select,select>>` and `<<stream-decorator-reference.adoc#cartesianproduct,cartesianProduct>>` streaming expressions.+The `analyze` function analyzes text using a Lucene/Solr analyzer and returns a list of tokens emitted by the analyzer.+The `analyze` function can be called on its own or within the `<<stream-decorator-reference.adoc#select,select>>` and `<<stream-decorator-reference.adoc#cartesianproduct,cartesianProduct>>` streaming expressions.=== analyze Parameters@@ -107,12 +113,16 @@ emitted by the analyzer. The `analyze` function can be called on its own or withThe expressions below show the various ways in which you can use the `analyze` evaluator.* Analyze the raw text: `analyze("hello world", analyzerField)`-* Analyze a text field within a `select` expression. This will annotate tuples with the output of the analyzer: `select(expr, analyze(textField, analyzerField) as outField)`-* Analyze a text field with a `cartesianProduct` expression. This will stream each token emitted by the analyzer in its own tuple: `cartesianProduct(expr, analyze(textField, analyzer) as outField)`+* Analyze a text field within a `select` expression.+This will annotate tuples with the output of the analyzer: `select(expr, analyze(textField, analyzerField) as outField)`+* Analyze a text field with a `cartesianProduct` expression.+This will stream each token emitted by the analyzer in its own tuple: `cartesianProduct(expr, analyze(textField, analyzer) as outField)`== and-The `and` function will return the logical AND of at least 2 boolean parameters. The function will fail to execute if any parameters are non-boolean or null. Returns a boolean value.+The `and` function will return the logical AND of at least 2 boolean parameters.+The function will fail to execute if any parameters are non-boolean or null.+Returns a boolean value.=== and Parameters@@ -123,7 +133,8 @@ The `and` function will return the logical AND of at least 2 boolean parameters.=== and Syntax-The expressions below show the various ways in which you can use the `and` evaluator. At least two parameters are required, but there is no limit to how many you can use.+The expressions below show the various ways in which you can use the `and` evaluator.+At least two parameters are required, but there is no limit to how many you can use.[source,text]----@@ -198,9 +209,8 @@ if(gt(fieldA,fieldB),atan(fieldA),atan(fieldB)) // if fieldA > fieldB then retur== betaDistribution-The `betaDistribution` function returns a https://en.wikipedia.org/wiki/Beta_distribution[beta probability distribution]-based on its parameters. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+The `betaDistribution` function returns a https://en.wikipedia.org/wiki/Beta_distribution[beta probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.=== betaDistribution Parameters@@ -218,8 +228,7 @@ betaDistribution(1, 5)== binomialCoefficient-The `binomialCoefficient` function returns a https://en.wikipedia.org/wiki/Binomial_coefficient[Binomial Coefficient], the number of k-element subsets that can-be selected from an n-element set.+The `binomialCoefficient` function returns a https://en.wikipedia.org/wiki/Binomial_coefficient[Binomial Coefficient], the number of k-element subsets that can be selected from an n-element set.=== binomialCoefficient Parameters@@ -237,9 +246,8 @@ binomialCoefficient(8, 3) // Returns the number of 3 element subsets from an 8 e== binomialDistribution-The `binomialDistribution` function returns a https://en.wikipedia.org/wiki/Binomial_distribution[binomial probability distribution]-based on its parameters. This function is part of the probability distribution framework and is designed to-work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.+The `binomialDistribution` function returns a https://en.wikipedia.org/wiki/Binomial_distribution[binomial probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.=== binomialDistribution Parameters@@ -293,8 +301,8 @@ if(gt(fieldA,fieldB),ceil(fieldA),ceil(fieldB)) // if fieldA > fieldB then retur== col-The `col` function returns a numeric array from a list of tuples. The `col`-function is used to create numeric arrays from stream sources.+The `col` function returns a numeric array from a list of tuples.+The `col` function is used to create numeric arrays from stream sources.=== col Parameters@@ -346,8 +354,7 @@ integer: number columns in the matrix.== constantDistributionThe `constantDistribution` function returns a constant probability distribution based on its parameter.-This function is part of the probability distribution framework and is designed to-work with the `<<sample>>` and `<<cumulativeProbability>>` functions.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>` and `<<cumulativeProbability>>` functions.When sampled the constant distribution always returns its constant value.@@ -385,7 +392,8 @@ The `copyOf` function creates a copy of a numeric array.=== copyOf Parameters* `numeric array`-* `length`: The length of the copied array. The returned array will be right padded with zeros if the length parameter exceeds the size of the original array.+* `length`: The length of the copied array.+The returned array will be right padded with zeros if the length parameter exceeds the size of the original array.=== copyOf Syntax@@ -421,11 +429,14 @@ The `corr` function support Pearson's, Kendall's and Spearman's correlations.OR-* `matrix`: The matrix to compute the correlation matrix for. Note that correlation is computed between the `columns` in the matrix.+* `matrix`: The matrix to compute the correlation matrix for.+Note that correlation is computed between the `columns` in the matrix.=== corr Named Parameters-* `type`: (Optional) The type of correlation. Possible values are `pearsons`, `kendalls`, or `spearmans`. The default is `pearsons`.+* `type`: (Optional) The type of correlation.+Possible values are `pearsons`, `kendalls`, or `spearmans`.+The default is `pearsons`.=== corr Syntax@@ -486,7 +497,8 @@ The `cov` function returns the covariance of two numeric array or the covarianceOR-* `matrix`: The matrix to compute the covariance matrix from. Note that covariance is computed between the `columns` in the matrix.+* `matrix`: The matrix to compute the covariance matrix from.+Note that covariance is computed between the `columns` in the matrix.=== cov Syntax@@ -500,9 +512,8 @@ number | matrix: Either the covariance or covariance matrix.== cumulativeProbability-The `cumulativeProbability` function returns the cumulative probability of a random variable within a-probability distribution. The cumulative probability is the total probability of-all random variables less then or equal to a random variable.+The `cumulativeProbability` function returns the cumulative probability of a random variable within a probability distribution.+The cumulative probability is the total probability of all random variables less then or equal to a random variable.=== cumulativeProbability Parameters@@ -520,9 +531,9 @@ cumulativeProbability(normalDistribution(500, 25), 502) // Returns the cumulativ== derivative-The `derivative` function returns the https://en.wikipedia.org/wiki/Derivative[derivative] of a function. The derivative function-can compute the derivative of the <<spline>> function and the <<loess>> function. The derivative can also-take the derivative of a derivative.+The `derivative` function returns the https://en.wikipedia.org/wiki/Derivative[derivative] of a function.+The derivative function can compute the derivative of the <<spline>> function and the <<loess>> function.+The derivative can also take the derivative of a derivative.=== derivative Parameters@@ -561,7 +572,8 @@ Time series differencing is often used to make a time series stationary before f=== diff Parameters* `numeric array`: The time series data.-* `integer`: (Optional) The lag. Defaults to 1.+* `integer`: (Optional) The lag.+Defaults to 1.=== diff Syntax@@ -571,7 +583,8 @@ diff(numericArray1, 30) // Perform time series differencing with a lag of 30.=== diff Returns-numeric array: The differenced time series data. The size of the array will be equal to (original array size - lag).+numeric array: The differenced time series data.+The size of the array will be equal to (original array size - lag).== distance@@ -584,11 +597,14 @@ The `distance` function computes the distance of two numeric arrays or the distaOR-* `matrix`: The matrix to compute the distance matrix for. Note that distance is computed between the `columns` in the matrix.+* `matrix`: The matrix to compute the distance matrix for.+Note that distance is computed between the `columns` in the matrix.=== distance Named Parameters-* `type`: (Optional) The distance type. Possible values are `euclidean`, `manhattan`, `canberra`, or `earthMovers`. The default is `euclidean`.+* `type`: (Optional) The distance type.+Possible values are `euclidean`, `manhattan`, `canberra`, or `earthMovers`.+The default is `euclidean`.=== distance Syntax@@ -604,7 +620,8 @@ number | matrix: Either the distance or distance matrix.== div-The `div` function will take two numeric values and divide them. The function will fail to execute if any of the values are non-numeric or null, or the 2nd value is 0. Returns a numeric value.+The `div` function will take two numeric values and divide them.+The function will fail to execute if any of the values are non-numeric or null, or the 2nd value is 0. Returns a numeric value.=== div Parameters@@ -613,7 +630,8 @@ The `div` function will take two numeric values and divide them. The function wi=== div Syntax-The expressions below show the various ways in which you can use the `div` evaluator. The first value will be divided by the second and as such the second cannot be 0.+The expressions below show the various ways in which you can use the `div` evaluator.+The first value will be divided by the second and as such the second cannot be 0.[source,text]----@@ -715,10 +733,11 @@ ebeSubtract(numericArray, numericArray)== empiricalDistribution-The `empiricalDistribution` function returns https://en.wikipedia.org/wiki/Empirical_distribution_function[empirical distribution function], a continuous probability distribution function based-on an actual data set. This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+The `empiricalDistribution` function returns https://en.wikipedia.org/wiki/Empirical_distribution_function[empirical distribution function], a continuous probability distribution function based on an actual data set.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.-This function is designed to work with continuous data. To build a distribution from+This function is designed to work with continuous data.+To build a distribution froma discrete data set use the `<<enumeratedDistribution>>`.=== empiricalDistribution Parameters@@ -736,21 +755,18 @@ empiricalDistribution(numericArray)== enumeratedDistribution-The `enumeratedDistribution` function returns a discrete probability distribution function based-on an actual data set or a pre-defined set of data and probabilities.-This function is part of the probability distribution framework and is designed to-work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.+The `enumeratedDistribution` function returns a discrete probability distribution function based on an actual data set or a pre-defined set of data and probabilities.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.The enumeratedDistribution can be called in two different scenarios:-1) Single array of discrete values. This works like an empirical distribution for-discrete data.+. Single array of discrete values.+This works like an empirical distribution for discrete data.-2) An array of singleton discrete values and an array of double values representing-the probabilities of the discrete values.+. An array of singleton discrete values and an array of double values representing the probabilities of the discrete values.-This function is designed to work with discrete data. To build a distribution from-a continuous data set use the `<<empiricalDistribution>>`.+This function is designed to work with discrete data.+To build a distribution from a continuous data set use the `<<empiricalDistribution>>`.=== enumeratedDistribution Parameters@@ -769,7 +785,9 @@ enumeratedDistribution(array(1,2,3,4), array(.25,.25,.25,.25)) // This creates a== eor-The `eor` function will return the logical exclusive or of at least two boolean parameters. The function will fail to execute if any parameters are non-boolean or null. Returns a boolean value.+The `eor` function will return the logical exclusive or of at least two boolean parameters.+The function will fail to execute if any parameters are non-boolean or null.+Returns a boolean value.=== eor Parameters@@ -780,7 +798,8 @@ The `eor` function will return the logical exclusive or of at least two boolean=== eor Syntax-The expressions below show the various ways in which you can use the `eor` evaluator. At least two parameters are required, but there is no limit to how many you can use.+The expressions below show the various ways in which you can use the `eor` evaluator.+At least two parameters are required, but there is no limit to how many you can use.[source,text]----@@ -791,7 +810,11 @@ eor(eq(fieldA,fieldB),eq(fieldC,fieldD)) // true iff either fieldA == fieldB or== eq-The `eq` function will return whether all the parameters are equal, as per Java's standard `equals(...)` function. The function accepts parameters of any type, but will fail to execute if all the parameters are not of the same type. That is, all are Boolean, all are String, or all are Numeric. If any any parameters are null and there is at least one parameter that is not null then false will be returned. Returns a boolean value.+The `eq` function will return whether all the parameters are equal, as per Java's standard `equals(...)` function.+The function accepts parameters of any type, but will fail to execute if all the parameters are not of the same type.+That is, all are Boolean, all are String, or all are Numeric.+If any any parameters are null and there is at least one parameter that is not null then false will be returned.+Returns a boolean value.=== eq Parameters@@ -823,7 +846,8 @@ The `expMovingAverage` function computes an https://en.wikipedia.org/wiki/Movin=== expMovingAvg Returns-A numeric array. The first element of the returned array will start from the windowSize-1 index of the original array.+A numeric array.+The first element of the returned array will start from the windowSize-1 index of the original array.=== expMovingAvg Syntax@@ -838,7 +862,8 @@ The `factorial` function returns the https://en.wikipedia.org/wiki/Factorial[fac=== factorial Parameters-* `integer`: The value to compute the factorial for. The largest supported value of this parameter is 170.+* `integer`: The value to compute the factorial for.+The largest supported value of this parameter is 170.=== factorial Returns@@ -885,11 +910,10 @@ if(gt(fieldA,fieldB),floor(fieldA),floor(fieldB)) // if fieldA > fieldB then ret== freqTable-The `freqTable` function returns a https://en.wikipedia.org/wiki/Frequency_distribution[frequency distribution] from-an array of discrete values.+The `freqTable` function returns a https://en.wikipedia.org/wiki/Frequency_distribution[frequency distribution] from an array of discrete values.-This function is designed to work with discrete values. To work with continuous data-use the `<<hist>>` function.+This function is designed to work with discrete values.+To work with continuous data use the `<<hist>>` function.=== freqTable Parameters@@ -908,8 +932,8 @@ freqTable(integerArray)== gammaDistribution-The `gammaDistribution` function returns a https://en.wikipedia.org/wiki/Gamma_distribution[gamma probability distribution] based on its parameters. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+The `gammaDistribution` function returns a https://en.wikipedia.org/wiki/Gamma_distribution[gamma probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.=== gammaDistribution Parameters@@ -929,8 +953,8 @@ gammaDistribution(1, 10)== geometricDistribution-The `geometricDistribution` function returns a https://en.wikipedia.org/wiki/Geometric_distribution[geometric probability distribution] based on its parameters. This function is part of the-probability distribution framework and is designed to work with the <<sample>>, <<probability>> and <<cumulativeProbability>> functions.+The `geometricDistribution` function returns a https://en.wikipedia.org/wiki/Geometric_distribution[geometric probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the <<sample>>, <<probability>> and <<cumulativeProbability>> functions.=== geometricDistribution Parameters@@ -947,10 +971,11 @@ A probability distribution function== getAttribute-The `getAttribute` function returns an attribute from a `matrix` by its key. Any function that returns a `matrix` can-also set attributes on the `matrix` with additional information. The `setAttribute` function can also be used-to set attributes on a `matrix`. The key to an attribute is always a string. The value of attribute can be any object-including numerics, arrays, maps, matrixes, etc.+The `getAttribute` function returns an attribute from a `matrix` by its key.+Any function that returns a `matrix` can also set attributes on the `matrix` with additional information.+The `setAttribute` function can also be used to set attributes on a `matrix`.+The key to an attribute is always a string.+The value of attribute can be any object including numerics, arrays, maps, matrixes, etc.=== getAttribute Parameters@@ -968,8 +993,8 @@ object: any object== getAttributes-The `getAttributes` function returns the attribute map from matrix. See the `getAttribute` function for more details-on attributes.+The `getAttributes` function returns the attribute map from matrix.+See the `getAttribute` function for more details on attributes.=== getAttributes Parameters@@ -986,8 +1011,9 @@ map: The map of attributes.== getColumnLabels-The `getColumnLabels` function returns the columns labels of a matrix. The column labels can be optionally-set by any function that returns a matrix. The column labels can also be set via the `setColumnLabels` function.+The `getColumnLabels` function returns the columns labels of a matrix.+The column labels can be optionally set by any function that returns a matrix.+The column labels can also be set via the `setColumnLabels` function.=== getColumnLabels Parameters@@ -1003,8 +1029,9 @@ string array: The labels for each column in the matrix== getRowLabels-The `getRowLabels` function returns the row labels of a matrix. The row labels can be optionally-set by any function that returns a matrix. The row labels can also be set via the `setRowLabels` function.+The `getRowLabels` function returns the row labels of a matrix.+The row labels can be optionally set by any function that returns a matrix.+The row labels can also be set via the `setRowLabels` function.=== getRowLabels Parameters@@ -1054,7 +1081,11 @@ number: the sum of all the values in the matrix.== gt-The `gt` function will return whether the first parameter is greater than the second parameter. The function accepts numeric or string parameters, but will fail to execute if all the parameters are not of the same type. That is, all are String or all are Numeric. If any any parameters are null then an error will be raised. Returns a boolean value.+The `gt` function will return whether the first parameter is greater than the second parameter.+The function accepts numeric or string parameters, but will fail to execute if all the parameters are not of the same type.+That is, all are String or all are Numeric.+If any any parameters are null then an error will be raised.+Returns a boolean value.=== gt Parameters@@ -1075,7 +1106,11 @@ gt(add(fieldA,fieldB),6) // fieldA + fieldB > 6== gteq-The `gteq` function will return whether the first parameter is greater than or equal to the second parameter. The function accepts numeric and string parameters, but will fail to execute if all the parameters are not of the same type. That is, all are String or all are Numeric. If any any parameters are null then an error will be raised. Returns a boolean value.+The `gteq` function will return whether the first parameter is greater than or equal to the second parameter.+The function accepts numeric and string parameters, but will fail to execute if all the parameters are not of the same type.+That is, all are String or all are Numeric.+If any any parameters are null then an error will be raised.+Returns a boolean value.=== gteq Parameters@@ -1096,15 +1131,15 @@ gteq(add(fieldA,fieldB),6) // fieldA + fieldB >= 6== hist-The `hist` function creates a histogram from a numeric array. The hist function is designed-to work with continuous variables.+The `hist` function creates a histogram from a numeric array.+The hist function is designed to work with continuous variables.=== hist Parameters//TODO fill in details of Parameters* `numeric array`-* `bins`: The number of bins in the histogram. Each returned tuple contains-summary statistics for the observations that were within the bin.+* `bins`: The number of bins in the histogram.+Each returned tuple contains summary statistics for the observations that were within the bin.=== hist Syntax@@ -1129,7 +1164,10 @@ if(gt(fieldA,fieldB),sin(fieldA),sin(fieldB)) // if fieldA > fieldB then return== if-The `if` function works like a standard conditional if/then statement. If the first parameter is true, then the second parameter will be returned, else the third parameter will be returned. The function accepts a boolean as the first parameter and anything as the second and third parameters. An error will occur if the first parameter is not a boolean or is null.+The `if` function works like a standard conditional if/then statement.+If the first parameter is true, then the second parameter will be returned, else the third parameter will be returned.+The function accepts a boolean as the first parameter and anything as the second and third parameters.+An error will occur if the first parameter is not a boolean or is null.=== if Parameters@@ -1204,13 +1242,13 @@ length(numericArray)== loessThe `leoss` function is a smoothing curve fitter which uses a https://en.wikipedia.org/wiki/Local_regression[local regression] algorithm.-Unlike the <<spline>> function which touches each control point, the `loess` function puts a smooth curve through-the control points without having to touch the control points. The `loess` result can be used by the <<derivative>> function to produce smooth derivatives from-data that is not smooth.+Unlike the <<spline>> function which touches each control point, the `loess` function puts a smooth curve through the control points without having to touch the control points.+The `loess` result can be used by the <<derivative>> function to produce smooth derivatives from data that is not smooth.=== loess Positional Parameters-* `numeric array`: (Optional) x values. If omitted a sequence will be created for the x values.+* `numeric array`: (Optional) x values.+If omitted a sequence will be created for the x values.* `numeric array`: y values=== loess Named Parameters@@ -1231,7 +1269,9 @@ function: The function can be treated as both a `numeric array` of the smoothed== log-The `log` function will return the natural log of the provided single parameter. The `log` function will fail to execute if the value is non-numeric. If a null value is found, then null will be returned as the result.+The `log` function will return the natural log of the provided single parameter.+The `log` function will fail to execute if the value is non-numeric.+If a null value is found, then null will be returned as the result.=== log Parameters@@ -1239,7 +1279,9 @@ The `log` function will return the natural log of the provided single parameter.=== log Syntax-The expressions below show the various ways in which you can use the `log` evaluator. Only one parameter is accepted. Returns a numeric value.+The expressions below show the various ways in which you can use the `log` evaluator.+Only one parameter is accepted.+Returns a numeric value.[source,text]----@@ -1250,8 +1292,8 @@ log(fieldA)== logNormalDistribution-The `logNormalDistribution` function returns a https://en.wikipedia.org/wiki/Log-normal_distribution[log normal probability distribution] based on its parameters. This function is part of the probability distribution framework and is designed to-work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+The `logNormalDistribution` function returns a https://en.wikipedia.org/wiki/Log-normal_distribution[log normal probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.=== logNormalDistribution Parameters@@ -1269,8 +1311,7 @@ logNormalDistribution(.3, .0)== kolmogorovSmirnov-The `kolmogorovSmirnov` function performs a https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test[Kolmogorov Smirnov test],-between a reference continuous probability distribution and a sample set.+The `kolmogorovSmirnov` function performs a https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test[Kolmogorov Smirnov test], between a reference continuous probability distribution and a sample set.The supported distribution functions are: `<<empiricalDistribution>>`, `<<normalDistribution>>`, `<<logNormalDistribution>>`, `<<weibullDistribution>>`, `<<gammaDistribution>>`, and `<<betaDistribution>>`.@@ -1290,7 +1331,11 @@ kolmogorovSmirnov(normalDistribution(10, 2), sampleSet)== lt-The `lt` function will return whether the first parameter is less than the second parameter. The function accepts numeric or string parameters, but will fail to execute if all the parameters are not of the same type. That is, all are String or all are Numeric. If any any parameters are null then an error will be raised. Returns a boolean value.+The `lt` function will return whether the first parameter is less than the second parameter.+The function accepts numeric or string parameters, but will fail to execute if all the parameters are not of the same type.+That is, all are String or all are Numeric.+If any any parameters are null then an error will be raised.+Returns a boolean value.=== lt Parameters@@ -1311,7 +1356,11 @@ lt(add(fieldA,fieldB),6) // fieldA + fieldB < 6== lteq-The `lteq` function will return whether the first parameter is less than or equal to the second parameter. The function accepts numeric and string parameters, but will fail to execute if all the parameters are not of the same type. That is, all are String or all are Numeric. If any any parameters are null then an error will be raised. Returns a boolean value.+The `lteq` function will return whether the first parameter is less than or equal to the second parameter.+The function accepts numeric and string parameters, but will fail to execute if all the parameters are not of the same type.+That is, all are String or all are Numeric.+If any any parameters are null then an error will be raised.+Returns a boolean value.=== lteq Parameters@@ -1333,9 +1382,8 @@ lteq(add(fieldA,fieldB),6) // fieldA + fieldB <= 6== markovChainThe `markovChain` function can be used to perform https://en.wikipedia.org/wiki/Markov_chain[Markov Chain] simulations.-The `markovChain` function takes as its parameter a https://en.wikipedia.org/wiki/Stochastic_matrix[transition matrix] and-returns a mathematical model that can be sampled using the <<sample>> function. Each sample taken-from the Markov Chain represents the current state of system.+The `markovChain` function takes as its parameter a https://en.wikipedia.org/wiki/Stochastic_matrix[transition matrix] and returns a mathematical model that can be sampled using the <<sample>> function.+Each sample taken from the Markov Chain represents the current state of system.=== markovChain Parameters@@ -1344,7 +1392,8 @@ from the Markov Chain represents the current state of system.=== markovChain Syntax[source,text]-sample(markovChain(transitionMatrix), 5) // This creates a Markov Chain given a specific transition matrix. The sample function takes 5 samples from the Markov Chain, representing the next five states of the system.+sample(markovChain(transitionMatrix), 5) // This creates a Markov Chain given a specific transition matrix.+The sample function takes 5 samples from the Markov Chain, representing the next five states of the system.=== markovChain Returns@@ -1390,16 +1439,17 @@ meanDifference(numericArray, numericArray)== minMaxScaleThe `minMaxScale` function scales numeric arrays within a minimum and maximum value.-By default `minMaxScale` scales between 0 and 1. The `minMaxScale` function can operate on-both numeric arrays and matrices.+By default `minMaxScale` scales between 0 and 1. The `minMaxScale` function can operate on both numeric arrays and matrices.When operating on a matrix the `minMaxScale` function operates on each row of the matrix.=== minMaxScale Parameters* `numeric array` | `matrix`: The array or matrix to scale-* `double`: (Optional) The min value. Defaults to 0.-* `double`: (Optional) The max value. Defaults to 1.+* `double`: (Optional) The min value.+Defaults to 0.+* `double`: (Optional) The max value.+Defaults to 1.=== minMaxScale Syntax@@ -1435,13 +1485,12 @@ if(gt(fieldA,fieldB),mod(fieldA,fieldB),mod(fieldB,fieldA)) // if fieldA > field== monteCarlo-The `monteCarlo` function performs a https://en.wikipedia.org/wiki/Monte_Carlo_method[Monte Carlo simulation]-based on its parameters. The `monteCarlo` function runs another function a specified number of times and returns the results.-The function being run typically has one or more variables that are drawn from probability-distributions on each run. The `<<sample>>` function is used in the function to draw the samples.+The `monteCarlo` function performs a https://en.wikipedia.org/wiki/Monte_Carlo_method[Monte Carlo simulation] based on its parameters.+The `monteCarlo` function runs another function a specified number of times and returns the results.+The function being run typically has one or more variables that are drawn from probability distributions on each run.+The `<<sample>>` function is used in the function to draw the samples.-The simulation's result array can then be treated as an empirical distribution to understand-the probabilities of the simulation results.+The simulation's result array can then be treated as an empirical distribution to understand the probabilities of the simulation results.=== monteCarlo Parameters@@ -1460,8 +1509,8 @@ let(a=uniformIntegerDistribution(1, 6),c=monteCarlo(add(sample(a), sample(b)), 1000))In the expression above the `monteCarlo` function is running the function `add(sample(a), sample(b))`-1000 times and returning the result. Each time the function is run samples are drawn from the-probability distributions stored in variables `a` and `b`.+1000 times and returning the result.+Each time the function is run samples are drawn from the probability distributions stored in variables `a` and `b`.== movingAvg@@ -1474,7 +1523,8 @@ The `movingAvg` function calculates a https://en.wikipedia.org/wiki/Moving_avera=== movingAvg Returns-A numeric array. The first element of the returned array will start from the windowSize-1 index of the original array.+A numeric array.+The first element of the returned array will start from the windowSize-1 index of the original array.=== movingAvg Syntax@@ -1492,7 +1542,8 @@ The `movingMedian` function calculates a moving median over an array of numbers.=== movingMedian Returns-A numeric array. The first element of the returned array will start from the windowSize-1 index of the original array.+A numeric array.+The first element of the returned array will start from the windowSize-1 index of the original array.=== movingMedian Syntax@@ -1501,7 +1552,9 @@ movingMedian(numericArray, 30)== mult-The `mult` function will take two or more numeric values and multiply them together. The `mult` function will fail to execute if any of the values are non-numeric. If a null value is found then null will be returned as the result.+The `mult` function will take two or more numeric values and multiply them together.+The `mult` function will fail to execute if any of the values are non-numeric.+If a null value is found then null will be returned as the result.=== mult Parameters@@ -1512,7 +1565,9 @@ The `mult` function will take two or more numeric values and multiply them toget=== mult Syntax-The expressions below show the various ways in which you can use the `mult` evaluator. The number and order of these parameters do not matter and is not limited except that at least two parameters are required. Returns a numeric value.+The expressions below show the various ways in which you can use the `mult` evaluator.+The number and order of these parameters do not matter and is not limited except that at least two parameters are required.+Returns a numeric value.[source,text]----@@ -1527,8 +1582,8 @@ mult(fieldA,if(gt(fieldA,fieldB),fieldA,fieldB)) // if fieldA > fieldB then fiel== normalDistributionThe `normalDistribution` function returns a https://en.wikipedia.org/wiki/Normal_distribution[normal probability distribution]-based on its parameters. This function is part of the probability distribution framework and is designed to-work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.=== normalDistribution Parameters@@ -1568,7 +1623,9 @@ numeric array | matrix== not-The `not` function will return the logical NOT of a single boolean parameter. The function will fail to execute if the parameter is non-boolean or null. Returns a boolean value.+The `not` function will return the logical NOT of a single boolean parameter.+The function will fail to execute if the parameter is non-boolean or null.+Returns a boolean value.=== not Parameters@@ -1576,7 +1633,8 @@ The `not` function will return the logical NOT of a single boolean parameter. Th=== not Syntax-The expressions below show the various ways in which you can use the `not` evaluator. Only one parameter is allowed.+The expressions below show the various ways in which you can use the `not` evaluator.+Only one parameter is allowed.[source,text]----@@ -1595,7 +1653,9 @@ The output of `olsRegress` can be used with the <<predict>> function to predict=== olsRegress Parameters-* `matrix`: The regressor observation matrix. Each row in the matrix represents a single multi-variate regressor observation. Note that there is no need to add an initial unitary column (column of 1's) when specifying a model including an intercept term, this column will be added automatically.+* `matrix`: The regressor observation matrix.+Each row in the matrix represents a single multi-variate regressor observation.+Note that there is no need to add an initial unitary column (column of 1's) when specifying a model including an intercept term, this column will be added automatically.* `numeric array`: The outcomes array which matches up with each row in the regressor observation matrix.=== olsRegress Syntax@@ -1609,7 +1669,9 @@ Tuple: The regression model including the estimated regression parameters and di== or-The `or` function will return the logical OR of at least 2 boolean parameters. The function will fail to execute if any parameters are non-boolean or null. Returns a boolean value.+The `or` function will return the logical OR of at least 2 boolean parameters.+The function will fail to execute if any parameters are non-boolean or null.+Returns a boolean value.=== or Parameters@@ -1620,7 +1682,8 @@ The `or` function will return the logical OR of at least 2 boolean parameters. T=== or Syntax-The expressions below show the various ways in which you can use the `or` evaluator. At least two parameters are required, but there is no limit to how many you can use.+The expressions below show the various ways in which you can use the `or` evaluator.+At least two parameters are required, but there is no limit to how many you can use.[source,text]----@@ -1633,8 +1696,8 @@ or(fieldA,fieldB,fieldC,and(fieldD,fieldE),fieldF)== poissonDistributionThe `poissonDistribution` function returns a https://en.wikipedia.org/wiki/Poisson_distribution[poisson probability distribution]-based on its parameter. This function is part of the probability distribution framework and is designed to-work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.+based on its parameter.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.=== poissonDistribution Parameters@@ -1655,9 +1718,11 @@ The `polyFit` function performs https://en.wikipedia.org/wiki/Curve_fitting#Fitt=== polyFit Parameters-* `numeric array`: (Optional) x values. If omitted a sequence will be created for the x values.+* `numeric array`: (Optional) x values.+If omitted a sequence will be created for the x values.* `numeric array`: y values-* `integer`: (Optional) polynomial degree. Defaults to 3.+* `integer`: (Optional) polynomial degree.+Defaults to 3.=== polyFit Returns@@ -1760,7 +1825,8 @@ The supported discreet distributions are:* `probability distribution`: the probability distribution to compute the probability from.* `number`: low value of the range.-* `number`: (Optional for discrete probability distributions) high value of the range. If the high range is omitted then the probability function will compute a probability for the low range value.+* `number`: (Optional for discrete probability distributions) high value of the range.+If the high range is omitted then the probability function will compute a probability for the low range value.=== probability Syntax@@ -1792,7 +1858,8 @@ rank(numericArray)== raw-The `raw` function will return whatever raw value is the parameter. This is useful for cases where you want to use a string as part of another evaluator.+The `raw` function will return whatever raw value is the parameter.+This is useful for cases where you want to use a string as part of another evaluator.=== raw Parameters@@ -1800,7 +1867,9 @@ The `raw` function will return whatever raw value is the parameter. This is usef=== raw Syntax-The expressions below show the various ways in which you can use the `raw` evaluator. Whatever is inside will be returned as-is. Internal evaluators are considered strings and are not evaluated.+The expressions below show the various ways in which you can use the `raw` evaluator.+Whatever is inside will be returned as-is.+Internal evaluators are considered strings and are not evaluated.[source,text]----@@ -1900,7 +1969,8 @@ The `sample` function can be used to draw random samples from a probability dist=== sample Parameters* `probability distribution` | `Markov Chain`: The distribution or Markov Chain to sample.-* `integer`: (Optional) Sample size. Defaults to 1.+* `integer`: (Optional) Sample size.+Defaults to 1.=== sample Returns@@ -1916,8 +1986,8 @@ sample(markovChain(transitionMatrix), 1000) // Returns 1000 random samples from== scalarAddThe `scalarAdd` function adds a scalar value to every value in a numeric array or matrix.-When working with numeric arrays, `scalarAdd` returns a new array with the new values. When working-with a matrix, `scalarAdd` returns a new matrix with new values.+When working with numeric arrays, `scalarAdd` returns a new array with the new values.+When working with a matrix, `scalarAdd` returns a new matrix with new values.=== scalarAdd Parameters@@ -1937,8 +2007,8 @@ numericArray | matrix: Depending on what is being operated on.== scalarDivideThe `scalarDivide` function divides each number in numeric array or matrix by a scalar value.-When working with numeric arrays, `scalarDivide` returns a new array with the new values. When working-with a matrix, `scalarDivide` returns a new matrix with new values.+When working with numeric arrays, `scalarDivide` returns a new array with the new values.+When working with a matrix, `scalarDivide` returns a new matrix with new values.=== scalarDivide Parameters@@ -1958,8 +2028,9 @@ numericArray | matrix: depending on what is being operated on.== scalarMultiplyThe `scalarMultiply` function multiplies each element in a numeric array or matrix by a-scalar value. When working with numeric arrays, `scalarMultiply` returns a new array with the new values. When working-with a matrix, `scalarMultiply` returns a new matrix with new values.+scalar value.+When working with numeric arrays, `scalarMultiply` returns a new array with the new values.+When working with a matrix, `scalarMultiply` returns a new matrix with new values.=== scalarMultiply Parameters@@ -1979,8 +2050,8 @@ numericArray | matrix: depending on what is being operated on== scalarSubtractThe `scalarSubtract` function subtracts a scalar value from every value in a numeric array or matrix.-When working with numeric arrays, `scalarSubtract` returns a new array with the new values. When working-with a matrix, `scalarSubtract` returns a new matrix with new values.+When working with numeric arrays, `scalarSubtract` returns a new array with the new values.+When working with a matrix, `scalarSubtract` returns a new matrix with new values.=== scalarSubtract Parameters@@ -2117,14 +2188,14 @@ if(gt(fieldA,fieldB),sin(fieldA),sin(fieldB)) // if fieldA > fieldB then return== spline-The `spline` function performs a cubic spline interpolation (https://en.wikiversity.org/wiki/Cubic_Spline_Interpolation) of a curve-given a set of x,y coordinates. The return value of the spline function is an-interpolation function which can be used to <<predict>> values along the curve and generate a <<derivative>> of+The `spline` function performs a cubic spline interpolation (https://en.wikiversity.org/wiki/Cubic_Spline_Interpolation) of a curve given a set of x,y coordinates.+The return value of the spline function is an interpolation function which can be used to <<predict>> values along the curve and generate a <<derivative>> ofthe curve.=== spline Parameters-* `numeric array`: (Optional) x values. If omitted a sequence will be created for the x values.+* `numeric array`: (Optional) x values.+If omitted a sequence will be created for the x values.* `numeric array`: y values=== spline Syntax@@ -2157,8 +2228,7 @@ if(gt(fieldA,fieldB),sqrt(fieldA),sqrt(fieldB)) // if fieldA > fieldB then retur== standardize-The `standardize` function standardizes a numeric array so that values within the array-have a mean of 0 and standard deviation of 1.+The `standardize` function standardizes a numeric array so that values within the array have a mean of 0 and standard deviation of 1.=== standardize Parameters@@ -2175,7 +2245,9 @@ numeric array: the standardized values== sub-The `sub` function will take 2 or more numeric values and subtract them, from left to right. The `sub` function will fail to execute if any of the values are non-numeric. If a null value is found then `null` will be returned as the result.+The `sub` function will take 2 or more numeric values and subtract them, from left to right.+The `sub` function will fail to execute if any of the values are non-numeric.+If a null value is found then `null` will be returned as the result.=== sub Parameters@@ -2186,7 +2258,9 @@ The `sub` function will take 2 or more numeric values and subtract them, from le=== sub Syntax-The expressions below show the various ways in which you can use the `sub` evaluator. The number of these parameters does not matter and is not limited except that at least two parameters are required. Returns a numeric value.+The expressions below show the various ways in which you can use the `sub` evaluator.+The number of these parameters does not matter and is not limited except that at least two parameters are required.+Returns a numeric value.[source,text]----@@ -2288,9 +2362,8 @@ matrix: the transposed matrix== triangularDistribution-The `triangularDistribution` function returns a https://en.wikipedia.org/wiki/Triangular_distribution[triangular probability distribution]-based on its parameters. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.+The `triangularDistribution` function returns a https://en.wikipedia.org/wiki/Triangular_distribution[triangular probability distribution] based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.=== triangularDistribution Parameters@@ -2310,8 +2383,9 @@ Probability distribution function== uniformDistributionThe `uniformDistribution` function returns a https://en.wikipedia.org/wiki/Uniform_distribution_(continuous)[continuous uniform probability distribution]-based on its parameters. See the `<<uniformIntegerDistribution>>` to work with discrete uniform distributions. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>` and `<<cumulativeProbability>>` functions.+based on its parameters.+See the `<<uniformIntegerDistribution>>` to work with discrete uniform distributions.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>` and `<<cumulativeProbability>>` functions.=== uniforDistribution Parameters@@ -2330,8 +2404,9 @@ uniformDistribution(0.0, 100.0)== uniformIntegerDistributionThe `uniformIntegerDistribution` function returns a https://en.wikipedia.org/wiki/Discrete_uniform_distribution[discrete uniform probability distribution]-based on its parameters. See the `<<uniformDistribution>>` to work with continuous uniform distributions. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.+based on its parameters.+See the `<<uniformDistribution>>` to work with continuous uniform distributions.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.=== uniformIntegerDistribution Parameters@@ -2371,8 +2446,8 @@ numeric array | matrix== weibullDistributionThe `weibullDistribution` function returns a https://en.wikipedia.org/wiki/Weibull_distribution[Weibull probability distribution]-based on its parameters. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.+based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<kolmogorovSmirnov>>` and `<<cumulativeProbability>>` functions.=== weibullDistribution Parameters@@ -2391,9 +2466,8 @@ weibullDistribution(.5, 10)== zipFDistributionThe `zipFDistribution` function returns a https://en.wikipedia.org/wiki/Zeta_distribution[ZipF distribution]-based on its parameters. This function is part of the-probability distribution framework and is designed to work with the `<<sample>>`,-`<<probability>>` and `<<cumulativeProbability>>` functions.+based on its parameters.+This function is part of the probability distribution framework and is designed to work with the `<<sample>>`, `<<probability>>` and `<<cumulativeProbability>>` functions.=== zipFDistribution Parametersdiff --git a/solr/solr-ref-guide/src/streaming-expressions.adoc b/solr/solr-ref-guide/src/streaming-expressions.adocindex df93f985dab..05d6b1a2335 100644--- a/solr/solr-ref-guide/src/streaming-expressions.adoc+++ b/solr/solr-ref-guide/src/streaming-expressions.adoc@@ -1,5 +1,11 @@= Streaming Expressions-:page-children: stream-source-reference, stream-decorator-reference, stream-evaluator-reference, math-expressions, graph-traversal, stream-api+:page-children: stream-source-reference, \+ stream-decorator-reference, \+ stream-evaluator-reference, \+ math-expressions, \+ graph-traversal, \+ stream-api, \+ stream-screen// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -39,13 +45,18 @@ They emit a stream of tuples (key/value Maps).Some of the provided streaming functions are designed to work with entire result sets rather than the top N results like normal search.This is supported by the <<exporting-result-sets.adoc#,/export handler>>.-Some streaming functions act as stream sources to originate the stream flow. Other streaming functions act as stream decorators to wrap other stream functions and perform operations on the stream of tuples. Many streams functions can be parallelized across a worker collection. This can be particularly powerful for relational algebra functions.+Some streaming functions act as stream sources to originate the stream flow.+Other streaming functions act as stream decorators to wrap other stream functions and perform operations on the stream of tuples.+Many streams functions can be parallelized across a worker collection.+This can be particularly powerful for relational algebra functions.=== Streaming Requests and Responses-Solr has a `/stream` request handler that takes streaming expression requests and returns the tuples as a JSON stream. This request handler is implicitly defined, meaning there is nothing that has to be defined in `solrconfig.xml` - see <<implicit-requesthandlers.adoc#,Implicit RequestHandlers>>.+Solr has a `/stream` request handler that takes streaming expression requests and returns the tuples as a JSON stream.+This request handler is implicitly defined, meaning there is nothing that has to be defined in `solrconfig.xml` - see <<implicit-requesthandlers.adoc#,Implicit Request Handlers>>.-The `/stream` request handler takes one parameter, `expr`, which is used to specify the streaming expression. For example, this curl command encodes and POSTs a simple `search()` expression to the `/stream` handler:+The `/stream` request handler takes one parameter, `expr`, which is used to specify the streaming expression.+For example, this curl command encodes and POSTs a simple `search()` expression to the `/stream` handler:[source,bash]----@@ -80,7 +91,9 @@ For the above example the `/stream` handler responded with the following JSON re}-----Note the last tuple in the above example stream is `{"EOF":true,"RESPONSE_TIME":33}`. The `EOF` indicates the end of the stream. To process the JSON response, you'll need to use a streaming JSON implementation because streaming expressions are designed to return the entire result set which may have millions of records. In your JSON client you'll need to iterate each doc (tuple) and check for the EOF tuple to determine the end of stream.+Note the last tuple in the above example stream is `{"EOF":true,"RESPONSE_TIME":33}`. The `EOF` indicates the end of the stream.+To process the JSON response, you'll need to use a streaming JSON implementation because streaming expressions are designed to return the entire result set which may have millions of records.+In your JSON client you'll need to iterate each doc (tuple) and check for the EOF tuple to determine the end of stream.=== Configuration@@ -90,7 +103,8 @@ Timeouts for Streaming Expressions can be configured with the `socketTimeout` an=== Stream Sources-Stream sources originate streams. There are rich set of searching, sampling and aggregation stream sources to choose from.+Stream sources originate streams.+There are rich set of searching, sampling and aggregation stream sources to choose from.A full reference to all available source expressions is available in <<stream-source-reference.adoc#,Stream Source Reference>>.@@ -124,3 +138,7 @@ Visualization of both streaming expressions and math expressions is done using AVisualizing Streaming expressions and setting up of Apache Zeppelin is documented in <<math-start.adoc#zeppelin-solr-interpreter,Zeppelin-Solr Interpreter>>.The <<math-expressions.adoc#,Math Expressions User Guide>> has in depth coverage of visualization techniques.++=== Stream Screen++* <<stream-screen.adoc#,Stream Screen>>: Submit streaming expressions and see results and parsing explanations.diff --git a/solr/solr-ref-guide/src/suggester.adoc b/solr/solr-ref-guide/src/suggester.adocindex 7926124c06b..689655b127a 100644--- a/solr/solr-ref-guide/src/suggester.adoc+++ b/solr/solr-ref-guide/src/suggester.adoc@@ -30,7 +30,7 @@ The main features of this Suggester are:* Term dictionary pluggability, giving you the flexibility to choose the dictionary implementation* Distributed support-The `solrconfig.xml` found in Solr's "```techproducts```" example has a Suggester implementation configured already. For more on search components, see the section <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,RequestHandlers and SearchComponents in SolrConfig>>.+The `solrconfig.xml` found in Solr's "```techproducts```" example has a Suggester implementation configured already. For more on search components, see the section <<requesthandlers-searchcomponents.adoc#,Request Handlers and Search Components>>.The "```techproducts```" example `solrconfig.xml` has a `suggest` search component and a `/suggest` request handler already configured. You can use that as the basis for your configuration, or create it from scratch, as detailed below.@@ -78,7 +78,7 @@ If not set, the default dictionary implementation is `HighFrequencyDictionaryFac`field`::A field from the index to use as the basis of suggestion terms. If `sourceLocation` is empty (meaning any dictionary implementation other than `FileDictionaryFactory`), then terms from this field in the index will be used.+-To be used as the basis for a suggestion, the field must be stored. You may want to <<copying-fields.adoc#,use copyField rules>> to create a special 'suggest' field comprised of terms from other fields in documents. In any event, you very likely want a minimal amount of analysis on the field, so an additional option is to create a field type in your schema that only uses basic tokenizers or filters. One option for such a field type is shown here:+To be used as the basis for a suggestion, the field must be stored. You may want to <<copy-fields.adoc#,use copyField rules>> to create a special 'suggest' field comprised of terms from other fields in documents. In any event, you very likely want a minimal amount of analysis on the field, so an additional option is to create a field type in your schema that only uses basic tokenizers or filters. One option for such a field type is shown here:+[source,xml]----@@ -349,7 +349,7 @@ When using these Suggesters in a query, you would define multiple `suggest.dicti== Adding the Suggest Request Handler-After adding the search component, a request handler must be added to `solrconfig.xml`. This request handler works the <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,same as any other request handler>>, and allows you to configure default parameters for serving suggestion requests. The request handler definition must incorporate the "suggest" search component defined previously.+After adding the search component, a request handler must be added to `solrconfig.xml`. This request handler works the <<requesthandlers-searchcomponents.adoc#,same as any other request handler>>, and allows you to configure default parameters for serving suggestion requests. The request handler definition must incorporate the "suggest" search component defined previously.[source,xml]----diff --git a/solr/solr-ref-guide/src/solr-system-requirements.adoc b/solr/solr-ref-guide/src/system-requirements.adocsimilarity index 73%rename from solr/solr-ref-guide/src/solr-system-requirements.adocrename to solr/solr-ref-guide/src/system-requirements.adocindex fd0da17c58f..00b524ec41a 100644--- a/solr/solr-ref-guide/src/solr-system-requirements.adoc+++ b/solr/solr-ref-guide/src/system-requirements.adoc@@ -1,4 +1,4 @@-= Solr System Requirements+= System Requirements// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -27,7 +27,8 @@ Solr is tested on several versions of Linux, macOS and Windows.//TODO: this with the next big section on Java fragments the Java requirements too much. Consider merging sections.=== Java Requirements-You will need the Java Runtime Environment (JRE) version 1.8 or higher. At a command line, check your Java version like this:+You will need the Java Runtime Environment (JRE) version 1.8 or higher.+At a command line, check your Java version like this:[source,bash]----@@ -37,13 +38,18 @@ Java(TM) SE Runtime Environment (build 1.8.0_60-b27)Java HotSpot(TM) 64-Bit Server VM (build 25.60-b23, mixed mode)-----The exact output will vary, but you need to make sure you meet the minimum version requirement. We also recommend choosing a version that is not end-of-life from its vendor. Oracle/OpenJDK are the most tested JREs and are preferred. It's also preferred to use the latest available official release.+The exact output will vary, but you need to make sure you meet the minimum version requirement.+We also recommend choosing a version that is not end-of-life from its vendor. Oracle/OpenJDK are the most tested JREs and are preferred.+It's also preferred to use the latest available official release.-Some versions of Java VM have bugs that may impact your implementation. To be sure, check the page https://cwiki.apache.org/confluence/display/LUCENE/JavaBugs[Lucene Java Bugs].+Some versions of Java VM have bugs that may impact your implementation.+To be sure, check the page https://cwiki.apache.org/confluence/display/LUCENE/JavaBugs[Lucene Java Bugs].=== Sources for Java-Java is available from a number of providers. Lucene and Solr regularly test with https://jdk.java.net/[OpenJDK] and Oracle versions of Java. Some are free, others have a cost, some provide security patches and support, others do not. We recommend you read the article https://medium.com/@javachampions/java-is-still-free-2-0-0-6b9aa8d6d244[Java is still free by Java Champions] to help you decide.+Java is available from a number of providers.+Lucene and Solr regularly test with https://jdk.java.net/[OpenJDK] and Oracle versions of Java.+Some are free, others have a cost, some provide security patches and support, others do not. We recommend you read the article https://medium.com/@javachampions/java-is-still-free-2-0-0-6b9aa8d6d244[Java is still free by Java Champions] to help you decide.The Lucene project does not endorse any particular provider of Java.@@ -51,25 +57,31 @@ NOTE: While we reference the Java Development (JDK) on this page, any Java Runti== Java and Lucene/Solr Combinations-Each Lucene/Solr release has an extensively tested minimum Java version. For instance the minimum Java version for Solr 8 is Java 8. This section provides guidance when running Lucene/Solr with a more recent Java version than the minimum specified.+Each Lucene/Solr release has an extensively tested minimum Java version.+For instance the minimum Java version for Solr 8 is Java 8.+This section provides guidance when running Lucene/Solr with a more recent Java version than the minimum specified.* OpenJDK and Oracle Java distributions are tested extensively and will continue to be tested going forward.** Distributions of Java from other sources are not regularly tested by our testing infrastructure, therefore you must test Java from those sources in your environment.** For the purposes of Lucene and Solr, Oracle's Java and OpenJDK are identical.* Upgrading Java is not required with the understanding that no Java bugs will be addressed unless you are using a version of Java that provides LTS.-* Java 8 has been extensively tested by both automated tests and users through Solr 8. Long Term Support (LTS) for Java 8 is provided by some sources, see https://medium.com/@javachampions/java-is-still-free-2-0-0-6b9aa8d6d244[Java is still free].+* Java 8 has been extensively tested by both automated tests and users through Solr 8.+Long Term Support (LTS) for Java 8 is provided by some sources, see https://medium.com/@javachampions/java-is-still-free-2-0-0-6b9aa8d6d244[Java is still free].* The project's testing infrastructure continuously tests with the minimum and greater versions of Java for each development branch.-* Java 9 and 10 have no LTS. For this reason, Java 11 is preferred over 9 or 10 when upgrading Java.+* Java 9 and 10 have no LTS.+For this reason, Java 11 is preferred over 9 or 10 when upgrading Java.* For specific questions the http://solr.apache.org/community.html#mailing-lists-chat[Solr User's List] is a great resource.=== Project Testing of Java-Solr Combinations-Solr and Lucene run a continuous integration model, running automated unit and integration tests using several versions of Java. In addition, some organizations also maintain their own test infrastructure and feed their results back to the community.+Solr and Lucene run a continuous integration model, running automated unit and integration tests using several versions of Java.+In addition, some organizations also maintain their own test infrastructure and feed their results back to the community.Our continuous testing is against the two code lines under active development, Solr 8x and the future Solr 9.0:* Lucene/Solr 8.x is the current stable release line and will have "point releases", i.e., 8.1, 8.2, etc. until Lucene/Solr 9.0 is released.** Solr 8.x is currently tested against Java 8, 9, 10, 11, 12 and (pre-release) 13.-* There is also development and testing with the future Lucene/Solr 9.x release line, which will require Java 11 as a minimum version. This line is currently tested against Java 11, 12 and (pre-release) 13.+* There is also development and testing with the future Lucene/Solr 9.x release line, which will require Java 11 as a minimum version.+This line is currently tested against Java 11, 12 and (pre-release) 13.* Lucene/Solr 7.x and earlier release lines are not tested on a continuous basis.=== Released Lucene/Solr and Java Versions@@ -82,7 +94,8 @@ The success rate in our automated tests is similar with all the Java versions te==== Lucene/Solr 7.x* Requires Java 8 or higher.-* This version had continuous testing with Java 9, 10, 11, 12 and the pre-release version of Java 13. Regular testing stopped when Lucene/Solr 8.0 was released.+* This version had continuous testing with Java 9, 10, 11, 12 and the pre-release version of Java 13.+Regular testing stopped when Lucene/Solr 8.0 was released.* Hadoop with Java 9+ may not work in all situations, test in your environment.* Kerberos with Java 9+ may not work in all situations, test in your environment.* Be sure to test with SSL/TLS and/or authorization enabled in your environment if you require either when using Java 9+.@@ -91,5 +104,6 @@ The success rate in our automated tests is similar with all the Java versions te* Requires Java 8 or higher.* This version has continuous testing with Java 9, 10, 11, 12 and the pre-release version of Java 13.-* There were known issues with Kerberos with Java 9+ prior to Solr 8.1. If using 8.0, you should test in your environment.+* There were known issues with Kerberos with Java 9+ prior to Solr 8.1.+If using 8.0, you should test in your environment.* Be sure to test with SSL/TLS and/or authorization enabled in your environment if you require either when using Java 9+.diff --git a/solr/solr-ref-guide/src/the-tagger-handler.adoc b/solr/solr-ref-guide/src/tagger-handler.adocsimilarity index 98%rename from solr/solr-ref-guide/src/the-tagger-handler.adocrename to solr/solr-ref-guide/src/tagger-handler.adocindex 4fdd78861ca..3606c3baf94 100644--- a/solr/solr-ref-guide/src/the-tagger-handler.adoc+++ b/solr/solr-ref-guide/src/tagger-handler.adoc@@ -1,4 +1,4 @@-= The Tagger Handler+= Tagger Handler// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -39,7 +39,7 @@ be supported; the maximum is mostly limited only by memory.To configure the tagger, your Solr schema needs 2 fields:-* A unique key field (see <<other-schema-elements.adoc#unique-key,Unique Key>> for how to define a unique key in your schema).+* A unique key field (see <<schema-elements.adoc#unique-key,Unique Key>> for how to define a unique key in your schema).Recommended field settings: set `docValues=true`.* A tag field, which must be a `TextField`, with `ConcatenateGraphFilterFactory` at the end of the index chain (not the query chain):Set `preservePositionIncrements=false` on that filter.diff --git a/solr/solr-ref-guide/src/taking-solr-to-production.adoc b/solr/solr-ref-guide/src/taking-solr-to-production.adocindex de5e5101cbd..db5ad47c004 100644--- a/solr/solr-ref-guide/src/taking-solr-to-production.adoc+++ b/solr/solr-ref-guide/src/taking-solr-to-production.adoc@@ -16,11 +16,15 @@// specific language governing permissions and limitations// under the License.-This section provides guidance on how to setup Solr to run in production on *nix platforms, such as Ubuntu. Specifically, we’ll walk through the process of setting up to run a single Solr instance on a Linux host and then provide tips on how to support multiple Solr nodes running on the same host.+This section provides guidance on how to setup Solr to run in production on *nix platforms, such as Ubuntu.+Specifically, we’ll walk through the process of setting up to run a single Solr instance on a Linux host and then provide tips on how to support multiple Solr nodes running on the same host.== Service Installation Script-Solr includes a service installation script (`bin/install_solr_service.sh`) to help you install Solr as a service on Linux. Currently, the script only supports CentOS, Debian, Red Hat, SUSE and Ubuntu Linux distributions. Before running the script, you need to determine a few parameters about your setup. Specifically, you need to decide where to install Solr and which system user should be the owner of the Solr files and process.+Solr includes a service installation script (`bin/install_solr_service.sh`) to help you install Solr as a service on Linux.+Currently, the script only supports CentOS, Debian, Red Hat, SUSE and Ubuntu Linux distributions.+Before running the script, you need to determine a few parameters about your setup.+Specifically, you need to decide where to install Solr and which system user should be the owner of the Solr files and process.=== Planning Your Directory Structure@@ -28,7 +32,10 @@ We recommend separating your live Solr files, such as logs and index files, from==== Solr Installation Directory-By default, the service installation script will extract the distribution archive into `/opt`. You can change this location using the `-i` option when running the installation script. The script will also create a symbolic link to the versioned directory of Solr. For instance, if you run the installation script for Solr {solr-docs-version}.0, then the following directory structure will be used:+By default, the service installation script will extract the distribution archive into `/opt`.+You can change this location using the `-i` option when running the installation script.+The script will also create a symbolic link to the versioned directory of Solr.+For instance, if you run the installation script for Solr {solr-docs-version}.0, then the following directory structure will be used:[source,plain,subs="attributes"]----@@ -36,15 +43,22 @@ By default, the service installation script will extract the distribution archiv/opt/solr -> /opt/solr-{solr-docs-version}.0-----Using a symbolic link insulates any scripts from being dependent on the specific Solr version. If, down the road, you need to upgrade to a later version of Solr, you can just update the symbolic link to point to the upgraded version of Solr. We’ll use `/opt/solr` to refer to the Solr installation directory in the remaining sections of this page.+Using a symbolic link insulates any scripts from being dependent on the specific Solr version.+If, down the road, you need to upgrade to a later version of Solr, you can just update the symbolic link to point to the upgraded version of Solr.+We’ll use `/opt/solr` to refer to the Solr installation directory in the remaining sections of this page.==== Separate Directory for Writable Files-You should also separate writable Solr files into a different directory; by default, the installation script uses `/var/solr`, but you can override this location using the `-d` option. With this approach, the files in `/opt/solr` will remain untouched and all files that change while Solr is running will live under `/var/solr`.+You should also separate writable Solr files into a different directory; by default, the installation script uses `/var/solr`, but you can override this location using the `-d` option.+With this approach, the files in `/opt/solr` will remain untouched and all files that change while Solr is running will live under `/var/solr`.=== Create the Solr User-Running Solr as `root` is not recommended for security reasons, and the <<solr-control-script-reference.adoc#,control script>> start command will refuse to do so. Consequently, you should determine the username of a system user that will own all of the Solr files and the running Solr process. By default, the installation script will create the *solr* user, but you can override this setting using the -u option. If your organization has specific requirements for creating new user accounts, then you should create the user before running the script. The installation script will make the Solr user the owner of the `/opt/solr` and `/var/solr` directories.+Running Solr as `root` is not recommended for security reasons, and the <<solr-control-script-reference.adoc#,control script>> start command will refuse to do so.+Consequently, you should determine the username of a system user that will own all of the Solr files and the running Solr process.+By default, the installation script will create the *solr* user, but you can override this setting using the -u option.+If your organization has specific requirements for creating new user accounts, then you should create the user before running the script.+The installation script will make the Solr user the owner of the `/opt/solr` and `/var/solr` directories.You are now ready to run the installation script.@@ -57,21 +71,24 @@ To run the script, you'll need to download the latest Solr distribution archivetar xzf solr-{solr-docs-version}.0.tgz solr-{solr-docs-version}.0/bin/install_solr_service.sh --strip-components=2-----The previous command extracts the `install_solr_service.sh` script from the archive into the current directory. If installing on Red Hat, please make sure *lsof* is installed before running the Solr installation script (`sudo yum install lsof`). The installation script must be run as root:+The previous command extracts the `install_solr_service.sh` script from the archive into the current directory.+If installing on Red Hat, please make sure *lsof* is installed before running the Solr installation script (`sudo yum install lsof`). The installation script must be run as root:[source,bash,subs="attributes"]----sudo bash ./install_solr_service.sh solr-{solr-docs-version}.0.tgz-----By default, the script extracts the distribution archive into `/opt`, configures Solr to write files into `/var/solr`, and runs Solr as the `solr` user. Consequently, the following command produces the same result as the previous command:+By default, the script extracts the distribution archive into `/opt`, configures Solr to write files into `/var/solr`, and runs Solr as the `solr` user.+Consequently, the following command produces the same result as the previous command:[source,bash,subs="attributes"]----sudo bash ./install_solr_service.sh solr-{solr-docs-version}.0.tgz -i /opt -d /var/solr -u solr -s solr -p 8983-----You can customize the service name, installation directories, port, and owner using options passed to the installation script. To see available options, simply do:+You can customize the service name, installation directories, port, and owner using options passed to the installation script.+To see available options, simply do:[source,bash]----@@ -85,17 +102,31 @@ Once the script completes, Solr will be installed as a service and running in thsudo service solr status-----If you do not want to start the service immediately, pass the `-n` option. You can then start the service manually later, e.g., after completing the configuration setup.+If you do not want to start the service immediately, pass the `-n` option.+You can then start the service manually later, e.g., after completing the configuration setup.-We'll cover some additional configuration settings you can make to fine-tune your Solr setup in a moment. Before moving on, let's take a closer look at the steps performed by the installation script. This gives you a better overview and will help you understand important details about your Solr installation when reading other pages in this guide; such as when a page refers to Solr home, you'll know exactly where that is on your system.+We'll cover some additional configuration settings you can make to fine-tune your Solr setup in a moment.+Before moving on, let's take a closer look at the steps performed by the installation script.+This gives you a better overview and will help you understand important details about your Solr installation when reading other pages in this guide; such as when a page refers to Solr home, you'll know exactly where that is on your system.==== Solr Home Directory-The Solr home directory (not to be confused with the Solr installation directory) is where Solr manages core directories with index files. By default, the installation script uses `/var/solr/data`. If the `-d` option is used on the install script, then this will change to the `data` subdirectory in the location given to the -d option. Take a moment to inspect the contents of the Solr home directory on your system. If you do not <<using-zookeeper-to-manage-configuration-files.adoc#,store `solr.xml` in ZooKeeper>>, the home directory must contain a `solr.xml` file. When Solr starts up, the Solr Control Script passes the location of the home directory using the `-Dsolr.solr.home=...` system property.+The Solr home directory (not to be confused with the Solr installation directory) is where Solr manages core directories with index files.+By default, the installation script uses `/var/solr/data`.+If the `-d` option is used on the install script, then this will change to the `data` subdirectory in the location given to the -d option.+Take a moment to inspect the contents of the Solr home directory on your system.+If you do not <<zookeeper-file-management.adoc#,store `solr.xml` in ZooKeeper>>, the home directory must contain a `solr.xml` file.+When Solr starts up, the Solr Control Script passes the location of the home directory using the `-Dsolr.solr.home=...` system property.==== Environment Overrides Include File-The service installation script creates an environment specific include file that overrides defaults used by the `bin/solr` script. The main advantage of using an include file is that it provides a single location where all of your environment-specific overrides are defined. Take a moment to inspect the contents of the `/etc/default/solr.in.sh` file, which is the default path setup by the installation script. If you used the `-s` option on the install script to change the name of the service, then the first part of the filename will be different. For a service named `solr-demo`, the file will be named `/etc/default/solr-demo.in.sh`. There are many settings that you can override using this file. However, at a minimum, this script needs to define the `SOLR_PID_DIR` and `SOLR_HOME` variables, such as:+The service installation script creates an environment specific include file that overrides defaults used by the `bin/solr` script.+The main advantage of using an include file is that it provides a single location where all of your environment-specific overrides are defined.+Take a moment to inspect the contents of the `/etc/default/solr.in.sh` file, which is the default path setup by the installation script.+If you used the `-s` option on the install script to change the name of the service, then the first part of the filename will be different.+For a service named `solr-demo`, the file will be named `/etc/default/solr-demo.in.sh`.+There are many settings that you can override using this file.+However, at a minimum, this script needs to define the `SOLR_PID_DIR` and `SOLR_HOME` variables, such as:[source,bash]----@@ -107,7 +138,9 @@ The `SOLR_PID_DIR` variable sets the directory where the <<solr-control-script-r==== Log Settings-Solr uses Apache Log4J for logging. The installation script copies `/opt/solr/server/resources/log4j2.xml` to `/var/solr/log4j2.xml`. Take a moment to verify that the Solr include file is configured to send logs to the correct location by checking the following settings in `/etc/default/solr.in.sh`:+Solr uses Apache Log4J for logging.+The installation script copies `/opt/solr/server/resources/log4j2.xml` to `/var/solr/log4j2.xml`.+Take a moment to verify that the Solr include file is configured to send logs to the correct location by checking the following settings in `/etc/default/solr.in.sh`:[source,bash]----@@ -119,7 +152,11 @@ For more information about Log4J configuration, please see: <<configuring-loggin==== init.d Script-When running a service like Solr on Linux, it’s common to setup an init.d script so that system administrators can control Solr using the service tool, such as: `service solr start`. The installation script creates a very basic init.d script to help you get started. Take a moment to inspect the `/etc/init.d/solr` file, which is the default script name setup by the installation script. If you used the `-s` option on the install script to change the name of the service, then the filename will be different. Notice that the following variables are setup for your environment based on the parameters passed to the installation script:+When running a service like Solr on Linux, it’s common to setup an init.d script so that system administrators can control Solr using the service tool, such as: `service solr start`.+The installation script creates a very basic init.d script to help you get started.+Take a moment to inspect the `/etc/init.d/solr` file, which is the default script name setup by the installation script.+If you used the `-s` option on the install script to change the name of the service, then the filename will be different.+Notice that the following variables are setup for your environment based on the parameters passed to the installation script:[source,bash]----@@ -128,18 +165,27 @@ SOLR_ENV=/etc/default/solr.in.shRUNAS=solr-----The `SOLR_INSTALL_DIR` and `SOLR_ENV` variables should be self-explanatory. The `RUNAS` variable sets the owner of the Solr process, such as `solr`; if you don’t set this value, the script will run Solr as **root**, which is not recommended for production. You can use the `/etc/init.d/solr` script to start Solr by doing the following as root:+The `SOLR_INSTALL_DIR` and `SOLR_ENV` variables should be self-explanatory.+The `RUNAS` variable sets the owner of the Solr process, such as `solr`; if you don’t set this value, the script will run Solr as **root**, which is not recommended for production.+You can use the `/etc/init.d/solr` script to start Solr by doing the following as root:[source,bash]----service solr start-----The `/etc/init.d/solr` script also supports the **stop**, **restart**, and *status* commands. Please keep in mind that the init script that ships with Solr is very basic and is intended to show you how to setup Solr as a service. However, it’s also common to use more advanced tools like *supervisord* or *upstart* to control Solr as a service on Linux. While showing how to integrate Solr with tools like supervisord is beyond the scope of this guide, the `init.d/solr` script should provide enough guidance to help you get started. Also, the installation script sets the Solr service to start automatically when the host machine initializes.+The `/etc/init.d/solr` script also supports the **stop**, **restart**, and *status* commands.+Please keep in mind that the init script that ships with Solr is very basic and is intended to show you how to setup Solr as a service.+However, it’s also common to use more advanced tools like *supervisord* or *upstart* to control Solr as a service on Linux.+While showing how to integrate Solr with tools like supervisord is beyond the scope of this guide, the `init.d/solr` script should provide enough guidance to help you get started.+Also, the installation script sets the Solr service to start automatically when the host machine initializes.=== Progress Check-In the next section, we cover some additional environment settings to help you fine-tune your production setup. However, before we move on, let's review what we've achieved thus far. Specifically, you should be able to control Solr using `/etc/init.d/solr`. Please verify the following commands work with your setup:+In the next section, we cover some additional environment settings to help you fine-tune your production setup.+However, before we move on, let's review what we've achieved thus far.+Specifically, you should be able to control Solr using `/etc/init.d/solr`.+Please verify the following commands work with your setup:[source,bash]----@@ -165,15 +211,19 @@ If the `status` command is not successful, look for error messages in `/var/solr=== Dynamic Defaults for ConcurrentMergeScheduler-The Merge Scheduler is configured in `solrconfig.xml` and defaults to `ConcurrentMergeScheduler`. This scheduler uses multiple threads to merge Lucene segments in the background.+The Merge Scheduler is configured in `solrconfig.xml` and defaults to `ConcurrentMergeScheduler`.+This scheduler uses multiple threads to merge Lucene segments in the background.-By default, the `ConcurrentMergeScheduler` auto-detects defaults for `maxThreadCount` and `maxMergeCount` accordingly. `maxThreadCount` is set to 4 or half the number of processors available to the JVM whichever is greater and `maxMergeCount` is set to `maxThreadCount+5`.+By default, the `ConcurrentMergeScheduler` auto-detects defaults for `maxThreadCount` and `maxMergeCount` accordingly.+`maxThreadCount` is set to 4 or half the number of processors available to the JVM whichever is greater and `maxMergeCount` is set to `maxThreadCount+5`.-If you have a spinning disk, it is best to explicitly set values for `maxThreadCount` and `maxMergeCount` in the <<indexconfig-in-solrconfig.adoc#mergescheduler, IndexConfig section of SolrConfig.xml>> so that values appropriate to your hardware are used.+If you have a spinning disk, it is best to explicitly set values for `maxThreadCount` and `maxMergeCount` in the <<index-segments-merging.adoc#mergescheduler, IndexConfig section of SolrConfig.xml>> so that values appropriate to your hardware are used.=== Memory and GC Settings-By default, the `bin/solr` script sets the maximum Java heap size to 512M (-Xmx512m), which is fine for getting started with Solr. For production, you’ll want to increase the maximum heap size based on the memory requirements of your search application; values between 8 and 16 gigabytes are not uncommon for production servers. When you need to change the memory settings for your Solr server, use the `SOLR_HEAP` variable in the include file, such as:+By default, the `bin/solr` script sets the maximum Java heap size to 512M (-Xmx512m), which is fine for getting started with Solr.+For production, you’ll want to increase the maximum heap size based on the memory requirements of your search application; values between 8 and 16 gigabytes are not uncommon for production servers.+When you need to change the memory settings for your Solr server, use the `SOLR_HEAP` variable in the include file, such as:[source,bash]----@@ -182,22 +232,30 @@ SOLR_HEAP="8g"[NOTE]====-Do not allocate a very large Java Heap unless you know you need it. See <<jvm-settings.adoc#choosing-memory-heap-settings,Choosing Memory Heap Settings>> for details.+Do not allocate a very large Java Heap unless you know you need it.+See <<jvm-settings.adoc#choosing-memory-heap-settings,Choosing Memory Heap Settings>> for details.====Also, the <<solr-control-script-reference.adoc#,Solr Control Script>> comes with a set of pre-configured Garbage First Garbage Collection settings that have shown to work well with Solr for a number of different workloads.-However, these settings may not work well for your specific use of Solr. Consequently, you may need to change the GC settings, which should also be done with the `GC_TUNE` variable in the `/etc/default/solr.in.sh` include file. For more information about garbage collection settings refer to following articles:+However, these settings may not work well for your specific use of Solr.+Consequently, you may need to change the GC settings, which should also be done with the `GC_TUNE` variable in the `/etc/default/solr.in.sh` include file.+For more information about garbage collection settings refer to following articles:1. https://cwiki.apache.org/confluence/display/solr/ShawnHeisey2. https://www.oracle.com/technetwork/articles/java/g1gc-1984535.htmlYou can also refer to <<jvm-settings.adoc#,JVM Settings>> for tuning your memory and garbage collection settings.==== Out-of-Memory Shutdown Hook-The `bin/solr` script registers the `bin/oom_solr.sh` script to be called by the JVM if an OutOfMemoryError occurs. The `oom_solr.sh` script will issue a `kill -9` to the Solr process that experiences the `OutOfMemoryError`. This behavior is recommended when running in SolrCloud mode so that ZooKeeper is immediately notified that a node has experienced a non-recoverable error. Take a moment to inspect the contents of the `/opt/solr/bin/oom_solr.sh` script so that you are familiar with the actions the script will perform if it is invoked by the JVM.+The `bin/solr` script registers the `bin/oom_solr.sh` script to be called by the JVM if an OutOfMemoryError occurs.+The `oom_solr.sh` script will issue a `kill -9` to the Solr process that experiences the `OutOfMemoryError`.+This behavior is recommended when running in SolrCloud mode so that ZooKeeper is immediately notified that a node has experienced a non-recoverable error.+Take a moment to inspect the contents of the `/opt/solr/bin/oom_solr.sh` script so that you are familiar with the actions the script will perform if it is invoked by the JVM.=== Going to Production with SolrCloud-To run Solr in SolrCloud mode, you need to set the `ZK_HOST` variable in the include file to point to your ZooKeeper ensemble. Running the embedded ZooKeeper is not supported in production environments. For instance, if you have a ZooKeeper ensemble hosted on the following three hosts on the default client port 2181 (zk1, zk2, and zk3), then you would set:+To run Solr in SolrCloud mode, you need to set the `ZK_HOST` variable in the include file to point to your ZooKeeper ensemble.+Running the embedded ZooKeeper is not supported in production environments.+For instance, if you have a ZooKeeper ensemble hosted on the following three hosts on the default client port 2181 (zk1, zk2, and zk3), then you would set:[source,bash]----@@ -208,7 +266,8 @@ When the `ZK_HOST` variable is set, Solr will launch in "cloud" mode.==== ZooKeeper chroot-If you're using a ZooKeeper instance that is shared by other systems, it's recommended to isolate the SolrCloud znode tree using ZooKeeper's chroot support. For instance, to ensure all znodes created by SolrCloud are stored under `/solr`, you can put `/solr` on the end of your `ZK_HOST` connection string, such as:+If you're using a ZooKeeper instance that is shared by other systems, it's recommended to isolate the SolrCloud znode tree using ZooKeeper's chroot support.+For instance, to ensure all znodes created by SolrCloud are stored under `/solr`, you can put `/solr` on the end of your `ZK_HOST` connection string, such as:[source,bash]----@@ -224,7 +283,8 @@ bin/solr zk mkroot /solr -z <ZK_node>:<ZK_PORT>[NOTE]====-If you also want to bootstrap ZooKeeper with existing `solr_home`, you can instead use the `zkcli.sh` / `zkcli.bat` `bootstrap` command, which will also create the chroot path if it does not exist. See <<command-line-utilities.adoc#,Command Line Utilities>> for more info.+If you also want to bootstrap ZooKeeper with existing `solr_home`, you can instead use the `zkcli.sh` / `zkcli.bat` `bootstrap` command, which will also create the chroot path if it does not exist.+See <<zookeeper-utilities.adoc#,ZooKeeper Utilities>> for more info.======= Solr Hostname@@ -240,7 +300,12 @@ Setting the hostname of the Solr server is recommended, especially when running=== Environment Banner in Admin UI-To guard against accidentally doing changes to the wrong cluster, you may configure a visual indication in the Admin UI of whether you currently work with a production environment or not. To do this, edit your `solr.in.sh` or `solr.in.cmd` file with a `-Dsolr.environment=prod` setting, or set the cluster property named `environment`. To specify label and/or color, use a comma-delimited format as below. The `+` character can be used instead of space to avoid quoting. Colors may be valid CSS colors or numeric, e.g., `#ff0000` for bright red. Examples of valid environment configs:+To guard against accidentally doing changes to the wrong cluster, you may configure a visual indication in the Admin UI of whether you currently work with a production environment or not.+To do this, edit your `solr.in.sh` or `solr.in.cmd` file with a `-Dsolr.environment=prod` setting, or set the cluster property named `environment`.+To specify label and/or color, use a comma-delimited format as below.+The `+` character can be used instead of space to avoid quoting.+Colors may be valid CSS colors or numeric, e.g., `#ff0000` for bright red.+Examples of valid environment configs:* `prod`* `test,label=Functional+test`@@ -249,7 +314,8 @@ To guard against accidentally doing changes to the wrong cluster, you may config=== Override Settings in solrconfig.xml-Solr allows configuration properties to be overridden using Java system properties passed at startup using the `-Dproperty=value` syntax. For instance, in `solrconfig.xml`, the default auto soft commit settings are set to:+Solr allows configuration properties to be overridden using Java system properties passed at startup using the `-Dproperty=value` syntax.+For instance, in `solrconfig.xml`, the default auto soft commit settings are set to:[source,xml]----@@ -258,14 +324,17 @@ Solr allows configuration properties to be overridden using Java system properti</autoSoftCommit>-----In general, whenever you see a property in a Solr configuration file that uses the `${solr.PROPERTY:DEFAULT_VALUE}` syntax, then you know it can be overridden using a Java system property. For instance, to set the maxTime for soft-commits to be 10 seconds, then you can start Solr with `-Dsolr.autoSoftCommit.maxTime=10000`, such as:+In general, whenever you see a property in a Solr configuration file that uses the `${solr.PROPERTY:DEFAULT_VALUE}` syntax, then you know it can be overridden using a Java system property.+For instance, to set the maxTime for soft-commits to be 10 seconds, then you can start Solr with `-Dsolr.autoSoftCommit.maxTime=10000`, such as:[source,bash]----bin/solr start -Dsolr.autoSoftCommit.maxTime=10000-----The `bin/solr` script simply passes options starting with `-D` on to the JVM during startup. For running in production, we recommend setting these properties in the `SOLR_OPTS` variable defined in the include file. Keeping with our soft-commit example, in `/etc/default/solr.in.sh`, you would do:+The `bin/solr` script simply passes options starting with `-D` on to the JVM during startup.+For running in production, we recommend setting these properties in the `SOLR_OPTS` variable defined in the include file.+Keeping with our soft-commit example, in `/etc/default/solr.in.sh`, you would do:[source,bash]----@@ -274,7 +343,8 @@ SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=10000"=== Ulimit Settings (*nix Operating Systems)-There are several settings that should be monitored and set as high as possible, "unlimited" by preference. On most "*nix" operating systems, you can see the current values by typing the following at a command prompt.+There are several settings that should be monitored and set as high as possible, "unlimited" by preference.+On most *nix operating systems, you can see the current values by typing the following at a command prompt.[source,bash]----@@ -285,25 +355,34 @@ These four settings in particular are important to have set very high, unlimited* max processes (`ulimit -u`): 65,000 is the recommended _minimum_.* file handles (`ulimit -n`): 65,000 is the recommended _minimum_. All the files used by all replicas have their file handles open at once so this can grow quite large.-* virtual memory (`ulimit -v`): Set to unlimited. This is used to by MMapping the indexes.+* virtual memory (`ulimit -v`): Set to unlimited.+This is used to by MMapping the indexes.* max memory size (`ulimit -m`): Also used by MMap, set to unlimited.* If your system supports it, `sysctl vm.max_map_count`, should be set to unlimited as well.-We strongly recommend that these settings be permanently raised. The exact process to permanently raise them will vary per operating system. Some systems require editing configuration files and restarting your server. Consult your system administrators for guidance in your particular environment.+We strongly recommend that these settings be permanently raised.+The exact process to permanently raise them will vary per operating system.+Some systems require editing configuration files and restarting your server.+Consult your system administrators for guidance in your particular environment.[WARNING]====-Check these limits every time you upgrade your kernel or operating system. These operations can reset these to their defaults.+Check these limits every time you upgrade your kernel or operating system.+These operations can reset these to their defaults.====[WARNING]====-If these limits are exceeded, the problems reported by Solr vary depending on the specific operation responsible for exceeding the limit. Errors such as "too many open files", "connection error", and "max processes exceeded" have been reported, as well as SolrCloud recovery failures.+If these limits are exceeded, the problems reported by Solr vary depending on the specific operation responsible for exceeding the limit.+Errors such as "too many open files", "connection error", and "max processes exceeded" have been reported, as well as SolrCloud recovery failures.======= Avoid Swapping (*nix Operating Systems)-When running a Java application like Lucene/Solr, having the OS swap memory to disk is a very bad situation. We usually prefer a hard crash so other healthy Solr nodes can take over, instead of letting a Solr node swap, causing terrible performance, timeouts and an unstable system. So our recommendation is to disable swap on the host altogether or reduce the "swappiness". These instructions are valid for Linux environments. Also note that when running Solr in a Docker container, these changes must be applied to the *host*.+When running a Java application like Lucene/Solr, having the OS swap memory to disk is a very bad situation.+We usually prefer a hard crash so other healthy Solr nodes can take over, instead of letting a Solr node swap, causing terrible performance, timeouts and an unstable system.+So our recommendation is to disable swap on the host altogether or reduce the "swappiness". These instructions are valid for Linux environments.+Also note that when running Solr in a Docker container, these changes must be applied to the *host*.==== Disabling Swap@@ -318,14 +397,18 @@ If you want to make this setting permanent, first make sure you have more than e==== Reduce Swappiness-An alternative option is to reduce the "swappiness" of your system. A Linux system will by default be quite aggressive in swapping out memory to disk, by having a high default "swappiness" value. By reducing this to a very low value, it will have almost the same effect as using `swapoff`, but Linux will still be allowed to swap in case of emergency. To check the current swappiness setting, run:+An alternative option is to reduce the "swappiness" of your system.+A Linux system will by default be quite aggressive in swapping out memory to disk, by having a high default "swappiness" value.+By reducing this to a very low value, it will have almost the same effect as using `swapoff`, but Linux will still be allowed to swap in case of emergency.+To check the current swappiness setting, run:[source,bash]----cat /proc/sys/vm/swappiness-----Next, to change the setting permanently, open `/etc/sysctl.conf` as the root user. Then, change or add this line to the file:+Next, to change the setting permanently, open `/etc/sysctl.conf` as the root user.+Then, change or add this line to the file:[source,bash]----@@ -345,28 +428,40 @@ include::securing-solr.adoc[tag=security-network-binding-1]== Running Multiple Solr Nodes per Host-The `bin/solr` script is capable of running multiple instances on one machine, but for a *typical* installation, this is not a recommended setup. Extra CPU and memory resources are required for each additional instance. A single instance is easily capable of handling multiple indexes.+The `bin/solr` script is capable of running multiple instances on one machine, but for a *typical* installation, this is not a recommended setup.+Extra CPU and memory resources are required for each additional instance.+A single instance is easily capable of handling multiple indexes..When to ignore the recommendation[NOTE]====-For every recommendation, there are exceptions. For the recommendation above, that exception is mostly applicable when discussing extreme scalability. The best reason for running multiple Solr nodes on one host is decreasing the need for extremely large heaps.+For every recommendation, there are exceptions.+For the recommendation above, that exception is mostly applicable when discussing extreme scalability.+The best reason for running multiple Solr nodes on one host is decreasing the need for extremely large heaps.-When the Java heap gets very large, it can result in extremely long garbage collection pauses, even with the GC tuning that the startup script provides by default. The exact point at which the heap is considered "very large" will vary depending on how Solr is used. This means that there is no hard number that can be given as a threshold, but if your heap is reaching the neighborhood of 16 to 32 gigabytes, it might be time to consider splitting nodes. Ideally this would mean more machines, but budget constraints might make that impossible.+When the Java heap gets very large, it can result in extremely long garbage collection pauses, even with the GC tuning that the startup script provides by default.+The exact point at which the heap is considered "very large" will vary depending on how Solr is used.+This means that there is no hard number that can be given as a threshold, but if your heap is reaching the neighborhood of 16 to 32 gigabytes, it might be time to consider splitting nodes.+Ideally this would mean more machines, but budget constraints might make that impossible.-There is another issue once the heap reaches 32GB. Below 32GB, Java is able to use compressed pointers, but above that point, larger pointers are required, which uses more memory and slows down the JVM.+There is another issue once the heap reaches 32GB.+Below 32GB, Java is able to use compressed pointers, but above that point, larger pointers are required, which uses more memory and slows down the JVM.Because of the potential garbage collection issues and the particular issues that happen at 32GB, if a single instance would require a 64GB heap, performance is likely to improve greatly if the machine is set up with two nodes that each have a 31GB heap.====-If your use case requires multiple instances, at a minimum you will need unique Solr home directories for each node you want to run; ideally, each home should be on a different physical disk so that multiple Solr nodes don’t have to compete with each other when accessing files on disk. Having different Solr home directories implies that you’ll need a different include file for each node. Moreover, if using the `/etc/init.d/solr` script to control Solr as a service, then you’ll need a separate script for each node. The easiest approach is to use the service installation script to add multiple services on the same host, such as:+If your use case requires multiple instances, at a minimum you will need unique Solr home directories for each node you want to run; ideally, each home should be on a different physical disk so that multiple Solr nodes don’t have to compete with each other when accessing files on disk.+Having different Solr home directories implies that you’ll need a different include file for each node.+Moreover, if using the `/etc/init.d/solr` script to control Solr as a service, then you’ll need a separate script for each node.+The easiest approach is to use the service installation script to add multiple services on the same host, such as:[source,bash,subs="attributes"]----sudo bash ./install_solr_service.sh solr-{solr-docs-version}.0.tgz -s solr2 -p 8984-----The command shown above will add a service named `solr2` running on port 8984 using `/var/solr2` for writable (aka "live") files; the second server will still be owned and run by the `solr` user and will use the Solr distribution files in `/opt`. After installing the solr2 service, verify it works correctly by doing:+The command shown above will add a service named `solr2` running on port 8984 using `/var/solr2` for writable (aka "live") files; the second server will still be owned and run by the `solr` user and will use the Solr distribution files in `/opt`.+After installing the solr2 service, verify it works correctly by doing:[source,bash]----diff --git a/solr/solr-ref-guide/src/the-term-vector-component.adoc b/solr/solr-ref-guide/src/term-vector-component.adocsimilarity index 99%rename from solr/solr-ref-guide/src/the-term-vector-component.adocrename to solr/solr-ref-guide/src/term-vector-component.adocindex 3d88e3746af..145b093e147 100644--- a/solr/solr-ref-guide/src/the-term-vector-component.adoc+++ b/solr/solr-ref-guide/src/term-vector-component.adoc@@ -1,4 +1,4 @@-= The Term Vector Component+= Term Vector Component// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional informationdiff --git a/solr/solr-ref-guide/src/the-terms-component.adoc b/solr/solr-ref-guide/src/terms-component.adocsimilarity index 98%rename from solr/solr-ref-guide/src/the-terms-component.adocrename to solr/solr-ref-guide/src/terms-component.adocindex 4f1a13b1ef2..b7c84d81fb9 100644--- a/solr/solr-ref-guide/src/the-terms-component.adoc+++ b/solr/solr-ref-guide/src/terms-component.adoc@@ -1,4 +1,4 @@-= The Terms Component+= Terms Component// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -22,7 +22,7 @@ In a sense, this search component provides fast field-faceting over the whole in== Configuring the Terms Component-Terms Component is one of <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#defining-search-components,the default search components>>+Terms Component is one of <<requesthandlers-searchcomponents.adoc#defining-search-components,the default search components>>and does not need to be defined in `solrconfig.xml`.The definition is equivalent to:diff --git a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc b/solr/solr-ref-guide/src/the-well-configured-solr-instance.adocdeleted file mode 100644index 87f5e0f662d..00000000000--- a/solr/solr-ref-guide/src/the-well-configured-solr-instance.adoc+++ /dev/null@@ -1,48 +0,0 @@-= The Well-Configured Solr Instance-:page-children: configuring-solrconfig-xml, \- solr-cores-and-solr-xml, \- resource-loading, \- configuration-apis, \- implicit-requesthandlers, \- jvm-settings, \- v2-api--// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section tells you how to fine-tune your Solr instance for optimum performance.--This section covers the following topics:--<<configuring-solrconfig-xml.adoc#,Configuring solrconfig.xml>>: Describes how to work with the main configuration file for Solr, `solrconfig.xml`, covering the major sections of the file.--<<solr-cores-and-solr-xml.adoc#,Solr Cores and solr.xml>>: Describes how to work with `solr.xml` and `core.properties` to configure your Solr core, or multiple Solr cores within a single instance.--<<resource-loading.adoc#,Resource Loading>>: Describes how word lists, model files, and other related data are resolved by the components that need them.--<<configuration-apis.adoc#,Configuration APIs>>: Describes several APIs used to configure Solr: Blob Store, Config, Request Parameters and Managed Resources.--<<implicit-requesthandlers.adoc#,Implicit RequestHandlers>>: Describes various end-points automatically provided by Solr and how to configure them.--<<jvm-settings.adoc#,JVM Settings>>: Gives some guidance on best practices for working with Java Virtual Machines.--<<v2-api.adoc#,V2 API>>: Describes how to use the new V2 APIs, a redesigned API framework covering most Solr APIs.--[IMPORTANT]-====-The focus of this section is generally on configuring a single Solr instance, but for those interested in scaling a Solr implementation in a cluster environment, see also the section <<solrcloud.adoc#,SolrCloud>>. There are also options to scale through sharding or replication, described in the section <<legacy-scaling-and-distribution.adoc#,Legacy Scaling and Distribution>>.-====diff --git a/solr/solr-ref-guide/src/thread-dump.adoc b/solr/solr-ref-guide/src/thread-dump.adocindex 3def77c8542..e6e7eb35523 100644--- a/solr/solr-ref-guide/src/thread-dump.adoc+++ b/solr/solr-ref-guide/src/thread-dump.adoc@@ -1,4 +1,4 @@-= Thread Dump+= Thread Dump Screen// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,17 +18,16 @@The Thread Dump screen lets you inspect the currently active threads on your server.-Each thread is listed and access to the stacktraces is available where applicable. Icons to the left indicate the state of the thread: for example, threads with a green check-mark in a green circle are in a "RUNNABLE" state. On the right of the thread name, a down-arrow means you can expand to see the stacktrace for that thread.+Each thread is listed and access to the stacktraces is available where applicable.+Icons to the left indicate the state of the thread: for example, threads with a green check-mark in a green circle are in a "RUNNABLE" state.+On the right of the thread name, a down-arrow means you can expand to see the stacktrace for that thread..List of Threadsimage::images/thread-dump/thread_dump_1.png[image,width=484,height=250]-When you move your cursor over a thread name, a box floats over the name with the state for that thread. Thread states can be:-// TODO: Change column width to %autowidth.spread when https://github.com/asciidoctor/asciidoctor-pdf/issues/599 is fixed--[cols="25,75",options="header"]+[%autowidth.stretch,options="header"]|===|State |Meaning|NEW |A thread that has not yet started.diff --git a/solr/solr-ref-guide/src/tokenizers.adoc b/solr/solr-ref-guide/src/tokenizers.adocindex 64ee6c19aab..872904e6e0c 100644--- a/solr/solr-ref-guide/src/tokenizers.adoc+++ b/solr/solr-ref-guide/src/tokenizers.adoc@@ -18,6 +18,19 @@Tokenizers are responsible for breaking field data into lexical units, or _tokens_.+Each token is (usually) a sub-sequence of the characters in the text.+An analyzer is aware of the field it is configured for, but a tokenizer is not.+Tokenizers read from a character stream (a _Reader_) and produce a sequence of token objects (a _TokenStream_).++Characters such as whitespace or other delimiters in the input stream may be discarded.+They may also be added to or replaced, such as mapping aliases or abbreviations to normalized forms.++A token contains various metadata in addition to its text value, such as the location at which the token occurs in the field.+Because a tokenizer may produce tokens that diverge from the input text, you should not assume that the text of the token is the same text that occurs in the field, or that its length is the same as the original text.+It's also possible for more than one token to have the same position or refer to the same offset in the original text.+Keep this in mind if you use token metadata for things like highlighting search results in the field text.++== About TokenizersYou configure the tokenizer for a text field type in `schema.xml` with a `<tokenizer>` element, as a child of `<analyzer>`:[.dynamic-tabs]@@ -50,7 +63,10 @@ You configure the tokenizer for a text field type in `schema.xml` with a `<token====---The name/class attribute names a factory class that will instantiate a tokenizer object when needed. Tokenizer factory classes implement the `org.apache.lucene.analysis.TokenizerFactory`. A TokenizerFactory's `create()` method accepts a Reader and returns a TokenStream. When Solr creates the tokenizer it passes a Reader object that provides the content of the text field.+The name/class attribute names a factory class that will instantiate a tokenizer object when needed.+Tokenizer factory classes implement the `org.apache.lucene.analysis.TokenizerFactory`.+A TokenizerFactory's `create()` method accepts a Reader and returns a TokenStream.+When Solr creates the tokenizer it passes a Reader object that provides the content of the text field.Arguments may be passed to tokenizer factories by setting attributes on the `<tokenizer>` element.@@ -82,6 +98,15 @@ Arguments may be passed to tokenizer factories by setting attributes on the `<to====--+=== When to Use a CharFilter vs. a TokenFilter++There are several pairs of CharFilters and TokenFilters that have related (i.e., `MappingCharFilter` and `ASCIIFoldingFilter`) or nearly identical (i.e., `PatternReplaceCharFilterFactory` and `PatternReplaceFilterFactory`) functionality and it may not always be obvious which is the best choice.++The decision about which to use depends largely on which Tokenizer you are using, and whether you need to preprocess the stream of characters.++For example, suppose you have a tokenizer such as `StandardTokenizer` and although you are pretty happy with how it works overall, you want to customize how some specific characters behave.+You could modify the rules and re-build your own tokenizer with JFlex, but it might be easier to simply map some of the characters before tokenization with a `CharFilter`.+The following sections describe the tokenizer factory classes included in this release of Solr.== Standard Tokenizerdiff --git a/solr/solr-ref-guide/src/transforming-and-indexing-custom-json.adoc b/solr/solr-ref-guide/src/transforming-and-indexing-custom-json.adocindex 9bbea92e8c0..1c3b5fe4010 100644--- a/solr/solr-ref-guide/src/transforming-and-indexing-custom-json.adoc+++ b/solr/solr-ref-guide/src/transforming-and-indexing-custom-json.adoc@@ -84,7 +84,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\[example.tab-pane#v2transform]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json/docs'\@@ -209,7 +209,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\[example.tab-pane#v2simpler]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json/docs'\@@ -303,7 +303,7 @@ Say we wanted to define parameters to split documents at the `exams` field, and[example.tab-pane#v2splitparams]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl http://localhost:8983/api/cores/techproducts/config/params -H 'Content-type:application/json' -d '{@@ -360,7 +360,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs?useParams=my_para[example.tab-pane#v2useparams]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json?useParams=my_params' -H 'Content-type:application/json' -d '{@@ -454,7 +454,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\[example.tab-pane#v2wildcards]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json'\@@ -542,7 +542,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\[example.tab-pane#v2wildcardtxt]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json'\@@ -628,7 +628,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs?split=/exams'\[example.tab-pane#v2wildcardfqn]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json?split=/exams'\@@ -718,7 +718,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-type[example.tab-pane#v2multidocs]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-type:application/json' -d '@@ -756,7 +756,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-type[example.tab-pane#v2array]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl 'http://localhost:8983/api/cores/techproducts/update/json' -H 'Content-type:application/json' -d '[@@ -826,7 +826,7 @@ Alternately, use the Request Parameters feature to set these parameters, as show[example.tab-pane#v2jsondefault]====-[.tab-label]*V2 API Standalone Solr*+[.tab-label]*V2 API User-Managed / Single-Node Solr*[source,bash]----curl http://localhost:8983/api/cores/techproducts/config/params -H 'Content-type:application/json' -d '{diff --git a/solr/solr-ref-guide/src/aws-solrcloud-tutorial.adoc b/solr/solr-ref-guide/src/tutorial-aws.adocsimilarity index 69%rename from solr/solr-ref-guide/src/aws-solrcloud-tutorial.adocrename to solr/solr-ref-guide/src/tutorial-aws.adocindex 325def69b41..24633e87f0e 100644--- a/solr/solr-ref-guide/src/aws-solrcloud-tutorial.adoc+++ b/solr/solr-ref-guide/src/tutorial-aws.adoc@@ -19,7 +19,9 @@This guide is a tutorial on how to set up a multi-node SolrCloud cluster on https://aws.amazon.com/ec2[Amazon Web Services (AWS) EC2] instances for early development and design.-This tutorial is not meant for production systems. For one, it uses Solr's embedded ZooKeeper instance, and for production you should have at least 3 ZooKeeper nodes in an ensemble. There are additional steps you should take for a production installation; refer to <<taking-solr-to-production.adoc#,Taking Solr to Production>> for how to deploy Solr in production.+This tutorial is not meant for production systems.+For one, it uses Solr's embedded ZooKeeper instance, and for production you should have at least 3 ZooKeeper nodes in an ensemble.+There are additional steps you should take for a production installation; refer to <<taking-solr-to-production.adoc#,Taking Solr to Production>> for how to deploy Solr in production.In this guide we are going to:@@ -48,25 +50,37 @@ To use this guide, you must have the following:. Navigate to the https://console.aws.amazon.com/ec2/v2/home[AWS EC2 console] and to the region of your choice.. Configure an http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html[AWS security group] which will limit access to the installation and allow our launched EC2 instances to talk to each other without restrictions... From the EC2 Dashboard, click btn:[Security Groups] from the left-hand menu, under "Network & Security".- .. Click btn:[Create Security Group] under the _Security Groups_ section. Give your security group a descriptive name.+ .. Click btn:[Create Security Group] under the _Security Groups_ section.+ Give your security group a descriptive name... You can select one of the existing https://aws.amazon.com/vpc[VPCs] or create a new one... We need two ports open for our cloud here:... Solr port. In this example we will use Solr's default port 8983.... ZooKeeper Port: We'll use Solr's embedded ZooKeeper, so we'll use the default port 9983 (see the <<Deploying with External ZooKeeper>> to configure external ZooKeeper).-.. Click btn:[Inbound] to set inbound network rules, then select btn:[Add Rule]. Select "Custom TCP" as the type. Enter 8983 for the "Port Range" and choose "My IP for the Source, then enter your public IP. Create a second rule with the same type and source, but enter 9983 for the port.-+-This will limit access to your current machine. If you want wider access to the instance in order to collaborate with others, you can specify that, but make sure you only allow as much access as needed. A Solr instance should not be exposed to general Internet traffic.-.. Add another rule for SSH access. Choose "SSH" as the type, and again "My IP" for the source and again enter your public IP. You need SSH access on all instances to install and configure Solr.+.. Click btn:[Inbound] to set inbound network rules, then select btn:[Add Rule].+Select "Custom TCP" as the type.+Enter `8983` for the "Port Range" and choose "My IP for the Source, then enter your public IP.+Create a second rule with the same type and source, but enter `9983` for the port.+++This will limit access to your current machine.+If you want wider access to the instance in order to collaborate with others, you can specify that, but make sure you only allow as much access as needed.+A Solr instance should never be exposed to general Internet traffic.+.. Add another rule for SSH access.+Choose "SSH" as the type, and again "My IP" for the source and again enter your public IP.+You need SSH access on all instances to install and configure Solr... Review the details, your group configuration should look like this:+-image::images/aws-solrcloud-tutorial/aws-security-create.png[image,width=600,height=400]+image::images/tutorial-aws/aws-security-create.png[image,width=600,height=400].. Click btn:[Create] when finished.-.. We need to modify the rules so that instances that are part of the group can talk to all other instances that are part of the same group. We could not do this while creating the group, so we need to edit the group after creating it to add this.-... Select the newly created group in the Security Group overview table. Under the "Inbound" tab, click btn:[Edit].-... Click btn:[Add rule]. Choose `All TCP` from the pulldown list for the type, and enter `0-65535` for the port range. Specify the name of the current Security Group as the `solr-sample`.+.. We need to modify the rules so that instances that are part of the group can talk to all other instances that are part of the same group.+We could not do this while creating the group, so we need to edit the group after creating it to add this.+... Select the newly created group in the Security Group overview table.+Under the "Inbound" tab, click btn:[Edit].+... Click btn:[Add rule].+Choose `All TCP` from the pulldown list for the type, and enter `0-65535` for the port range.+Specify the name of the current Security Group as the `solr-sample`... Review the details, your group configuration should now look like this:+-image::images/aws-solrcloud-tutorial/aws-security-edit.png[image,width=600,height=400]+image::images/tutorial-aws/aws-security-edit.png[image,width=600,height=400].. Click btn:[Save] when finished.=== Configure Instances and Launch@@ -76,17 +90,27 @@ Once the security group is in place, you can choose btn:[Instances] from the lefUnder Instances, click btn:[Launch Instance] button and follow the wizard steps:. Choose your Amazon Machine Image (AMI):-Choose *Amazon Linux AMI, SSD Volume Type* as the AMI. There are both commercial AMIs and Community based AMIs available, e.g., Amazon Linux AMI (HVM), SSD Volume Type, but this is a nice AMI to use for our purposes. Click btn:[Select] next to the image you choose.-. The next screen asks you to choose the instance type, *t2.medium* is sufficient. Choose it from the list, then click btn:[Configure Instance Details].-. Configure the instance. Enter *2* in the "Number of instances" field. Make sure the setting for "Auto-assign Public IP" is "Enabled".-. When finished, click btn:[Add Storage]. The default of *8 GB* for size and *General Purpose SSD* for the volume type is sufficient for running this quick start. Optionally select "Delete on termination" if you know you won't need the data stored in Solr indexes after you terminate the instances.-. When finished, click btn:[Add Tags]. You do not have to add any tags for this quick start, but you can add them if you want.- . Click btn:[Configure Security Group]. Choose *Select an existing security group* and select the security group you created earlier: `solr-sample`. You should see the expected inbound rules at the bottom of the page.+Choose *Amazon Linux AMI, SSD Volume Type* as the AMI.+There are both commercial AMIs and Community based AMIs available, e.g., Amazon Linux AMI (HVM), SSD Volume Type, but this is a nice AMI to use for our purposes.+Click btn:[Select] next to the image you choose.+. The next screen asks you to choose the instance type, *t2.medium* is sufficient.+Choose it from the list, then click btn:[Configure Instance Details].+. Configure the instance.+Enter *2* in the "Number of instances" field.+Make sure the setting for "Auto-assign Public IP" is "Enabled".+. When finished, click btn:[Add Storage].+The default of *8 GB* for size and *General Purpose SSD* for the volume type is sufficient for running this quick start.+Optionally select "Delete on termination" if you know you won't need the data stored in Solr indexes after you terminate the instances.+. When finished, click btn:[Add Tags].+You do not have to add any tags for this quick start, but you can add them if you want.+ . Click btn:[Configure Security Group].+ Choose *Select an existing security group* and select the security group you created earlier: `solr-sample`.+ You should see the expected inbound rules at the bottom of the page.. Click btn:[Review].. If everything looks correct, click btn:[Launch].. Select an existing “private key file” or create a new one and download to your local machine so you will be able to login into the instances via SSH.+-image::images/aws-solrcloud-tutorial/aws-key.png[image,width=600,height=400]+image::images/tutorial-aws/aws-key.png[image,width=600,height=400]. On the instances list, you can watch the states change. You cannot use the instances until they become *“running”*.@@ -157,7 +181,8 @@ $ cd $SOLR_HOME# start Solr node on 8983 and connect to ZooKeeper running on first node$ bin/solr start -c -p 8983 -h solr-node-2 -z solr-node-1:9983-. Inspect and Verify. Inspect the Solr nodes state from browser on local machine:+. Inspect and Verify.+Inspect the Solr nodes state from browser on local machine:+Go to:+@@ -172,7 +197,7 @@ You should able to see Solr UI dashboard for both nodes.== Create Collection, Index and Query-You can refer <<solr-tutorial.adoc#exercise-1,Solr Tutorial>> for an extensive walkthrough on creating collections with multiple shards and replicas, indexing data via different methods and querying documents accordingly.+You can refer <<solr-tutorial.adoc#,Solr Tutorial>> for an extensive walkthrough on creating collections with multiple shards and replicas, indexing data via different methods and querying documents accordingly.== Deploying with External ZooKeeper@@ -192,9 +217,13 @@ $ sudo vim /etc/hosts=== Install ZooKeeper-These steps will help you install and configure a single instance of ZooKeeper on AWS. This is not sufficient for a production, use, however, where a ZooKeeper ensemble of at least three nodes is recommended. See the section <<setting-up-an-external-zookeeper-ensemble.adoc#,Setting Up an External ZooKeeper Ensemble>> for information about how to change this single-instance into an ensemble.+These steps will help you install and configure a single instance of ZooKeeper on AWS.+This is not sufficient for a production, use, however, where a ZooKeeper ensemble of at least three nodes is recommended.+See the section <<zookeeper-ensemble.adoc#,ZooKeeper Ensemble>> for information about how to change this single-instance into an ensemble.-. Download a stable version of ZooKeeper. In this example we're using ZooKeeper v{ivy-zookeeper-version}. On the node you're using to host ZooKeeper (`zookeeper-node`), download the package and untar it:+. Download a stable version of ZooKeeper.+In this example we're using ZooKeeper v{ivy-zookeeper-version}.+On the node you're using to host ZooKeeper (`zookeeper-node`), download the package and untar it:+[source,bash,subs="attributes"]----@@ -204,7 +233,9 @@ $ wget https://archive.apache.org/dist/zookeeper/zookeeper-{ivy-zookeeper-versio$ tar -zxvf zookeeper-{ivy-zookeeper-version}.tar.gz----+-Add an environment variable for ZooKeeper's home directory (`ZOO_HOME`) to the `.bashrc` for the user who will be running the process. The rest of the instructions assume you have set this variable. Correct the path to the ZooKeeper installation as appropriate if where you put it does not match the below.+Add an environment variable for ZooKeeper's home directory (`ZOO_HOME`) to the `.bashrc` for the user who will be running the process.+The rest of the instructions assume you have set this variable.+Correct the path to the ZooKeeper installation as appropriate if where you put it does not match the below.+[source,bash,subs="attributes"]----@@ -263,8 +294,9 @@ $ bin/solr start -c -p 8983 -h solr-node-1 -z zookeeper-node:2181[TIP]====-As noted earlier, a single ZooKeeper node is not sufficient for a production installation. See these additional resources for more information about deploying Solr in production, which can be used once you have the EC2 instances up and running:+As noted earlier, a single ZooKeeper node is not sufficient for a production installation.+See these additional resources for more information about deploying Solr in production, which can be used once you have the EC2 instances up and running:* <<taking-solr-to-production.adoc#,Taking Solr to Production>>-* <<setting-up-an-external-zookeeper-ensemble.adoc#,Setting Up an External ZooKeeper Ensemble>>+* <<zookeeper-ensemble.adoc#,ZooKeeper Ensemble>>====diff --git a/solr/solr-ref-guide/src/tutorial-diy.adoc b/solr/solr-ref-guide/src/tutorial-diy.adocnew file mode 100644index 00000000000..86d9473eb84--- /dev/null+++ b/solr/solr-ref-guide/src/tutorial-diy.adoc@@ -0,0 +1,107 @@+= Exercise 3 Index Your Own Data+:experimental:+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[[exercise-3]]+== Exercise 3: Index Your Own Data++For this last exercise, work with a dataset of your choice.+This can be files on your local hard drive, a set of data you have worked with before, or maybe a sample of the data you intend to index to Solr for your production application.++This exercise is intended to get you thinking about what you will need to do for your application:++* What sorts of data do you need to index?+* What will you need to do to prepare Solr for your data (such as, create specific fields, set up copy fields, determine analysis rules, etc.)+* What kinds of search options do you want to provide to users?+* How much testing will you need to do to ensure everything works the way you expect?++=== Create Your Own Collection++Before you get started, create a new collection, named whatever you'd like.+In this example, the collection will be named "localDocs"; replace that name with whatever name you choose if you want to.++`./bin/solr create -c localDocs -s 2 -rf 2`++Again, as we saw from Exercise 2 above, this will use the `_default` configset and all the schemaless features it provides.+As we noted previously, this may cause problems when we index our data.+You may need to iterate on indexing a few times before you get the schema right.++=== Indexing Ideas++Solr has lots of ways to index data.+Choose one of the approaches below and try it out with your system:++Local Files with `bin/post`::+If you have a local directory of files, the Post Tool (`bin/post`) can index a directory of files. We saw this in action in our first exercise.+++We used only JSON, XML and CSV in our exercises, but the Post Tool can also handle HTML, PDF, Microsoft Office formats (such as MS Word), plain text, and more.+++In this example, assume there is a directory named "Documents" locally.+To index it, we would issue a command like this (correcting the collection name after the `-c` parameter as needed):+++`./bin/post -c localDocs ~/Documents`+++You may get errors as it works through your documents.+These might be caused by the field guessing, or the file type may not be supported.+Indexing content such as this demonstrates the need to plan Solr for your data, which requires understanding it and perhaps also some trial and error.++SolrJ::+SolrJ is a Java-based client for interacting with Solr. Use <<solrj.adoc#,SolrJ>> for JVM-based languages or other <<client-apis.adoc#,Solr clients>> to programmatically create documents to send to Solr.++Documents Screen::+Use the Admin UI <<documents-screen.adoc#,Documents tab>> (at http://localhost:8983/solr/#/localDocs/documents) to paste in a document to be indexed, or select `Document Builder` from the `Document Type` dropdown to build a document one field at a time.+Click on the btn:[Submit Document] button below the form to index your document.++=== Updating Data++You may notice that even if you index content in this tutorial more than once, it does not duplicate the results found.+This is because the example Solr schema (a file named either `managed-schema` or `schema.xml`) specifies a `uniqueKey` field called `id`.+Whenever you POST commands to Solr to add a document with the same value for the `uniqueKey` as an existing document, it automatically replaces it for you.++You can see that that has happened by looking at the values for `numDocs` and `maxDoc` in the core-specific Overview section of the Solr Admin UI.++`numDocs` represents the number of searchable documents in the index (and will be larger than the number of XML, JSON, or CSV files since some files contained more than one document).+The `maxDoc` value may be larger as the `maxDoc` count includes logically deleted documents that have not yet been physically removed from the index.+You can re-post the sample files over and over again as much as you want and `numDocs` will never increase, because the new documents will constantly be replacing the old.++Go ahead and edit any of the existing example data files, change some of the data, and re-run the PostTool (`bin/post`). You'll see your changes reflected in subsequent searches.++=== Deleting Data++If you need to iterate a few times to get your schema right, you may want to delete documents to clear out the collection and try again.+Note, however, that merely removing documents doesn't change the underlying field definitions.+Essentially, this will allow you to reindex your data after making changes to fields for your needs.++You can delete data by POSTing a delete command to the update URL and specifying the value of the document's unique key field, or a query that matches multiple documents (be careful with that one!).+We can use `bin/post` to delete documents also if we structure the request properly.++Execute the following command to delete a specific document:++`bin/post -c localDocs -d "<delete><id>SP2514N</id></delete>"`++To delete all documents, you can use "delete-by-query" command like:++`bin/post -c localDocs -d "<delete><query>*:*</query></delete>"`++You can also modify the above to only delete documents that match a specific query.++=== Exercise 3 Wrap Up++At this point, you're ready to start working on your own.++Jump ahead to the overall <<solr-tutorial.adoc#wrapping-up,wrap up>> when you're ready to stop Solr and remove all the examples you worked with and start fresh.diff --git a/solr/solr-ref-guide/src/tutorial-films.adoc b/solr/solr-ref-guide/src/tutorial-films.adocnew file mode 100644index 00000000000..4a7e1e2d49e--- /dev/null+++ b/solr/solr-ref-guide/src/tutorial-films.adoc@@ -0,0 +1,458 @@+= Exercise 2 Index Films Data+:experimental:+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[[exercise-2]]+== Exercise 2: Modify the Schema and Index Films Data++This exercise will build on the last one and introduce you to the index schema and Solr's powerful faceting features.++=== Restart Solr++Did you stop Solr after the last exercise? No?+Then go ahead to the next section.++If you did, though, and need to restart Solr, issue these commands:++`./bin/solr start -c -p 8983 -s example/cloud/node1/solr`++This starts the first node.+When it's done start the second node, and tell it how to connect to to ZooKeeper:++`./bin/solr start -c -p 7574 -s example/cloud/node2/solr -z localhost:9983`++NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.++=== Create a New Collection++We're going to use a whole new data set in this exercise, so it would be better to have a new collection instead of trying to reuse the one we had before.++One reason for this is we're going to use a feature in Solr called "field guessing", where Solr attempts to guess what type of data is in a field while it's indexing it.+It also automatically creates new fields in the schema for new fields that appear in incoming documents.+This mode is called "Schemaless".+We'll see the benefits and limitations of this approach to help you decide how and where to use it in your real application.++.What is a "schema" and why do I need one?+[sidebar]+****+Solr's schema is a single file (in XML) that stores the details about the fields and field types Solr is expected to understand.+The schema defines not only the field or field type names, but also any modifications that should happen to a field before it is indexed.+For example, if you want to ensure that a user who enters "abc" and a user who enters "ABC" can both find a document containing the term "ABC", you will want to normalize (lower-case, in this case) "ABC" when it is indexed, and normalize the user query to be sure of a match.+These rules are defined in your schema.++Earlier in the tutorial we mentioned copy fields, which are fields made up of data that originated from other fields.+You can also define dynamic fields, which use wildcards (such as `*_t` or `*_s`) to dynamically create fields of a specific field type.+These types of rules are also defined in the schema.+****++When you initially started Solr in the first exercise, we had a choice of a configset to use.+The one we chose had a schema that was pre-defined for the data we later indexed.+This time, we're going to use a configset that has a very minimal schema and let Solr figure out from the data what fields to add.++The data you're going to index is related to movies, so start by creating a collection named "films" that uses the `_default` configset:++`bin/solr create -c films -s 2 -rf 2`++Whoa, wait.+We didn't specify a configset!+That's fine, the `_default` is appropriately named, since it's the default and is used if you don't specify one at all.++We did, however, set two parameters `-s` and `-rf`.+Those are the number of shards to split the collection across (2) and how many replicas to create (2).+This is equivalent to the options we had during the interactive example from the first exercise.++You should see output like:++[source,subs="verbatim,attributes+"]+----+WARNING: Using _default configset. Data driven schema functionality is enabled by default, which is+ NOT RECOMMENDED for production use.++ To turn it off:+ bin/solr config -c films -p 7574 -action set-user-property -property update.autoCreateFields -value false++Connecting to ZooKeeper at localhost:9983 ...+INFO - 2017-07-27 15:07:46.191; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready+Uploading /{solr-docs-version}.0/server/solr/configsets/_default/conf for config films to ZooKeeper at localhost:9983++Creating new collection 'films' using command:+http://localhost:7574/solr/admin/collections?action=CREATE&name=films&numShards=2&replicationFactor=2&collection.configName=films++{+ "responseHeader":{+ "status":0,+ "QTime":3830},+ "success":{+ "192.168.0.110:8983_solr":{+ "responseHeader":{+ "status":0,+ "QTime":2076},+ "core":"films_shard2_replica_n1"},+ "192.168.0.110:7574_solr":{+ "responseHeader":{+ "status":0,+ "QTime":2494},+ "core":"films_shard1_replica_n2"}}}+----++The first thing the command printed was a warning about not using this configset in production.+That's due to some of the limitations we'll cover shortly.++Otherwise, though, the collection should be created.+If we go to the Admin UI at http://localhost:8983/solr/#/films/collection-overview we should see the overview screen.++==== Preparing Schemaless for the Films Data++There are two parallel things happening with the schema that comes with the `_default` configset.++First, we are using a "managed schema", which is configured to only be modified by Solr's Schema API.+That means we should not hand-edit it so there isn't confusion about which edits come from which source.+Solr's Schema API allows us to make changes to fields, field types, and other types of schema rules.++Second, we are using "field guessing", which is configured in the `solrconfig.xml` file (and includes most of Solr's various configuration settings).+Field guessing is designed to allow us to start using Solr without having to define all the fields we think will be in our documents before trying to index them.+This is why we call it "schemaless", because you can start quickly and let Solr create fields for you as it encounters them in documents.++Sounds great!+Well, not really, there are limitations.+It's a bit brute force, and if it guesses wrong, you can't change much about a field after data has been indexed without having to reindex.+If we only have a few thousand documents that might not be bad, but if you have millions and millions of documents, or, worse, don't have access to the original data anymore, this can be a real problem.++For these reasons, the Solr community does not recommend going to production without a schema that you have defined yourself.+By this we mean that the schemaless features are fine to start with, but you should still always make sure your schema matches your expectations for how you want your data indexed and how users are going to query it.++It is possible to mix schemaless features with a defined schema.+Using the Schema API, you can define a few fields that you know you want to control, and let Solr guess others that are less important or which you are confident (through testing) will be guessed to your satisfaction.+That's what we're going to do here.++===== Create the "names" Field+The films data we are going to index has a small number of fields for each movie: an ID, director name(s), film name, release date, and genre(s).++If you look at one of the files in `example/films`, you'll see the first film is named _.45_, released in 2006.+As the first document in the dataset, Solr is going to guess the field type based on the data in the record.+If we go ahead and index this data, that first film name is going to indicate to Solr that that field type is a "float" numeric field, and will create a "name" field with a type `FloatPointField`. All data after this record will be expected to be a float.++Well, that's not going to work.+We have titles like _A Mighty Wind_ and _Chicken Run_, which are strings - decidedly not numeric and not floats.+If we let Solr guess the "name" field is a float, what will happen is later titles will cause an error and indexing will fail.+That's not going to get us very far.++What we can do is set up the "name" field in Solr before we index the data to be sure Solr always interprets it as a string.+At the command line, enter this curl command:++[source,bash]+curl -X POST -H 'Content-type:application/json' --data-binary '{"add-field": {"name":"name", "type":"text_general", "multiValued":false, "stored":true}}' http://localhost:8983/solr/films/schema++This command uses the Schema API to explicitly define a field named "name" that has the field type "text_general" (a text field).+It will not be permitted to have multiple values, but it will be stored (meaning it can be retrieved by queries).++You can also use the Admin UI to create fields, but it offers a bit less control over the properties of your field.+It will work for our case, though:++.Creating a field+image::images/solr-tutorial/tutorial-add-field.png[Adding a Field,640,480,pdfwidth=75%]++===== Create a "catchall" Copy Field++There's one more change to make before we start indexing.++In the first exercise when we queried the documents we had indexed, we didn't have to specify a field to search because the configuration we used was set up to copy fields into a `text` field, and that field was the default when no other field was defined in the query.++The configuration we're using now doesn't have that rule.+We would need to define a field to search for every query.+We can, however, set up a "catchall field" by defining a copy field that will take all data from all fields and index it into a field named `\_text_`. Let's do that now.++You can use either the Admin UI or the Schema API for this.++At the command line, use the Schema API again to define a copy field:++[source,bash]+curl -X POST -H 'Content-type:application/json' --data-binary '{"add-copy-field" : {"source":"*","dest":"_text_"}}' http://localhost:8983/solr/films/schema++In the Admin UI, choose btn:[Add Copy Field], then fill out the source and destination for your field, as in this screenshot.++.Creating a copy field+image::images/solr-tutorial/tutorial-add-copy-field.png[Adding a copy field,640,480,pdfwidth=75%]++What this does is make a copy of all fields and put the data into the "\_text_" field.++TIP: It can be very expensive to do this with your production data because it tells Solr to effectively index everything twice.+It will make indexing slower, and make your index larger.+With your production data, you will want to be sure you only copy fields that really warrant it for your application.++OK, now we're ready to index the data and start playing around with it.++=== Index Sample Film Data++The films data we will index is located in the `example/films` directory of your installation.+It comes in three formats: JSON, XML and CSV.+Pick one of the formats and index it into the "films" collection (in each example, one command is for Unix/MacOS and the other is for Windows):++.To Index JSON Format+[source,subs="verbatim,attributes+"]+----+bin/post -c films example/films/films.json++C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dauto example\exampledocs\post.jar example\films\*.json+----++.To Index XML Format+[source,subs="verbatim,attributes+"]+----+bin/post -c films example/films/films.xml++C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dauto example\exampledocs\post.jar example\films\*.xml+----++.To Index CSV Format+[source,subs="verbatim,attributes+"]+----+bin/post -c films example/films/films.csv -params "f.genre.split=true&f.directed_by.split=true&f.genre.separator=|&f.directed_by.separator=|"++C:\solr-{solr-docs-version}.0> java -jar -Dc=films -Dparams=f.genre.split=true&f.directed_by.split=true&f.genre.separator=|&f.directed_by.separator=| -Dauto example\exampledocs\post.jar example\films\*.csv+----++Each command includes these main parameters:++* `-c films`: this is the Solr collection to index data to.+* `example/films/films.json` (or `films.xml` or `films.csv`): this is the path to the data file to index.+You could simply supply the directory where this file resides, but since you know the format you want to index, specifying the exact file for that format is more efficient.++Note the CSV command includes extra parameters.+This is to ensure multi-valued entries in the "genre" and "directed_by" columns are split by the pipe (`|`) character, used in this file as a separator.+Telling Solr to split these columns this way will ensure proper indexing of the data.++Each command will produce output similar to the below seen while indexing JSON:++[source,bash,subs="verbatim,attributes"]+----+$ ./bin/post -c films example/films/films.json+/bin/java -classpath /solr-{solr-docs-version}.0/dist/solr-core-{solr-docs-version}.0.jar -Dauto=yes -Dc=films -Ddata=files org.apache.solr.util.SimplePostTool example/films/films.json+SimplePostTool version 5.0.0+Posting files to [base] url http://localhost:8983/solr/films/update...+Entering auto mode. File endings considered are xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log+POSTing file films.json (application/json) to [base]/json/docs+1 files indexed.+COMMITting Solr index changes to http://localhost:8983/solr/films/update...+Time spent: 0:00:00.878+----++Hooray!++If you go to the Query screen in the Admin UI for films (http://localhost:8983/solr/#/films/query) and hit btn:[Execute Query] you should see 1100 results, with the first 10 returned to the screen.++Let's do a query to see if the "catchall" field worked properly.+Enter "comedy" in the `q` box and hit btn:[Execute Query] again.+You should see 417 results.+Feel free to play around with other searches before we move on to faceting.++[[tutorial-faceting]]+=== Faceting++One of Solr's most popular features is faceting.+Faceting allows the search results to be arranged into subsets (or buckets, or categories), providing a count for each subset.+There are several types of faceting: field values, numeric and date ranges, pivots (decision tree), and arbitrary query faceting.++==== Field Facets++In addition to providing search results, a Solr query can return the number of documents that contain each unique value in the whole result set.++On the Admin UI Query tab, if you check the `facet` checkbox, you'll see a few facet-related options appear:++.Facet options in the Query screen+image::images/solr-tutorial/tutorial-admin-ui-facet-options.png[Solr Quick Start: Query tab facet options]++To see facet counts from all documents (`q=\*:*`): turn on faceting (`facet=true`), and specify the field to facet on via the `facet.field` parameter.+If you only want facets, and no document contents, specify `rows=0`. The `curl` command below will return facet counts for the `genre_str` field:++`curl "http://localhost:8983/solr/films/select?q=\*:*&rows=0&facet=true&facet.field=genre_str"`++In your terminal, you'll see something like:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":11,+ "params":{+ "q":"*:*",+ "facet.field":"genre_str",+ "rows":"0",+ "facet":"true"}},+ "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]+ },+ "facet_counts":{+ "facet_queries":{},+ "facet_fields":{+ "genre_str":[+ "Drama",552,+ "Comedy",389,+ "Romance Film",270,+ "Thriller",259,+ "Action Film",196,+ "Crime Fiction",170,+ "World cinema",167]},+ "facet_ranges":{},+ "facet_intervals":{},+ "facet_heatmaps":{}}}++We've truncated the output here a little bit, but in the `facet_counts` section, you see by default you get a count of the number of documents using each genre for every genre in the index.+Solr has a parameter `facet.mincount` that you could use to limit the facets to only those that contain a certain number of documents (this parameter is not shown in the UI).+Or, perhaps you do want all the facets, and you'll let your application's front-end control how it's displayed to users.++If you wanted to control the number of items in a bucket, you could do something like this:++`curl "http://localhost:8983/solr/films/select?=&q=\*:*&facet.field=genre_str&facet.mincount=200&facet=on&rows=0"`++You should only see 4 facets returned.++There are a great deal of other parameters available to help you control how Solr constructs the facets and facet lists.+We'll cover some of them in this exercise, but you can also see the section <<faceting.adoc#,Faceting>> for more detail.++==== Range Facets++For numerics or dates, it's often desirable to partition the facet counts into ranges rather than discrete values.+A prime example of numeric range faceting, using the example techproducts data from our previous exercise, is `price`.+The films data includes the release date for films, and we could use that to create date range facets, which are another common use for range facets.++The Solr Admin UI doesn't yet support range facet options, so you will need to use curl or similar command line tool for the following examples.++If we construct a query that looks like this:++[source,bash]+curl 'http://localhost:8983/solr/films/select?q=*:*&rows=0'\+ '&facet=true'\+ '&facet.range=initial_release_date'\+ '&facet.range.start=NOW-20YEAR'\+ '&facet.range.end=NOW'\+ '&facet.range.gap=%2B1YEAR'++This will request all films and ask for them to be grouped by year starting with 20 years ago (our earliest release date is in 2000) and ending today.+Note that this query again URL encodes a `+` as `%2B`.++In the terminal you will see:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":8,+ "params":{+ "facet.range":"initial_release_date",+ "facet.limit":"300",+ "q":"*:*",+ "facet.range.gap":"+1YEAR",+ "rows":"0",+ "facet":"on",+ "facet.range.start":"NOW-20YEAR",+ "facet.range.end":"NOW"}},+ "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]+ },+ "facet_counts":{+ "facet_queries":{},+ "facet_fields":{},+ "facet_ranges":{+ "initial_release_date":{+ "counts":[+ "1997-07-28T17:12:06.919Z",0,+ "1998-07-28T17:12:06.919Z",0,+ "1999-07-28T17:12:06.919Z",48,+ "2000-07-28T17:12:06.919Z",82,+ "2001-07-28T17:12:06.919Z",103,+ "2002-07-28T17:12:06.919Z",131,+ "2003-07-28T17:12:06.919Z",137,+ "2004-07-28T17:12:06.919Z",163,+ "2005-07-28T17:12:06.919Z",189,+ "2006-07-28T17:12:06.919Z",92,+ "2007-07-28T17:12:06.919Z",26,+ "2008-07-28T17:12:06.919Z",7,+ "2009-07-28T17:12:06.919Z",3,+ "2010-07-28T17:12:06.919Z",0,+ "2011-07-28T17:12:06.919Z",0,+ "2012-07-28T17:12:06.919Z",1,+ "2013-07-28T17:12:06.919Z",1,+ "2014-07-28T17:12:06.919Z",1,+ "2015-07-28T17:12:06.919Z",0,+ "2016-07-28T17:12:06.919Z",0],+ "gap":"+1YEAR",+ "start":"1997-07-28T17:12:06.919Z",+ "end":"2017-07-28T17:12:06.919Z"}},+ "facet_intervals":{},+ "facet_heatmaps":{}}}++==== Pivot Facets++Another faceting type is pivot facets, also known as "decision trees", allowing two or more fields to be nested for all the various possible combinations.+Using the films data, pivot facets can be used to see how many of the films in the "Drama" category (the `genre_str` field) are directed by a director.+Here's how to get at the raw data for this scenario:++`curl "http://localhost:8983/solr/films/select?q=\*:*&rows=0&facet=on&facet.pivot=genre_str,directed_by_str"`++This results in the following response, which shows a facet for each category and director combination:++[source,json]+{"responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":1147,+ "params":{+ "q":"*:*",+ "facet.pivot":"genre_str,directed_by_str",+ "rows":"0",+ "facet":"on"}},+ "response":{"numFound":1100,"start":0,"maxScore":1.0,"docs":[]+ },+ "facet_counts":{+ "facet_queries":{},+ "facet_fields":{},+ "facet_ranges":{},+ "facet_intervals":{},+ "facet_heatmaps":{},+ "facet_pivot":{+ "genre_str,directed_by_str":[{+ "field":"genre_str",+ "value":"Drama",+ "count":552,+ "pivot":[{+ "field":"directed_by_str",+ "value":"Ridley Scott",+ "count":5},+ {+ "field":"directed_by_str",+ "value":"Steven Soderbergh",+ "count":5},+ {+ "field":"directed_by_str",+ "value":"Michael Winterbottom",+ "count":4}}]}]}}}++We've truncated this output as well - you will see a lot of genres and directors in your screen.++=== Exercise 2 Wrap Up++In this exercise, we learned a little bit more about how Solr organizes data in the indexes, and how to work with the Schema API to manipulate the schema file.+We also learned a bit about facets in Solr, including range facets and pivot facets.+In both of these things, we've only scratched the surface of the available options. If you can dream it, it might be possible!++Like our previous exercise, this data may not be relevant to your needs.+We can clean up our work by deleting the collection.+To do that, issue this command at the command line:++`bin/solr delete -c films`diff --git a/solr/solr-ref-guide/src/getting-started-with-solrcloud.adoc b/solr/solr-ref-guide/src/tutorial-solrcloud.adocsimilarity index 88%rename from solr/solr-ref-guide/src/getting-started-with-solrcloud.adocrename to solr/solr-ref-guide/src/tutorial-solrcloud.adocindex a5451713985..ff983c3cb88 100644--- a/solr/solr-ref-guide/src/getting-started-with-solrcloud.adoc+++ b/solr/solr-ref-guide/src/tutorial-solrcloud.adoc@@ -86,11 +86,14 @@ After starting up all nodes in the cluster, the script prompts you for the nameThe suggested default is "gettingstarted" but you might want to choose a name more appropriate for your specific search application.-Next, the script prompts you for the number of shards to distribute the collection across. <<shards-and-indexing-data-in-solrcloud.adoc#,Sharding>> is covered in more detail later on, so if you're unsure, we suggest using the default of 2 so that you can see how a collection is distributed across multiple nodes in a SolrCloud cluster.+Next, the script prompts you for the number of shards to distribute the collection across.+<<solrcloud-shards-indexing.adoc#,Sharding>> is covered in more detail later on, so if you're unsure, we suggest using the default of 2 so that you can see how a collection is distributed across multiple nodes in a SolrCloud cluster.-Next, the script will prompt you for the number of replicas to create for each shard. <<shards-and-indexing-data-in-solrcloud.adoc#,Replication>> is covered in more detail later in the guide, so if you're unsure, then use the default of 2 so that you can see how replication is handled in SolrCloud.+Next, the script will prompt you for the number of replicas to create for each shard.+<<solrcloud-shards-indexing.adoc#,Replication>> is covered in more detail later in the guide, so if you're unsure, then use the default of 2 so that you can see how replication is handled in SolrCloud.-Lastly, the script will prompt you for the name of a configuration directory for your collection. You can choose *_default*, or *sample_techproducts_configs*. The configuration directories are pulled from `server/solr/configsets/` so you can review them beforehand if you wish. The *_default* configuration is useful when you're still designing a schema for your documents and need some flexibility as you experiment with Solr, since it has schemaless functionality. However, after creating your collection, the schemaless functionality can be disabled in order to lock down the schema (so that documents indexed after doing so will not alter the schema) or to configure the schema by yourself. This can be done as follows (assuming your collection name is `mycollection`):+Lastly, the script will prompt you for the name of a configuration directory for your collection. You can choose *_default*, or *sample_techproducts_configs*.+The configuration directories are pulled from `server/solr/configsets/` so you can review them beforehand if you wish. The *_default* configuration is useful when you're still designing a schema for your documents and need some flexibility as you experiment with Solr, since it has schemaless functionality. However, after creating your collection, the schemaless functionality can be disabled in order to lock down the schema (so that documents indexed after doing so will not alter the schema) or to configure the schema by yourself. This can be done as follows (assuming your collection name is `mycollection`):[.dynamic-tabs]--diff --git a/solr/solr-ref-guide/src/tutorial-techproducts.adoc b/solr/solr-ref-guide/src/tutorial-techproducts.adocnew file mode 100644index 00000000000..13ac40dfcbb--- /dev/null+++ b/solr/solr-ref-guide/src/tutorial-techproducts.adoc@@ -0,0 +1,507 @@+= Exercise 1 Index Techproducts Data+:experimental:+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++This exercise will walk you through how to start Solr as a two-node cluster (both nodes on the same machine) and create a collection during startup.+Then you will index some sample data that ships with Solr and do some basic searches.++== Launch Solr in SolrCloud Mode+To launch Solr, run: `bin/solr start -e cloud` on Unix or MacOS; `bin\solr.cmd start -e cloud` on Windows.++This will start an interactive session that will start two Solr "servers" on your machine.+This command has an option to run without prompting you for input (`-noprompt`), but we want to modify two of the defaults so we won't use that option now.++[source,subs="verbatim,attributes+"]+----+solr-{solr-docs-version}.0:$ ./bin/solr start -e cloud++Welcome to the SolrCloud example!++This interactive session will help you launch a SolrCloud cluster on your local workstation.+To begin, how many Solr nodes would you like to run in your local cluster? (specify 1-4 nodes) [2]:+----+The first prompt asks how many nodes we want to run.+Note the `[2]` at the end of the last line; that is the default number of nodes.+Two is what we want for this example, so you can simply press kbd:[enter].++[source,subs="verbatim,attributes+"]+----+Ok, let's start up 2 Solr nodes for your example SolrCloud cluster.+Please enter the port for node1 [8983]:+----+This will be the port that the first node runs on.+Unless you know you have something else running on port 8983 on your machine, accept this default option also by pressing kbd:[enter].+If something is already using that port, you will be asked to choose another port.++[source,subs="verbatim,attributes+"]+----+Please enter the port for node2 [7574]:+----+This is the port the second node will run on.+Again, unless you know you have something else running on port 7574 on your machine, accept this default option also by pressing kbd:[enter].+If something is already using that port, you will be asked to choose another port.++Solr will now initialize itself and start running on those two nodes.+The script will print the commands it uses for your reference.++[source,subs="verbatim,attributes+"]+----+Starting up 2 Solr nodes for your example SolrCloud cluster.++Creating Solr home directory /solr-{solr-docs-version}.0/example/cloud/node1/solr+Cloning /solr-{solr-docs-version}.0/example/cloud/node1 into+ /solr-{solr-docs-version}.0/example/cloud/node2++Starting up Solr on port 8983 using command:+"bin/solr" start -cloud -p 8983 -s "example/cloud/node1/solr"++Waiting up to 180 seconds to see Solr running on port 8983 [\]+Started Solr server on port 8983 (pid=34942). Happy searching!+++Starting up Solr on port 7574 using command:+"bin/solr" start -cloud -p 7574 -s "example/cloud/node2/solr" -z localhost:9983++Waiting up to 180 seconds to see Solr running on port 7574 [\]+Started Solr server on port 7574 (pid=35036). Happy searching!++INFO - 2017-07-27 12:28:02.835; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready+----++Notice that two instances of Solr have started on two nodes.+Because we are starting in SolrCloud mode, and did not define any details about an external ZooKeeper cluster, Solr launches its own ZooKeeper and connects both nodes to it.++After startup is complete, you'll be prompted to create a collection to use for indexing data.++[source,subs="verbatim,attributes+"]+----+Now let's create a new collection for indexing documents in your 2-node cluster.+Please provide a name for your new collection: [gettingstarted]+----++Here's the first place where we'll deviate from the default options.+This tutorial will ask you to index some sample data included with Solr, called the "techproducts" data.+Let's name our collection "techproducts" so it's easy to differentiate from other collections we'll create later.+Enter `techproducts` at the prompt and hit kbd:[enter].++[source,subs="verbatim,attributes+"]+----+How many shards would you like to split techproducts into? [2]+----++This is asking how many <<solr-glossary.adoc#shard,shards>> you want to split your index into across the two nodes.+Choosing "2" (the default) means we will split the index relatively evenly across both nodes, which is a good way to start.+Accept the default by hitting kbd:[enter].++[source,subs="verbatim,attributes+"]+----+How many replicas per shard would you like to create? [2]+----++A replica is a copy of the index that's used for failover (see also the <<solr-glossary.adoc#replica,Solr Glossary definition>>).+Again, the default of "2" is fine to start with here also, so accept the default by hitting kbd:[enter].++[source,subs="verbatim,attributes+"]+----+Please choose a configuration for the techproducts collection, available options are:+_default or sample_techproducts_configs [_default]+----++We've reached another point where we will deviate from the default option.+Solr has two sample sets of configuration files (called a configset) available out-of-the-box.++A collection must have a configset, which at a minimum includes the two main configuration files for Solr: the schema file (named either `managed-schema` or `schema.xml`), and `solrconfig.xml`.+The question here is which configset you would like to start with.+The `_default` is a bare-bones option, but note there's one whose name includes "techproducts", the same as we named our collection.+This configset is specifically designed to support the sample data we want to use, so enter `sample_techproducts_configs` at the prompt and hit kbd:[enter].++At this point, Solr will create the collection and again output to the screen the commands it issues.++[source,subs="verbatim,attributes+"]+----+Uploading /solr-{solr-docs-version}.0/server/solr/configsets/_default/conf for config techproducts to ZooKeeper at localhost:9983++Connecting to ZooKeeper at localhost:9983 ...+INFO - 2017-07-27 12:48:59.289; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready+Uploading /solr-{solr-docs-version}.0/server/solr/configsets/sample_techproducts_configs/conf for config techproducts to ZooKeeper at localhost:9983++Creating new collection 'techproducts' using command:+http://localhost:8983/solr/admin/collections?action=CREATE&name=techproducts&numShards=2&replicationFactor=2&collection.configName=techproducts++{+ "responseHeader":{+ "status":0,+ "QTime":5460},+ "success":{+ "192.168.0.110:7574_solr":{+ "responseHeader":{+ "status":0,+ "QTime":4056},+ "core":"techproducts_shard1_replica_n1"},+ "192.168.0.110:8983_solr":{+ "responseHeader":{+ "status":0,+ "QTime":4056},+ "core":"techproducts_shard2_replica_n2"}}}++Enabling auto soft-commits with maxTime 3 secs using the Config API++POSTing request to Config API: http://localhost:8983/solr/techproducts/config+{"set-property":{"updateHandler.autoSoftCommit.maxTime":"3000"}}+Successfully set-property updateHandler.autoSoftCommit.maxTime to 3000++SolrCloud example running, please visit: http://localhost:8983/solr+----++*Congratulations!* Solr is ready for data!++You can see that Solr is running by launching the Solr Admin UI in your web browser: http://localhost:8983/solr/.+This is the main starting point for administering Solr.++Solr will now be running two "nodes", one on port 7574 and one on port 8983.+There is one collection created automatically, `techproducts`, a two shard collection, each with two replicas.++The http://localhost:8983/solr/#/~cloud[Cloud tab] in the Admin UI diagrams the collection nicely:++.SolrCloud Diagram+image::images/solr-tutorial/tutorial-solrcloud.png[]++== Index the Techproducts Data++Your Solr server is up and running, but it doesn't contain any data yet, so we can't do any queries.++Solr includes the `bin/post` tool in order to facilitate indexing various types of documents easily.+We'll use this tool for the indexing examples below.++You'll need a command shell to run some of the following examples, rooted in the Solr install directory; the shell from where you launched Solr works just fine.++NOTE: Currently the `bin/post` tool does not have a comparable Windows script, but the underlying Java program invoked is available.+We'll show examples below for Windows, but you can also see the <<post-tool.adoc#post-tool-windows-support,Windows section>> of the Post Tool documentation for more details.++The data we will index is in the `example/exampledocs` directory.+The documents are in a mix of document formats (JSON, CSV, etc.), and fortunately we can index them all at once:++.Linux/Mac+[source,subs="verbatim,attributes+"]+----+solr-{solr-docs-version}.0:$ bin/post -c techproducts example/exampledocs/*+----++.Windows+[source,subs="verbatim,attributes+"]+----+C:\solr-{solr-docs-version}.0> java -jar -Dc=techproducts -Dauto example\exampledocs\post.jar example\exampledocs\*+----++You should see output similar to the following:++[source,subs="verbatim,attributes+"]+----+SimplePostTool version 5.0.0+Posting files to [base] url http://localhost:8983/solr/techproducts/update...+Entering auto mode. File endings considered are xml,json,jsonl,csv,pdf,doc,docx,ppt,pptx,xls,xlsx,odt,odp,ods,ott,otp,ots,rtf,htm,html,txt,log+POSTing file books.csv (text/csv) to [base]+POSTing file books.json (application/json) to [base]/json/docs+POSTing file gb18030-example.xml (application/xml) to [base]+POSTing file hd.xml (application/xml) to [base]+POSTing file ipod_other.xml (application/xml) to [base]+POSTing file ipod_video.xml (application/xml) to [base]+POSTing file manufacturers.xml (application/xml) to [base]+POSTing file mem.xml (application/xml) to [base]+POSTing file money.xml (application/xml) to [base]+POSTing file monitor.xml (application/xml) to [base]+POSTing file monitor2.xml (application/xml) to [base]+POSTing file more_books.jsonl (application/json) to [base]/json/docs+POSTing file mp500.xml (application/xml) to [base]+POSTing file post.jar (application/octet-stream) to [base]/extract+POSTing file sample.html (text/html) to [base]/extract+POSTing file sd500.xml (application/xml) to [base]+POSTing file solr-word.pdf (application/pdf) to [base]/extract+POSTing file solr.xml (application/xml) to [base]+POSTing file test_utf8.sh (application/octet-stream) to [base]/extract+POSTing file utf8-example.xml (application/xml) to [base]+POSTing file vidcard.xml (application/xml) to [base]+21 files indexed.+COMMITting Solr index changes to http://localhost:8983/solr/techproducts/update...+Time spent: 0:00:00.822+----++Congratulations again! You have data in your Solr!++Now we're ready to start searching.++== Basic Searching++Solr can be queried via REST clients, curl, wget, Chrome POSTMAN, etc., as well as via native clients available for many programming languages.++The Solr Admin UI includes a query builder interface via the Query tab for the `techproducts` collection (at http://localhost:8983/solr/#/techproducts/query).+If you click the btn:[Execute Query] button without changing anything in the form, you'll get 10 documents in JSON format:++.Query Screen+image::images/solr-tutorial/tutorial-query-screen.png[Solr Quick Start: techproducts Query screen with results]++The URL sent by the Admin UI to Solr is shown in light grey near the top right of the above screenshot.+If you click on it, your browser will show you the raw response.++To use curl, give the same URL shown in your browser in quotes on the command line:++`curl "http://localhost:8983/solr/techproducts/select?indent=on&q=\*:*"`++What's happening here is that we are using Solr's query parameter (`q`) with a special syntax that requests all documents in the index (`\*:*`).+All of the documents are not returned to us, however, because of the default for a parameter called `rows`, which you can see in the form is `10`.+You can change the parameter in the UI or in the defaults if you wish.++Solr has very powerful search options, and this tutorial won't be able to cover all of them.+But we can cover some of the most common types of queries.++=== Search for a Single Term++To search for a term, enter it as the `q` parameter value in the Solr Admin UI Query screen, replacing `\*:*` with the term you want to find.++Enter "foundation" and hit btn:[Execute Query] again.++If you prefer curl, enter something like this:++`curl "http://localhost:8983/solr/techproducts/select?q=foundation"`++You'll see something like this:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":8,+ "params":{+ "q":"foundation"}},+ "response":{"numFound":4,"start":0,"maxScore":2.7879646,"docs":[+ {+ "id":"0553293354",+ "cat":["book"],+ "name":"Foundation",+ "price":7.99,+ "price_c":"7.99,USD",+ "inStock":true,+ "author":"Isaac Asimov",+ "author_s":"Isaac Asimov",+ "series_t":"Foundation Novels",+ "sequence_i":1,+ "genre_s":"scifi",+ "_version_":1574100232473411586,+ "price_c____l_ns":799}]+}}++The response indicates that there are 4 hits (`"numFound":4`).+We've only included one document the above sample output, but since 4 hits is lower than the `rows` parameter default of 10 to be returned, you should see all 4 of them.++Note the `responseHeader` before the documents.+This header will include the parameters you have set for the search.+By default it shows only the parameters _you_ have set for this query, which in this case is only your query term.++The documents we got back include all the fields for each document that were indexed.+This is, again, default behavior.+If you want to restrict the fields in the response, you can use the `fl` parameter, which takes a comma-separated list of field names.+This is one of the available fields on the query form in the Admin UI.++Put "id" (without quotes) in the "fl" box and hit btn:[Execute Query] again. Or, to specify it with curl:++`curl "http://localhost:8983/solr/techproducts/select?q=foundation&fl=id"`++You should only see the IDs of the matching records returned.++=== Field Searches++All Solr queries look for documents using some field.+Often you want to query across multiple fields at the same time, and this is what we've done so far with the "foundation" query.+This is possible with the use of copy fields, which are set up already with this set of configurations.+We'll cover copy fields a little bit more in <<tutorial-films.adoc#,Exercise 2>>.++Sometimes, though, you want to limit your query to a single field.+This can make your queries more efficient and the results more relevant for users.++Much of the data in our small sample data set is related to products.+Let's say we want to find all the "electronics" products in the index.+In the Query screen, enter "electronics" (without quotes) in the `q` box and hit btn:[Execute Query].+You should get 14 results, such as:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":6,+ "params":{+ "q":"electronics"}},+ "response":{"numFound":14,"start":0,"maxScore":1.5579545,"docs":[+ {+ "id":"IW-02",+ "name":"iPod & iPod Mini USB 2.0 Cable",+ "manu":"Belkin",+ "manu_id_s":"belkin",+ "cat":["electronics",+ "connector"],+ "features":["car power adapter for iPod, white"],+ "weight":2.0,+ "price":11.5,+ "price_c":"11.50,USD",+ "popularity":1,+ "inStock":false,+ "store":"37.7752,-122.4232",+ "manufacturedate_dt":"2006-02-14T23:55:59Z",+ "_version_":1574100232554151936,+ "price_c____l_ns":1150}]+}}++This search finds all documents that contain the term "electronics" anywhere in the indexed fields.+However, we can see from the above there is a `cat` field (for "category").+If we limit our search for only documents with the category "electronics", the results will be more precise for our users.++Update your query in the `q` field of the Admin UI so it's `cat:electronics`. Now you get 12 results:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":6,+ "params":{+ "q":"cat:electronics"}},+ "response":{"numFound":12,"start":0,"maxScore":0.9614112,"docs":[+ {+ "id":"SP2514N",+ "name":"Samsung SpinPoint P120 SP2514N - hard drive - 250 GB - ATA-133",+ "manu":"Samsung Electronics Co. Ltd.",+ "manu_id_s":"samsung",+ "cat":["electronics",+ "hard drive"],+ "features":["7200RPM, 8MB cache, IDE Ultra ATA-133",+ "NoiseGuard, SilentSeek technology, Fluid Dynamic Bearing (FDB) motor"],+ "price":92.0,+ "price_c":"92.0,USD",+ "popularity":6,+ "inStock":true,+ "manufacturedate_dt":"2006-02-13T15:26:37Z",+ "store":"35.0752,-97.032",+ "_version_":1574100232511160320,+ "price_c____l_ns":9200}]+ }}++Using curl, this query would look like this:++`curl "http://localhost:8983/solr/techproducts/select?q=cat:electronics"`++=== Phrase Search++To search for a multi-term phrase, enclose it in double quotes: `q="multiple terms here"`.+For example, search for "CAS latency" by entering that phrase in quotes to the `q` box in the Admin UI.++If you're following along with curl, note that the space between terms must be converted to "+" in a URL, as so:++`curl "http://localhost:8983/solr/techproducts/select?q=\"CAS+latency\""`++We get 2 results:++[source,json]+{+ "responseHeader":{+ "zkConnected":true,+ "status":0,+ "QTime":7,+ "params":{+ "q":"\"CAS latency\""}},+ "response":{"numFound":2,"start":0,"maxScore":5.937691,"docs":[+ {+ "id":"VDBDB1A16",+ "name":"A-DATA V-Series 1GB 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) System Memory - OEM",+ "manu":"A-DATA Technology Inc.",+ "manu_id_s":"corsair",+ "cat":["electronics",+ "memory"],+ "features":["CAS latency 3, 2.7v"],+ "popularity":0,+ "inStock":true,+ "store":"45.18414,-93.88141",+ "manufacturedate_dt":"2006-02-13T15:26:37Z",+ "payloads":"electronics|0.9 memory|0.1",+ "_version_":1574100232590852096},+ {+ "id":"TWINX2048-3200PRO",+ "name":"CORSAIR XMS 2GB (2 x 1GB) 184-Pin DDR SDRAM Unbuffered DDR 400 (PC 3200) Dual Channel Kit System Memory - Retail",+ "manu":"Corsair Microsystems Inc.",+ "manu_id_s":"corsair",+ "cat":["electronics",+ "memory"],+ "features":["CAS latency 2, 2-3-3-6 timing, 2.75v, unbuffered, heat-spreader"],+ "price":185.0,+ "price_c":"185.00,USD",+ "popularity":5,+ "inStock":true,+ "store":"37.7752,-122.4232",+ "manufacturedate_dt":"2006-02-13T15:26:37Z",+ "payloads":"electronics|6.0 memory|3.0",+ "_version_":1574100232584560640,+ "price_c____l_ns":18500}]+ }}++=== Combining Searches++By default, when you search for multiple terms and/or phrases in a single query, Solr will only require that one of them is present in order for a document to match.+Documents containing more terms will be sorted higher in the results list.++You can require that a term or phrase is present by prefixing it with a `+` (plus); conversely, to disallow the presence of a term or phrase, prefix it with a `-` (minus).++To find documents that contain both terms "electronics" and "music", enter `+electronics +music` in the `q` box in the Admin UI Query tab.++If you're using curl, you must encode the `+` character because it has a reserved purpose in URLs (encoding the space character).+The encoding for `+` is `%2B` as in:++`curl "http://localhost:8983/solr/techproducts/select?q=%2Belectronics%20%2Bmusic"`++You should only get a single result.++To search for documents that contain the term "electronics" but *don't* contain the term "music", enter `+electronics -music` in the `q` box in the Admin UI.+For curl, again, URL encode `+` as `%2B` as in:++`curl "http://localhost:8983/solr/techproducts/select?q=%2Belectronics+-music"`++This time you get 13 results.++=== More Information on Searching++We have only scratched the surface of the search options available in Solr.+For more Solr search options, see the <<query-guide.adoc#,Query Guide>>.++== Exercise 1 Wrap Up++At this point, you've seen how Solr can index data and have done some basic queries.+You can choose now to continue to the next example which will introduce more Solr concepts, such as faceting results and managing your schema, or you can strike out on your own.++If you decide not to continue with this tutorial, the data we've indexed so far is likely of little value to you.+You can delete your installation and start over, or you can use the `bin/solr` script we started out with to delete this collection:++`bin/solr delete -c techproducts`++And then create a new collection:++`bin/solr create -c <yourCollection> -s 2 -rf 2`++To stop both of the Solr nodes we started, issue the command:++`bin/solr stop -all`++For more information on start/stop and collection options with `bin/solr`, see <<solr-control-script-reference.adoc#,Solr Control Script Reference>>.diff --git a/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc b/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adocdeleted file mode 100644index dc603f7a58c..00000000000--- a/solr/solr-ref-guide/src/understanding-analyzers-tokenizers-and-filters.adoc+++ /dev/null@@ -1,54 +0,0 @@-= Understanding Analyzers, Tokenizers, and Filters-:page-children: analyzers, about-tokenizers, about-filters, tokenizers, filter-descriptions, charfilterfactories, language-analysis, phonetic-matching, running-your-analyzer-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The following sections describe how Solr breaks down and works with textual data. There are three main concepts to understand: analyzers, tokenizers, and filters.--* <<analyzers.adoc#,Field analyzers>> are used both during ingestion, when a document is indexed, and at query time. An analyzer examines the text of fields and generates a token stream. Analyzers may be a single class or they may be composed of a series of tokenizer and filter classes.-* <<about-tokenizers.adoc#,Tokenizers>> break field data into lexical units, or _tokens_.-* <<about-filters.adoc#,Filters>> examine a stream of tokens and keep them, transform or discard them, or create new ones. Tokenizers and filters may be combined to form pipelines, or _chains_, where the output of one is input to the next. Such a sequence of tokenizers and filters is called an _analyzer_ and the resulting output of an analyzer is used to match query results or build indices.--== Using Analyzers, Tokenizers, and Filters--Although the analysis process is used for both indexing and querying, the same analysis process need not be used for both operations. For indexing, you often want to simplify, or normalize, words. For example, setting all letters to lowercase, eliminating punctuation and accents, mapping words to their stems, and so on. Doing so can increase recall because, for example, "ram", "Ram" and "RAM" would all match a query for "ram". To increase query-time precision, a filter could be employed to narrow the matches by, for example, ignoring all-cap acronyms if you're interested in male sheep, but not Random Access Memory.--The tokens output by the analysis process define the values, or _terms_, of that field and are used either to build an index of those terms when a new document is added, or to identify which documents contain the terms you are querying for.--=== For More Information--These sections will show you how to configure field analyzers and also serves as a reference for the details of configuring each of the available tokenizer and filter classes. It also serves as a guide so that you can configure your own analysis classes if you have special needs that cannot be met with the included filters or tokenizers.--*For Analyzers, see:*--* <<analyzers.adoc#,Analyzers>>: Detailed conceptual information about Solr analyzers.-* <<running-your-analyzer.adoc#,Running Your Analyzer>>: Detailed information about testing and running your Solr analyzer.--*For Tokenizers, see:*--* <<about-tokenizers.adoc#,About Tokenizers>>: Detailed conceptual information about Solr tokenizers.-* <<tokenizers.adoc#,Tokenizers>>: Information about configuring tokenizers, and about the tokenizer factory classes included in this distribution of Solr.--*For Filters, see:*--* <<about-filters.adoc#,About Filters>>: Detailed conceptual information about Solr filters.-* <<filter-descriptions.adoc#,Filter Descriptions>>: Information about configuring filters, and about the filter factory classes included in this distribution of Solr.-* <<charfilterfactories.adoc#,CharFilterFactories>>: Information about filters for pre-processing input characters.--*To find out how to use Tokenizers and Filters with various languages, see:*--* <<language-analysis.adoc#,Language Analysis>>: Information about tokenizers and filters for character set conversion or for use with specific languages.diff --git a/solr/solr-ref-guide/src/update-request-processors.adoc b/solr/solr-ref-guide/src/update-request-processors.adocindex c2ada3088d4..a613435c59d 100644--- a/solr/solr-ref-guide/src/update-request-processors.adoc+++ b/solr/solr-ref-guide/src/update-request-processors.adoc@@ -18,22 +18,35 @@Every update request received by Solr is run through a chain of plugins known as Update Request Processors, or _URPs_.-This can be useful, for example, to add a field to the document being indexed; to change the value of a particular field; or to drop an update if the incoming document doesn't fulfill certain criteria. In fact, a surprisingly large number of features in Solr are implemented as Update Processors and therefore it is necessary to understand how such plugins work and where are they configured.+This can be useful, for example, to add a field to the document being indexed; to change the value of a particular field; or to drop an update if the incoming document doesn't fulfill certain criteria.+In fact, a surprisingly large number of features in Solr are implemented as Update Processors and therefore it is necessary to understand how such plugins work and where are they configured.== URP Anatomy and Lifecycle-An Update Request Processor is created as part of a {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessorChain.html[chain] of one or more update processors. Solr creates a default update request processor chain comprising of a few update request processors which enable essential Solr features. This default chain is used to process every update request unless a user chooses to configure and specify a different custom update request processor chain.+An Update Request Processor is created as part of a {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessorChain.html[chain] of one or more update processors.+Solr creates a default update request processor chain comprising of a few update request processors which enable essential Solr features.+This default chain is used to process every update request unless a user chooses to configure and specify a different custom update request processor chain.-The easiest way to describe an Update Request Processor is to look at the Javadocs of the abstract class {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessor.html[UpdateRequestProcessor]. Every UpdateRequestProcessor must have a corresponding factory class which extends {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessorFactory.html[UpdateRequestProcessorFactory]. This factory class is used by Solr to create a new instance of this plugin. Such a design provides two benefits:+The easiest way to describe an Update Request Processor is to look at the Javadocs of the abstract class {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessor.html[UpdateRequestProcessor].+Every UpdateRequestProcessor must have a corresponding factory class which extends {solr-javadocs}/core/org/apache/solr/update/processor/UpdateRequestProcessorFactory.html[UpdateRequestProcessorFactory].+This factory class is used by Solr to create a new instance of this plugin.+Such a design provides two benefits:. An update request processor need not be thread safe because it is used by one and only one request thread and destroyed once the request is complete.-. The factory class can accept configuration parameters and maintain any state that may be required between requests. The factory class must be thread-safe.+. The factory class can accept configuration parameters and maintain any state that may be required between requests.+The factory class must be thread-safe.-Every update request processor chain is constructed during loading of a Solr core and cached until the core is unloaded. Each `UpdateRequestProcessorFactory` specified in the chain is also instantiated and initialized with configuration that may have been specified in `solrconfig.xml`.+Every update request processor chain is constructed during loading of a Solr core and cached until the core is unloaded.+Each `UpdateRequestProcessorFactory` specified in the chain is also instantiated and initialized with configuration that may have been specified in `solrconfig.xml`.-When an update request is received by Solr, it looks up the update chain to be used for this request. A new instance of each UpdateRequestProcessor specified in the chain is created using the corresponding factory. The update request is parsed into corresponding {solr-javadocs}/core/org/apache/solr/update/UpdateCommand.html[UpdateCommand] objects which are run through the chain. Each UpdateRequestProcessor instance is responsible for invoking the next plugin in the chain. It can choose to short circuit the chain by not invoking the next processor and even abort further processing by throwing an exception.+When an update request is received by Solr, it looks up the update chain to be used for this request.+A new instance of each UpdateRequestProcessor specified in the chain is created using the corresponding factory.+The update request is parsed into corresponding {solr-javadocs}/core/org/apache/solr/update/UpdateCommand.html[UpdateCommand] objects which are run through the chain.+Each UpdateRequestProcessor instance is responsible for invoking the next plugin in the chain.+It can choose to short circuit the chain by not invoking the next processor and even abort further processing by throwing an exception.-NOTE: A single update request may contain a batch of multiple new documents or deletes and therefore the corresponding processXXX methods of an UpdateRequestProcessor will be invoked multiple times for every individual update. However, it is guaranteed that a single thread will serially invoke these methods.+NOTE: A single update request may contain a batch of multiple new documents or deletes and therefore the corresponding processXXX methods of an UpdateRequestProcessor will be invoked multiple times for every individual update.+However, it is guaranteed that a single thread will serially invoke these methods.== Update Request Processor Configuration@@ -43,13 +56,16 @@ However, before we understand how to configure update processor chains, we must=== Default Update Request Processor Chain-In case no update processor chains are configured in `solrconfig.xml`, Solr will automatically create a default update processor chain which will be used for all update requests. This default update processor chain consists of the following processors (in order):+In case no update processor chains are configured in `solrconfig.xml`, Solr will automatically create a default update processor chain which will be used for all update requests.+This default update processor chain consists of the following processors (in order):1. `LogUpdateProcessorFactory` - Tracks the commands processed during this request and logs them-2. `DistributedUpdateProcessorFactory` - Responsible for distributing update requests to the right node e.g., routing requests to the leader of the right shard and distributing updates from the leader to each replica. This processor is activated only in SolrCloud mode.+2. `DistributedUpdateProcessorFactory` - Responsible for distributing update requests to the right node e.g., routing requests to the leader of the right shard and distributing updates from the leader to each replica.+This processor is activated only in SolrCloud mode.3. `RunUpdateProcessorFactory` - Executes the update using internal Solr APIs.-Each of these perform an essential function and as such any custom chain usually contain all of these processors. The `RunUpdateProcessorFactory` is usually the last update processor in any custom chain.+Each of these perform an essential function and as such any custom chain usually contain all of these processors.+The `RunUpdateProcessorFactory` is usually the last update processor in any custom chain.=== Custom Update Request Processor Chain@@ -71,12 +87,17 @@ The following example demonstrates how a custom chain can be configured inside `</updateRequestProcessorChain>-----In the above example, a new update processor chain named "dedupe" is created with `SignatureUpdateProcessorFactory`, `LogUpdateProcessorFactory` and `RunUpdateProcessorFactory` in the chain. The `SignatureUpdateProcessorFactory` is further configured with different parameters such as "signatureField", "overwriteDupes", etc. This chain is an example of how Solr can be configured to perform de-duplication of documents by calculating a signature using the value of name, features, cat fields which is then used as the "id" field. As you may have noticed, this chain does not specify the `DistributedUpdateProcessorFactory`. Because this processor is critical for Solr to operate properly, Solr will automatically insert `DistributedUpdateProcessorFactory` in any chain that does not include it just prior to the `RunUpdateProcessorFactory`.+In the above example, a new update processor chain named "dedupe" is created with `SignatureUpdateProcessorFactory`, `LogUpdateProcessorFactory` and `RunUpdateProcessorFactory` in the chain.+The `SignatureUpdateProcessorFactory` is further configured with different parameters such as "signatureField", "overwriteDupes", etc.+This chain is an example of how Solr can be configured to perform de-duplication of documents by calculating a signature using the value of name, features, cat fields which is then used as the "id" field.+As you may have noticed, this chain does not specify the `DistributedUpdateProcessorFactory`.+Because this processor is critical for Solr to operate properly, Solr will automatically insert `DistributedUpdateProcessorFactory` in any chain that does not include it just prior to the `RunUpdateProcessorFactory`..RunUpdateProcessorFactory[WARNING]====-Do not forget to add `RunUpdateProcessorFactory` at the end of any chains you define in `solrconfig.xml`. Otherwise update requests processed by that chain will not actually affect the indexed data.+Do not forget to add `RunUpdateProcessorFactory` at the end of any chains you define in `solrconfig.xml`.+Otherwise update requests processed by that chain will not actually affect the indexed data.======= Configuring Individual Processors as Top-Level Plugins@@ -96,7 +117,8 @@ Update request processors can also be configured independent of a chain in `solr<updateProcessor class="solr.RemoveBlankFieldUpdateProcessorFactory" name="remove_blanks"/>-----In this case, an instance of `SignatureUpdateProcessorFactory` is configured with the name "signature" and a `RemoveBlankFieldUpdateProcessorFactory` is defined with the name "remove_blanks". Once the above has been specified in `solrconfig.xml`, we can be refer to them in update request processor chains in `solrconfig.xml` as follows:+In this case, an instance of `SignatureUpdateProcessorFactory` is configured with the name "signature" and a `RemoveBlankFieldUpdateProcessorFactory` is defined with the name "remove_blanks".+Once the above has been specified in `solrconfig.xml`, we can be refer to them in update request processor chains in `solrconfig.xml` as follows:.updateRequestProcessorChain Configuration[source,xml]@@ -108,24 +130,43 @@ In this case, an instance of `SignatureUpdateProcessorFactory` is configured wit== Update Processors in SolrCloud-In a single node, stand-alone Solr, each update is run through all the update processors in a chain exactly once. But the behavior of update request processors in SolrCloud deserves special consideration.+In a user-managed cluster or a single-node installation, each update is run through all the update processors in a chain exactly once.+In SolrCloud, however, the behavior of update request processors deserves special consideration.-A critical SolrCloud functionality is the routing and distributing of requests. For update requests this routing is implemented by the `DistributedUpdateRequestProcessor`, and this processor is given a special status by Solr due to its important function.+A critical SolrCloud functionality is the routing and distributing of requests.+For update requests this routing is implemented by the `DistributedUpdateRequestProcessor`, and this processor is given a special status by Solr due to its important function.-In SolrCloud mode, all processors in the chain _before_ the `DistributedUpdateProcessor` are run on the first node that receives an update from the client, regardless of this node's status as a leader or replica. The `DistributedUpdateProcessor` then forwards the update to the appropriate shard leader for the update (or to multiple leaders in the event of an update that affects multiple documents, such as a delete by query or commit). The shard leader uses a transaction log to apply <<updating-parts-of-documents.adoc#,Atomic Updates & Optimistic Concurrency>> and then forwards the update to all of the shard replicas. The leader and each replica run all of the processors in the chain that are listed _after_ the `DistributedUpdateProcessor`.+In SolrCloud clusters, all processors in the chain _before_ the `DistributedUpdateProcessor` are run on the first node that receives an update from the client, regardless of this node's status as a leader or replica.+The `DistributedUpdateProcessor` then forwards the update to the appropriate shard leader for the update (or to multiple leaders in the event of an update that affects multiple documents, such as a delete by query or commit).+The shard leader uses a transaction log to apply <<partial-document-updates.adoc#,Atomic Updates & Optimistic Concurrency>> and then forwards the update to all of the shard replicas.+The leader and each replica run all of the processors in the chain that are listed _after_ the `DistributedUpdateProcessor`.-For example, consider the "dedupe" chain which we saw in a section above. Assume that a 3-node SolrCloud cluster exists where node A hosts the leader of shard1, node B hosts the leader of shard2 and node C hosts the replica of shard2. Assume that an update request is sent to node A which forwards the update to node B (because the update belongs to shard2) which then distributes the update to its replica node C. Let's see what happens at each node:+For example, consider the "dedupe" chain which we saw in a section above.+Assume that a 3-node SolrCloud cluster exists where node A hosts the leader of shard1, node B hosts the leader of shard2 and node C hosts a NRT-type replica of shard2.+Assume that an update request is sent to node A which forwards the update to node B (because the update belongs to shard2) which then distributes the update to its replica node C.+Let's see what happens at each node:-* *Node A*: Runs the update through the `SignatureUpdateProcessor` (which computes the signature and puts it in the "id" field), then `LogUpdateProcessor` and then `DistributedUpdateProcessor`. This processor determines that the update actually belongs to node B and is forwarded to node B. The update is not processed further. This is required because the next processor, `RunUpdateProcessor`, will execute the update against the local shard1 index which would lead to duplicate data on shard1 and shard2.-* *Node B*: Receives the update and sees that it was forwarded by another node. The update is directly sent to `DistributedUpdateProcessor` because it has already been through the `SignatureUpdateProcessor` on node A and doing the same signature computation again would be redundant. The `DistributedUpdateProcessor` determines that the update indeed belongs to this node, distributes it to its replica on Node C and then forwards the update further in the chain to `RunUpdateProcessor`.-* *Node C*: Receives the update and sees that it was distributed by its leader. The update is directly sent to `DistributedUpdateProcessor` which performs some consistency checks and forwards the update further in the chain to `RunUpdateProcessor`.+* *Node A*: Runs the update through the `SignatureUpdateProcessor` (which computes the signature and puts it in the "id" field), then `LogUpdateProcessor` and then `DistributedUpdateProcessor`.+This processor determines that the update actually belongs to node B and is forwarded to node B.+The update is not processed further.+This is required because the next processor, `RunUpdateProcessor`, will execute the update against the local shard1 index which would lead to duplicate data on shard1 and shard2.+* *Node B*: Receives the update and sees that it was forwarded by another node.+The update is directly sent to `DistributedUpdateProcessor` because it has already been through the `SignatureUpdateProcessor` on node A and doing the same signature computation again would be redundant.+The `DistributedUpdateProcessor` determines that the update indeed belongs to this node, distributes it to its replica on Node C and then forwards the update further in the chain to `RunUpdateProcessor`.+* *Node C*: Receives the update and sees that it was distributed by its leader.+The update is directly sent to `DistributedUpdateProcessor` which performs some consistency checks and forwards the update further in the chain to `RunUpdateProcessor`.In summary:-. All processors before `DistributedUpdateProcessor` are only run on the first node that receives an update request whether it be a forwarding node (e.g., node A in the above example) or a leader (e.g., node B). We call these "pre-processors" or just "processors".-. All processors after `DistributedUpdateProcessor` run only on the leader and the replica nodes. They are not executed on forwarding nodes. Such processors are called "post-processors".+. All processors before `DistributedUpdateProcessor` are only run on the first node that receives an update request whether it be a forwarding node (e.g., node A in the above example) or a leader (e.g., node B).+We call these "pre-processors" or just "processors".+. All processors after `DistributedUpdateProcessor` run only on the leader and the replica nodes.+They are not executed on forwarding nodes.+Such processors are called "post-processors".-In the previous section, we saw that the `updateRequestProcessorChain` was configured with `processor="remove_blanks, signature"`. This means that such processors are of the #1 kind and are run only on the forwarding nodes. Similarly, we can configure them as the #2 kind by specifying with the attribute "post-processor" as follows:+In the previous section, we saw that the `updateRequestProcessorChain` was configured with `processor="remove_blanks, signature"`.+This means that such processors are of the #1 kind and are run only on the forwarding nodes.+Similarly, we can configure them as the #2 kind by specifying with the attribute "post-processor" as follows:.post-processor Configuration[source,xml]@@ -135,26 +176,33 @@ In the previous section, we saw that the `updateRequestProcessorChain` was confi</updateProcessorChain>-----However executing a processor only on the forwarding nodes is a great way of distributing an expensive computation such as de-duplication across a SolrCloud cluster by sending requests randomly via a load balancer. Otherwise the expensive computation is repeated on both the leader and replica nodes.+However executing a processor only on the forwarding nodes is a great way of distributing an expensive computation such as de-duplication across a SolrCloud cluster by sending requests randomly via a load balancer.+Otherwise the expensive computation is repeated on both the leader and replica nodes..Custom update chain post-processors may never be invoked on a recovering replica[WARNING]====-While a replica is in <<solrcloud-recoveries-and-write-tolerance.adoc#,recovery>>, inbound update requests are buffered to the transaction log. After recovery has completed successfully, those buffered update requests are replayed. As of this writing, however, custom update chain post-processors are never invoked for buffered update requests. See https://issues.apache.org/jira/browse/SOLR-8030[SOLR-8030]. To work around this problem until SOLR-8030 has been fixed, *avoid specifying post-processors in custom update chains*.+While a replica is in <<solrcloud-recoveries-and-write-tolerance.adoc#,recovery>>, inbound update requests are buffered to the transaction log.+After recovery has completed successfully, those buffered update requests are replayed.+As of this writing, however, custom update chain post-processors are never invoked for buffered update requests.+See https://issues.apache.org/jira/browse/SOLR-8030[SOLR-8030].+To work around this problem until SOLR-8030 has been fixed, *avoid specifying post-processors in custom update chains*.======= Atomic Update Processor FactoryIf the `AtomicUpdateProcessorFactory` is in the update chain before the `DistributedUpdateProcessor`, the incoming document to the chain will be a partial document.-Because `DistributedUpdateProcessor` is responsible for processing <<updating-parts-of-documents.adoc#,Atomic Updates>> into full documents on the leader node, this means that pre-processors which are executed only on the forwarding nodes can only operate on the partial document. If you have a processor which must process a full document then the only choice is to specify it as a post-processor.+Because `DistributedUpdateProcessor` is responsible for processing <<partial-document-updates.adoc#,Atomic Updates>> into full documents on the leader node, this means that pre-processors which are executed only on the forwarding nodes can only operate on the partial document.+If you have a processor which must process a full document then the only choice is to specify it as a post-processor.== Using Custom Chains=== update.chain Request Parameter-The `update.chain` parameter can be used in any update request to choose a custom chain which has been configured in `solrconfig.xml`. For example, in order to choose the "dedupe" chain described in a previous section, one can issue the following request:+The `update.chain` parameter can be used in any update request to choose a custom chain which has been configured in `solrconfig.xml`.+For example, in order to choose the "dedupe" chain described in a previous section, one can issue the following request:.Using update.chain[source,bash]@@ -179,7 +227,9 @@ The above should dedupe the two identical documents and index only one of them.=== Processor & Post-Processor Request Parameters-We can dynamically construct a custom update request processor chain using the `processor` and `post-processor` request parameters. Multiple processors can be specified as a comma-separated value for these two parameters. For example:+We can dynamically construct a custom update request processor chain using the `processor` and `post-processor` request parameters.+Multiple processors can be specified as a comma-separated value for these two parameters.+For example:.Executing processors configured in solrconfig.xml as (pre)-processors[source,bash]@@ -225,7 +275,7 @@ In the first example, Solr will dynamically create a chain which has "signature"We can also specify a custom chain to be used by default for all requests sent to specific update handlers instead of specifying the names in request parameters for each request.-This can be done by adding either "update.chain" or "processor" and "post-processor" as default parameter for a given path which can be done either via <<initparams-in-solrconfig.adoc#,initParams>> or by adding them in a <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,"defaults" section>> which is supported by all request handlers.+This can be done by adding either "update.chain" or "processor" and "post-processor" as default parameter for a given path which can be done either via <<initparams.adoc#,initParams>> or by adding them in a <<requesthandlers-searchcomponents.adoc#,"defaults" section>> which is supported by all request handlers.The following is an `initParam` defined in the <<schemaless-mode.adoc#,schemaless configuration>> which applies a custom update chain to all request handlers starting with "/update/".@@ -253,15 +303,19 @@ Alternately, one can achieve a similar effect using the "defaults" as shown in t== Update Request Processor Factories-What follows are brief descriptions of the currently available update request processors. An `UpdateRequestProcessorFactory` can be integrated into an update chain in `solrconfig.xml` as necessary. You are strongly urged to examine the Javadocs for these classes; these descriptions are abridged snippets taken for the most part from the Javadocs.+What follows are brief descriptions of the currently available update request processors.+An `UpdateRequestProcessorFactory` can be integrated into an update chain in `solrconfig.xml` as necessary.+You are strongly urged to examine the Javadocs for these classes; these descriptions are abridged snippets taken for the most part from the Javadocs.=== General Use UpdateProcessorFactories{solr-javadocs}/core/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactory.html[AddSchemaFieldsUpdateProcessorFactory]:: This processor will dynamically add fields to the schema if an input document contains one or more fields that don't match any field or dynamic field in the schema.-{solr-javadocs}/core/org/apache/solr/update/processor/AtomicUpdateProcessorFactory.html[AtomicUpdateProcessorFactory]:: This processor will convert conventional field-value documents to atomic update documents. This processor can be used at runtime (without defining it in `solrconfig.xml`), see the section <<atomicupdateprocessorfactory>> below.+{solr-javadocs}/core/org/apache/solr/update/processor/AtomicUpdateProcessorFactory.html[AtomicUpdateProcessorFactory]:: This processor will convert conventional field-value documents to atomic update documents.+This processor can be used at runtime (without defining it in `solrconfig.xml`), see the section <<atomicupdateprocessorfactory>> below.-{solr-javadocs}/core/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.html[ClassificationUpdateProcessorFactory]:: This processor uses Lucene's classification module to provide simple document classification. See https://cwiki.apache.org/confluence/display/solr/SolrClassification for more details on how to use this processor.+{solr-javadocs}/core/org/apache/solr/update/processor/ClassificationUpdateProcessorFactory.html[ClassificationUpdateProcessorFactory]:: This processor uses Lucene's classification module to provide simple document classification.+See https://cwiki.apache.org/confluence/display/solr/SolrClassification for more details on how to use this processor.{solr-javadocs}/core/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.html[CloneFieldUpdateProcessorFactory]:: Clones the values found in any matching _source_ field into the configured _dest_ field.@@ -275,25 +329,31 @@ What follows are brief descriptions of the currently available update request pr{solr-javadocs}/core/org/apache/solr/update/processor/IgnoreCommitOptimizeUpdateProcessorFactory.html[IgnoreCommitOptimizeUpdateProcessorFactory]:: Allows you to ignore commit and/or optimize requests from client applications when running in SolrCloud mode, for more information, see: Shards and Indexing Data in SolrCloud-{solr-javadocs}/core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to prevent large documents with size more than `limit` (in KB) from getting indexed. It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.+{solr-javadocs}/core/org/apache/solr/update/processor/IgnoreLargeDocumentProcessorFactory.html[IgnoreLargeDocumentProcessorFactory]:: Allows you to prevent large documents with size more than `limit` (in KB) from getting indexed.+It can help to prevent unexpected problems on indexing as well as on recovering because of very large documents.{solr-javadocs}/core/org/apache/solr/update/processor/RegexpBoostProcessorFactory.html[RegexpBoostProcessorFactory]:: A processor which will match content of "inputField" against regular expressions found in "boostFilename", and if it matches will return the corresponding boost value from the file and output this to "boostField" as a double value.-{solr-javadocs}/core/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.html[SignatureUpdateProcessorFactory]:: Uses a defined set of fields to generate a hash "signature" for the document. Useful for only indexing one copy of "similar" documents.+{solr-javadocs}/core/org/apache/solr/update/processor/SignatureUpdateProcessorFactory.html[SignatureUpdateProcessorFactory]:: Uses a defined set of fields to generate a hash "signature" for the document.+Useful for only indexing one copy of "similar" documents.-{solr-javadocs}/contrib/scripting/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.html[ScriptUpdateProcessorFactory]:: An processor that enables the use of update processors implemented as scripts. Learn more at the <<script-update-processor.adoc#,script update processor>> page.+{solr-javadocs}/contrib/scripting/org/apache/solr/scripting/update/ScriptUpdateProcessorFactory.html[ScriptUpdateProcessorFactory]:: An processor that enables the use of update processors implemented as scripts.+Learn more at the <<script-update-processor.adoc#,script update processor>> page.-{solr-javadocs}/core/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.html[TemplateUpdateProcessorFactory]:: Allows adding new fields to documents based on a template pattern. This update processor can also be used at runtime (without defining it in `solrconfig.xml`), see the section <<templateupdateprocessorfactory>> below.+{solr-javadocs}/core/org/apache/solr/update/processor/TemplateUpdateProcessorFactory.html[TemplateUpdateProcessorFactory]:: Allows adding new fields to documents based on a template pattern.+This update processor can also be used at runtime (without defining it in `solrconfig.xml`), see the section <<templateupdateprocessorfactory>> below.{solr-javadocs}/core/org/apache/solr/update/processor/TimestampUpdateProcessorFactory.html[TimestampUpdateProcessorFactory]:: An update processor that adds a newly generated date value of "NOW" to any document being added that does not already have a value in the specified field.{solr-javadocs}/core/org/apache/solr/update/processor/URLClassifyProcessorFactory.html[URLClassifyProcessorFactory]:: Update processor which examines a URL and outputs to various other fields with characteristics of that URL, including length, number of path levels, whether it is a top level URL (levels==0), whether it looks like a landing/index page, a canonical representation of the URL (e.g., stripping index.html), the domain and path parts of the URL, etc.-{solr-javadocs}/core/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.html[UUIDUpdateProcessorFactory]:: An update processor that adds a newly generated UUID value to any document being added that does not already have a value in the specified field. This processor can also be used at runtime (without defining it in `solrconfig.xml`), see the section <<uuidupdateprocessorfactory>> below.+{solr-javadocs}/core/org/apache/solr/update/processor/UUIDUpdateProcessorFactory.html[UUIDUpdateProcessorFactory]:: An update processor that adds a newly generated UUID value to any document being added that does not already have a value in the specified field.+This processor can also be used at runtime (without defining it in `solrconfig.xml`), see the section <<uuidupdateprocessorfactory>> below.=== FieldMutatingUpdateProcessorFactory Derived Factories-These factories all provide functionality to _modify_ fields in a document as they're being indexed. When using any of these factories, please consult the {solr-javadocs}/core/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.html[FieldMutatingUpdateProcessorFactory javadocs] for details on the common options they all support for configuring which fields are modified.+These factories all provide functionality to _modify_ fields in a document as they're being indexed.+When using any of these factories, please consult the {solr-javadocs}/core/org/apache/solr/update/processor/FieldMutatingUpdateProcessorFactory.html[FieldMutatingUpdateProcessorFactory javadocs] for details on the common options they all support for configuring which fields are modified.{solr-javadocs}/core/org/apache/solr/update/processor/ConcatFieldUpdateProcessorFactory.html[ConcatFieldUpdateProcessorFactory]:: Concatenates multiple values for fields matching the specified conditions using a configurable delimiter.@@ -341,7 +401,8 @@ These factories all provide functionality to _modify_ fields in a document as th=== Update Processor Factories That Can Be Loaded as Plugins-These processors are included in Solr releases as "contribs", and require additional jars loaded at runtime. See the README files associated with each contrib for details:+These processors are included in Solr releases as "contribs", and require additional jars loaded at runtime.+See the README files associated with each contrib for details:The {solr-javadocs}/contrib/langid/index.html[`langid`] contrib provides::@@ -351,28 +412,36 @@ The {solr-javadocs}/contrib/langid/index.html[`langid`] contrib provides::The {solr-javadocs}/contrib/analysis-extras/index.html[`analysis-extras`] contrib provides::-{solr-javadocs}/contrib/analysis-extras/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.html[OpenNLPExtractNamedEntitiesUpdateProcessorFactory]::: Update document(s) to be indexed with named entities extracted using an OpenNLP NER model. Note that in order to use model files larger than 1MB on SolrCloud, you must either <<setting-up-an-external-zookeeper-ensemble#increasing-the-file-size-limit,configure both ZooKeeper server and clients>> or <<libs.adoc#lib-directives-in-solrconfig,store the model files on the filesystem>> on each node hosting a collection replica.+{solr-javadocs}/contrib/analysis-extras/org/apache/solr/update/processor/OpenNLPExtractNamedEntitiesUpdateProcessorFactory.html[OpenNLPExtractNamedEntitiesUpdateProcessorFactory]::: Update document(s) to be indexed with named entities extracted using an OpenNLP NER model.+Note that in order to use model files larger than 1MB on SolrCloud, you must either <<zookeeper-ensemble#increasing-the-file-size-limit,configure both ZooKeeper server and clients>> or <<libs.adoc#lib-directives-in-solrconfig,store the model files on the filesystem>> on each node hosting a collection replica.=== Update Processor Factories You Should _Not_ Modify or Remove-These are listed for completeness, but are part of the Solr infrastructure, particularly SolrCloud. Other than insuring you do _not_ remove them when modifying the update request handlers (or any copies you make), you will rarely, if ever, need to change these.+These are listed for completeness, but are part of the Solr infrastructure, particularly SolrCloud.+Other than insuring you do _not_ remove them when modifying the update request handlers (or any copies you make), you will rarely, if ever, need to change these.{solr-javadocs}/core/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.html[DistributedUpdateProcessorFactory]:: Used to distribute updates to all necessary nodes.-{solr-javadocs}/core/org/apache/solr/update/processor/NoOpDistributingUpdateProcessorFactory.html[NoOpDistributingUpdateProcessorFactory]::: An alternative No-Op implementation of `DistributingUpdateProcessorFactory` that always returns null. Designed for experts who want to bypass distributed updates and use their own custom update logic.+{solr-javadocs}/core/org/apache/solr/update/processor/NoOpDistributingUpdateProcessorFactory.html[NoOpDistributingUpdateProcessorFactory]::: An alternative No-Op implementation of `DistributingUpdateProcessorFactory` that always returns null.+Designed for experts who want to bypass distributed updates and use their own custom update logic.-{solr-javadocs}/core/org/apache/solr/update/processor/LogUpdateProcessorFactory.html[LogUpdateProcessorFactory]:: A logging processor. This keeps track of all commands that have passed through the chain and prints them on finish().+{solr-javadocs}/core/org/apache/solr/update/processor/LogUpdateProcessorFactory.html[LogUpdateProcessorFactory]:: A logging processor.+This keeps track of all commands that have passed through the chain and prints them on finish().-{solr-javadocs}/core/org/apache/solr/update/processor/RunUpdateProcessorFactory.html[RunUpdateProcessorFactory]:: Executes the update commands using the underlying UpdateHandler. Almost all processor chains should end with an instance of `RunUpdateProcessorFactory` unless the user is explicitly executing the update commands in an alternative custom `UpdateRequestProcessorFactory`.+{solr-javadocs}/core/org/apache/solr/update/processor/RunUpdateProcessorFactory.html[RunUpdateProcessorFactory]:: Executes the update commands using the underlying UpdateHandler.+Almost all processor chains should end with an instance of `RunUpdateProcessorFactory` unless the user is explicitly executing the update commands in an alternative custom `UpdateRequestProcessorFactory`.=== Update Processors That Can Be Used at Runtime-These Update processors do not need any configuration in `solrconfig.xml`. They are automatically initialized when their name is added to the `processor` parameter sent with an update request. Multiple processors can be used by appending multiple processor names separated by commas.+These Update processors do not need any configuration in `solrconfig.xml`. They are automatically initialized when their name is added to the `processor` parameter sent with an update request.+Multiple processors can be used by appending multiple processor names separated by commas.==== AtomicUpdateProcessorFactoryThe `AtomicUpdateProcessorFactory` is used to atomically update documents.-Use the parameter `processor=atomic` to invoke it. Use it to convert your normal `update` operations to atomic update operations. This is particularly useful when you use endpoints such as `/update/csv` or `/update/json/docs` which does not otherwise have a syntax for atomic operations.+Use the parameter `processor=atomic` to invoke it.+Use it to convert your normal `update` operations to atomic update operations.+This is particularly useful when you use endpoints such as `/update/csv` or `/update/json/docs` which does not otherwise have a syntax for atomic operations.For example:@@ -392,7 +461,10 @@ The above parameters convert a normal `update` operation in the following ways:The `TemplateUpdateProcessorFactory` can be used to add new fields to documents based on a template pattern.-Use the parameter `processor=template` to use it. The template parameter `template.field` (multivalued) defines the field to add and the pattern. Templates may contain placeholders which refer to other fields in the document. You can have multiple `Template.field` parameters in a single request.+Use the parameter `processor=template` to use it.+The template parameter `template.field` (multivalued) defines the field to add and the pattern.+Templates may contain placeholders which refer to other fields in the document.+You can have multiple `Template.field` parameters in a single request.For example:@@ -401,13 +473,16 @@ For example:processor=template&template.field=fullName:Mr. {firstName} {lastName}-----The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields. If either of them is missing, that part is replaced with an empty string. If those fields are multi-valued, only the first value is used.+The above example would add a new field to the document called `fullName`. The fields `firstName and` `lastName` are supplied from the document fields.+If either of them is missing, that part is replaced with an empty string.+If those fields are multi-valued, only the first value is used.==== UUIDUpdateProcessorFactoryThe `UUIDUpdateProcessorFactory` is used to add generated UUIDs to documents.-Use the parameter `processor=uuid` to invoke it. You will also need to specify the field where the UUID will be added with the `uuid.fieldName` parameter.+Use the parameter `processor=uuid` to invoke it.+You will also need to specify the field where the UUID will be added with the `uuid.fieldName` parameter.For example:diff --git a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc b/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adocdeleted file mode 100644index a7afaee7d8a..00000000000--- a/solr/solr-ref-guide/src/updatehandlers-in-solrconfig.adoc+++ /dev/null@@ -1,157 +0,0 @@-= UpdateHandlers in SolrConfig-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--The settings in this section are configured in the `<updateHandler>` element in `solrconfig.xml` and may affect the performance of index updates. These settings affect how updates are done internally. `<updateHandler>` configurations do not affect the higher level configuration of <<requesthandlers-and-searchcomponents-in-solrconfig.adoc#,RequestHandlers>> that process client update requests.--[source,xml]------<updateHandler class="solr.DirectUpdateHandler2">- ...-</updateHandler>-------== Commits--Data sent to Solr is not searchable until it has been _committed_ to the index. The reason for this is that in some cases commits can be slow and they should be done in isolation from other possible commit requests to avoid overwriting data. So, it's preferable to provide control over when data is committed. Several options are available to control the timing of commits.--=== commit and softCommit--In Solr, a `commit` is an action which asks Solr to "commit" those changes to the Lucene index files. By default commit actions result in a "hard commit" of all the Lucene index files to stable storage (disk). When a client includes a `commit=true` parameter with an update request, this ensures that all index segments affected by the adds & deletes on an update are written to disk as soon as index updates are completed.--If an additional flag `softCommit=true` is specified, then Solr performs a 'soft commit', meaning that Solr will commit your changes to the Lucene data structures quickly but not guarantee that the Lucene index files are written to stable storage. This is an implementation of Near Real Time storage, a feature that boosts document visibility, since you don't have to wait for background merges and storage (to ZooKeeper, if using <<solrcloud.adoc#,SolrCloud>>) to finish before moving on to something else. A full commit means that, if a server crashes, Solr will know exactly where your data was stored; a soft commit means that the data is stored, but the location information isn't yet stored. The tradeoff is that a soft commit gives you faster visibility because it's not waiting for background merges to finish.--For more information about Near Real Time operations, see <<near-real-time-searching.adoc#,Near Real Time Searching>>.--=== autoCommit--These settings control how often pending updates will be automatically pushed to the index. An alternative to `autoCommit` is to use `commitWithin`, which can be defined when making the update request to Solr (i.e., when pushing documents), or in an update RequestHandler.--`maxDocs`::-The number of updates that have occurred since the last commit.--`maxTime`::-The number of milliseconds since the oldest uncommitted update.--`maxSize`::-The maximum size of the transaction log (tlog) on disk, after which a hard commit is triggered. This is useful when the size of documents is unknown and the intention is to restrict the size of the transaction log to reasonable size. Valid values can be bytes (default with no suffix), kilobytes (if defined with a `k` suffix, as in `25k`), megabytes (`m`) or gigabytes (`g`).--`openSearcher`::-Whether to open a new searcher when performing a commit. If this is `false`, the commit will flush recent index changes to stable storage, but does not cause a new searcher to be opened to make those changes visible. The default is `true`.--If any of the `maxDocs`, `maxTime`, or `maxSize` limits are reached, Solr automatically performs a commit operation. If the `autoCommit` tag is missing, then only explicit commits will update the index. The decision whether to use auto-commit or not depends on the needs of your application.--Determining the best auto-commit settings is a tradeoff between performance and accuracy. Settings that cause frequent updates will improve the accuracy of searches because new content will be searchable more quickly, but performance may suffer because of the frequent updates. Less frequent updates may improve performance but it will take longer for updates to show up in queries.--[source,xml]------<autoCommit>- <maxDocs>10000</maxDocs>- <maxTime>30000</maxTime>- <maxSize>512m</maxSize>- <openSearcher>false</openSearcher>-</autoCommit>-------You can also specify 'soft' autoCommits in the same way that you can specify 'soft' commits, except that instead of using `autoCommit` you set the `autoSoftCommit` tag.--[source,xml]------<autoSoftCommit>- <maxTime>60000</maxTime>-</autoSoftCommit>-------=== commitWithin--The `commitWithin` settings allow forcing document commits to happen in a defined time period. This is used most frequently with <<near-real-time-searching.adoc#,Near Real Time Searching>>, and for that reason the default is to perform a soft commit. This does not, however, replicate new documents to follower servers in a leader/follower environment. If that's a requirement for your implementation, you can force a hard commit by adding a parameter, as in this example:--[source,xml]------<commitWithin>- <softCommit>false</softCommit>-</commitWithin>-------With this configuration, when you call `commitWithin` as part of your update message, it will automatically perform a hard commit every time.--== Event Listeners--The UpdateHandler section is also where update-related event listeners can be configured. These can be triggered to occur after any commit (`event="postCommit"`) or only after optimize commands (`event="postOptimize"`).--Users can write custom update event listener classes in Solr plugins. As of Solr 7.1,-`RunExecutableListener` was removed for security reasons.--== Transaction Log--As described in the section <<realtime-get.adoc#,RealTime Get>>, a transaction log is required for that feature. It is configured in the `updateHandler` section of `solrconfig.xml`.--Realtime Get currently relies on the update log feature, which is enabled by default. It relies on an update log, which is configured in `solrconfig.xml`, in a section like:--[source,xml]------<updateLog>- <str name="dir">${solr.ulog.dir:}</str>-</updateLog>-------Three additional expert-level configuration settings affect indexing performance and how far a replica can fall behind on updates before it must enter into full recovery - see the section on <<solrcloud-recoveries-and-write-tolerance.adoc#,write side fault tolerance>> for more information:--`numRecordsToKeep`::-The number of update records to keep per log. The default is `100`.--`maxNumLogsToKeep`::-The maximum number of logs keep. The default is `10`.--`numVersionBuckets`::-The number of buckets used to keep track of max version values when checking for re-ordered updates; increase this value to reduce the cost of synchronizing access to version buckets during high-volume indexing, this requires `(8 bytes (long) * numVersionBuckets)` of heap space per Solr core. The default is `65536`.--An example, to be included under `<config><updateHandler>` in `solrconfig.xml`, employing the above advanced settings:--[source,xml]------<updateLog>- <str name="dir">${solr.ulog.dir:}</str>- <int name="numRecordsToKeep">500</int>- <int name="maxNumLogsToKeep">20</int>- <int name="numVersionBuckets">65536</int>-</updateLog>-------== Other Options-In some cases complex updates (such as spatial/shape) may take very long time to complete. In the default-configuration other updates that fall into the same internal version bucket will wait indefinitely and-eventually these outstanding requests may pile up and lead to thread exhaustion and eventually to-OutOfMemory errors.--The option `versionBucketLockTimeoutMs` in the `updateHandler` section helps to prevent that by-specifying a limited timeout for such extremely long running update requests. If this limit-is reached this update will fail but it won't block forever all other updates. See SOLR-12833 for more details.--There's a memory cost associated with this setting. Values greater than the default 0 (meaning unlimited timeout)-cause Solr to use a different internal implementation of the version bucket, which increases memory consumption-from ~1.5MB to ~6.8MB per Solr core.--An example of specifying this option under `<config>` section of `solrconfig.xml`:--[source,xml]------<updateHandler class="solr.DirectUpdateHandler2">- ...- <int name="versionBucketLockTimeoutMs">10000</int>-</updateHandler>-----diff --git a/solr/solr-ref-guide/src/upgrading-a-solr-cluster.adoc b/solr/solr-ref-guide/src/upgrading-a-solr-cluster.adocindex daefd7edb70..ef88cd2255c 100644--- a/solr/solr-ref-guide/src/upgrading-a-solr-cluster.adoc+++ b/solr/solr-ref-guide/src/upgrading-a-solr-cluster.adoc@@ -19,41 +19,53 @@This page covers how to upgrade an existing Solr cluster that was installed using the <<taking-solr-to-production.adoc#,service installation scripts>>.-IMPORTANT: The steps outlined on this page assume you use the default service name of `solr`. If you use an alternate service name or Solr installation directory, some of the paths and commands mentioned below will have to be modified accordingly.+IMPORTANT: The steps outlined on this page assume you use the default service name of `solr`.+If you use an alternate service name or Solr installation directory, some of the paths and commands mentioned below will have to be modified accordingly.== Planning Your UpgradeHere is a checklist of things you need to prepare before starting the upgrade process:. Examine the <<solr-upgrade-notes.adoc#,Solr Upgrade Notes>> to determine if any behavior changes in the new version of Solr will affect your installation.-. If not using replication (i.e., collections with `replicationFactor` less than 1), then you should make a backup of each collection. If all of your collections use replication, then you don't technically need to make a backup since you will be upgrading and verifying each node individually.-. Determine which Solr node is currently hosting the Overseer leader process in SolrCloud, as you should upgrade this node last. To determine the Overseer, use the Overseer Status API, see: <<collections-api.adoc#,Collections API>>.-. Plan to perform your upgrade during a system maintenance window if possible. You'll be doing a rolling restart of your cluster (each node, one-by-one), but we still recommend doing the upgrade when system usage is minimal.+. If not using replication (i.e., collections with `replicationFactor` less than 1), then you should make a backup of each collection.+If all of your collections use replication, then you don't technically need to make a backup since you will be upgrading and verifying each node individually.+. Determine which Solr node is currently hosting the Overseer leader process in SolrCloud, as you should upgrade this node last.+To determine the Overseer, use the Overseer Status API, see: <<collections-api.adoc#,Collections API>>.+. Plan to perform your upgrade during a system maintenance window if possible.+You'll be doing a rolling restart of your cluster (each node, one-by-one), but we still recommend doing the upgrade when system usage is minimal.. Verify the cluster is currently healthy and all replicas are active, as you should not perform an upgrade on a degraded cluster.. Re-build and test all custom server-side components against the new Solr JAR files.. Determine the values of the following variables that are used by the Solr Control Scripts:* `ZK_HOST`: The ZooKeeper connection string your current SolrCloud nodes use to connect to ZooKeeper; this value will be the same for all nodes in the cluster.* `SOLR_HOST`: The hostname each Solr node used to register with ZooKeeper when joining the SolrCloud cluster; this value will be used to set the *host* Java system property when starting the new Solr process.* `SOLR_PORT`: The port each Solr node is listening on, such as 8983.-* `SOLR_HOME`: The absolute path to the Solr home directory for each Solr node; this directory must contain a `solr.xml` file. This value will be passed to the new Solr process using the `solr.solr.home` system property, see: <<solr-cores-and-solr-xml.adoc#,Solr Cores and solr.xml>>.+* `SOLR_HOME`: The absolute path to the Solr home directory for each Solr node; this directory must contain a `solr.xml` file.+This value will be passed to the new Solr process using the `solr.solr.home` system property, see: <<configuring-solr-xml.adoc#,Configuring solr.xml>>.+If you are upgrading from an installation of Solr 5.x or later, these values can typically be found in either `/var/solr/solr.in.sh` or `/etc/default/solr.in.sh`.-You should now be ready to upgrade your cluster. Please verify this process in a test or staging cluster before doing it in production.+You should now be ready to upgrade your cluster.+Please verify this process in a test or staging cluster before doing it in production.== Upgrade Process-The approach we recommend is to perform the upgrade of each Solr node, one-by-one. In other words, you will need to stop a node, upgrade it to the new version of Solr, and restart it before moving on to the next node. This means that for a short period of time, there will be a mix of "Old Solr" and "New Solr" nodes running in your cluster. We also assume that you will point the new Solr node to your existing Solr home directory where the Lucene index files are managed for each collection on the node. This means that you won't need to move any index files around to perform the upgrade.+The approach we recommend is to perform the upgrade of each Solr node, one-by-one.+In other words, you will need to stop a node, upgrade it to the new version of Solr, and restart it before moving on to the next node.+This means that for a short period of time, there will be a mix of "Old Solr" and "New Solr" nodes running in your cluster.+We also assume that you will point the new Solr node to your existing Solr home directory where the Lucene index files are managed for each collection on the node.+This means that you won't need to move any index files around to perform the upgrade.=== Step 1: Stop Solr-Begin by stopping the Solr node you want to upgrade. After stopping the node, if using a replication (i.e., collections with `replicationFactor` less than 1), verify that all leaders hosted on the downed node have successfully migrated to other replicas; you can do this by visiting the <<cloud-screens.adoc#,Cloud panel in the Solr Admin UI>>. If not using replication, then any collections with shards hosted on the downed node will be temporarily off-line.+Begin by stopping the Solr node you want to upgrade.+After stopping the node, if using a replication (i.e., collections with `replicationFactor` less than 1), verify that all leaders hosted on the downed node have successfully migrated to other replicas; you can do this by visiting the <<cloud-screens.adoc#,Cloud panel in the Solr Admin UI>>. If not using replication, then any collections with shards hosted on the downed node will be temporarily off-line.=== Step 2: Install Solr as a Service-Please follow the instructions to install Solr as a Service on Linux documented at <<taking-solr-to-production.adoc#,Taking Solr to Production>>. Use the `-n` parameter to avoid automatic start of Solr by the installer script. You need to update the `/etc/default/solr.in.sh` include file in the next step to complete the upgrade process.+Please follow the instructions to install Solr as a Service on Linux documented at <<taking-solr-to-production.adoc#,Taking Solr to Production>>. Use the `-n` parameter to avoid automatic start of Solr by the installer script.+You need to update the `/etc/default/solr.in.sh` include file in the next step to complete the upgrade process.[NOTE]====@@ -71,16 +83,19 @@ SOLR_HOST=SOLR_PORT=SOLR_HOME=-Make sure the user you plan to own the Solr process is the owner of the `SOLR_HOME` directory. For instance, if you plan to run Solr as the "solr" user and `SOLR_HOME` is `/var/solr/data`, then you would do: `sudo chown -R solr: /var/solr/data`+Make sure the user you plan to own the Solr process is the owner of the `SOLR_HOME` directory.+For instance, if you plan to run Solr as the "solr" user and `SOLR_HOME` is `/var/solr/data`, then you would do: `sudo chown -R solr: /var/solr/data`=== Step 4: Start Solr-You are now ready to start the upgraded Solr node by doing: `sudo service solr start`. The upgraded instance will join the existing cluster because you're using the same `SOLR_HOME`, `SOLR_PORT`, and `SOLR_HOST` settings used by the old Solr node; thus, the new server will look like the old node to the running cluster. Be sure to look in `/var/solr/logs/solr.log` for errors during startup.+You are now ready to start the upgraded Solr node by doing: `sudo service solr start`. The upgraded instance will join the existing cluster because you're using the same `SOLR_HOME`, `SOLR_PORT`, and `SOLR_HOST` settings used by the old Solr node; thus, the new server will look like the old node to the running cluster.+Be sure to look in `/var/solr/logs/solr.log` for errors during startup.=== Step 5: Run Healthcheck-You should run the Solr *healthcheck* command for all collections that are hosted on the upgraded node before proceeding to upgrade the next node in your cluster. For instance, if the newly upgraded node hosts a replica for the *MyDocuments* collection, then you can run the following command (replace `ZK_HOST` with the ZooKeeper connection string):+You should run the Solr *healthcheck* command for all collections that are hosted on the upgraded node before proceeding to upgrade the next node in your cluster.+For instance, if the newly upgraded node hosts a replica for the *MyDocuments* collection, then you can run the following command (replace `ZK_HOST` with the ZooKeeper connection string):[source,bash]----diff --git a/solr/solr-ref-guide/src/user-managed-clusters.adoc b/solr/solr-ref-guide/src/user-managed-clusters.adocnew file mode 100644index 00000000000..affcb452d80--- /dev/null+++ b/solr/solr-ref-guide/src/user-managed-clusters.adoc@@ -0,0 +1,35 @@+= User-Managed Clusters+:page-children: user-managed-index-replication, \+ user-managed-distributed-search+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++[.lead]+User-managed clusters provide flexibility for local control over cluster activities, without ZooKeeper to coordinate them.++This flexibility means that you are responsible for core management, document and query routing, failover, and load balancing.++****+// This tags the below list so it can be used in the parent page section list+// tag::user-managed-sections[]+[cols="1,1",frame=none,grid=none,stripes=none]+|===+| <<user-managed-index-replication.adoc#,User-Managed Index Replication>>: Replicating indexes across nodes.+| <<user-managed-distributed-search.adoc#,User-Managed Distributed Search>>: Distributed searching in a user-managed cluster.+|===+// end::user-managed-sections[]+****diff --git a/solr/solr-ref-guide/src/user-managed-distributed-search.adoc b/solr/solr-ref-guide/src/user-managed-distributed-search.adocnew file mode 100644index 00000000000..6397cf4059d--- /dev/null+++ b/solr/solr-ref-guide/src/user-managed-distributed-search.adoc@@ -0,0 +1,117 @@+= User-Managed Distributed Search+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++When using traditional index sharding, you will need to consider how to query your documents.++It is highly recommended that you use <<cluster-types.adoc#solrcloud-mode,SolrCloud>> when needing to scale up or scale out. The setup described below is legacy and was used prior to the existence of SolrCloud. SolrCloud provides for a truly distributed set of features with support for things like automatic routing, leader election, optimistic concurrency and other sanity checks that are expected out of a distributed system.++Everything on this page is specific to legacy setup of distributed search. Users trying out SolrCloud should not follow any of the steps or information below.++Update reorders (i.e., replica A may see update X then Y, and replica B may see update Y then X). *deleteByQuery* also handles reorders the same way, to ensure replicas are consistent. All replicas of a shard are consistent, even if the updates arrive in a different order on different replicas.++== Distributing Documents across Shards++When not using SolrCloud, it is up to you to get all your documents indexed on each shard of your server farm. Solr supports distributed indexing (routing) in its true form only in the SolrCloud mode.++In the legacy distributed mode, Solr does not calculate universal term/doc frequencies.+For most large-scale implementations, it is not likely to matter that Solr calculates TF/IDF at the shard level.+However, if your collection is heavily skewed in its distribution across servers, you may find misleading relevancy results in your searches.+In general, it is probably best to randomly distribute documents to your shards.++== Executing Distributed Searches with the shards Parameter++If a query request includes the `shards` parameter, the Solr server distributes the request across all the shards listed as arguments to the parameter.+The `shards` parameter uses this syntax:++`host:port/base_url,host:port/base_url*`++For example, the `shards` parameter below causes the search to be distributed across two Solr servers: *solr1* and **solr2**, both of which are running on port 8983:++`\http://localhost:8983/solr/core1/select?shards=solr1:8983/solr/core1,solr2:8983/solr/core1&indent=true&q=ipod+solr`++Rather than require users to include the shards parameter explicitly, it is usually preferred to configure this parameter as a default in the RequestHandler section of `solrconfig.xml`.++[IMPORTANT]+====+Do not add the `shards` parameter to the standard request handler; doing so may cause search queries may enter an infinite loop. Instead, define a new request handler that uses the `shards` parameter, and pass distributed search requests to that handler.+====++With Legacy mode, only query requests are distributed.+This includes requests to the SearchHandler (or any handler extending from `org.apache.solr.handler.component.SearchHandler`) using standard components that support distributed search.++As in SolrCloud mode, when `shards.info=true`, distributed responses will include information about the shard (where each shard represents a logically different index or physical location).++The following components support distributed search:++* The *Query* component, which returns documents matching a query.+* The *Facet* component, which processes facet.query and facet.field requests where facets are sorted by count (the default).+* The *Highlighting* component, which enables Solr to include "highlighted" matches in field values.+* The *Stats* component, which returns simple statistics for numeric fields within the DocSet.+* The *Debug* component, which helps with debugging.++=== allowUrls Parameter++The nodes allowed in the `shards` parameter is configurable through the `allowUrls` property in `solr.xml`.+This allow-list is automatically configured for SolrCloud but needs explicit configuration for user-managed clusters.+Read more details in the section <<solrcloud-distributed-requests.adoc#configuring-the-shardhandlerfactory,Configuring the ShardHandlerFactory>>.++== Limitations to User-Managed Distributed Search++Distributed searching in Solr has the following limitations:++* Each document indexed must have a unique key.+* If Solr discovers duplicate document IDs, Solr selects the first document and discards subsequent documents.+* The index for distributed searching may become momentarily out of sync if a commit happens between the first and second phase of the distributed search.+This might cause a situation where a document that once matched a query and was subsequently changed may no longer match the query but will still be retrieved.+This situation is expected to be quite rare, however, and is only possible for a single query request.+* The number of shards is limited by number of characters allowed for GET method's URI; most Web servers generally support at least 4000 characters, but many servers limit URI length to reduce their vulnerability to Denial of Service (DoS) attacks.+* Shard information can be returned with each document in a distributed search by including `fl=id, [shard]` in the search request.+This returns the shard URL.+* In a distributed search, the data directory from the core descriptor overrides any data directory in `solrconfig.xml.`+* Update commands may be sent to any server with distributed indexing configured correctly.+Document adds and deletes are forwarded to the appropriate server/shard based on a hash of the unique document id.+*commit* commands and *deleteByQuery* commands are sent to every server listed in `shards`.++Formerly a limitation was that TF/IDF relevancy computations only used shard-local statistics.+This is still the case by default.+If your data isn't randomly distributed, or if you want more exact statistics, then you can configure the <<solrcloud-distributed-requests#distributedidf,`ExactStatsCache`>>.++== Avoiding Distributed Deadlock++Like in SolrCloud mode, inter-shard requests could lead to a distributed deadlock.+It can be avoided by following the instructions in the section <<solrcloud-distributed-requests.adoc#,Distributed Requests>>.++== Load Balancing Requests++When managing a user-managed Solr cluster. query requests should be load balanced across each of the shard followers.+This gives you both increased query handling capacity and failover if a server goes down.++Because there is no centralized cluster management in this scenario, none of the leader shards in this configuration know about each other.+Documents are indexed to each leader, the index is replicated to each follower, and then searches are distributed across the followers, using one follower from each shard.++For high availability you should use a load balancer to set up a virtual IP for each shard's set of followers.+If you are new to load balancing, HAProxy (http://haproxy.1wt.eu/) is a good open source software load-balancer.+If a follower server goes down, a good load-balancer will detect the failure using some technique (generally a heartbeat system), and forward all requests to the remaining live followers that served with the failed follower.+A single virtual IP should then be set up so that requests can hit a single IP, and get load balanced to each of the virtual IPs for the search followers.++With this configuration you will have a fully load balanced, search-side fault-tolerant system.+Incoming searches will be handed off to one of the functioning followers, then the follower will distribute the search request across a follower for each of the shards in your configuration.+The follower will issue a request to each of the virtual IPs for each shard, and the load balancer will choose one of the available followers.+Finally, the results will be combined into a single results set and returned.+If any of the followers go down, they will be taken out of rotation and the remaining followers will be used.+If a shard leader goes down, searches can still be served from the followers until you have corrected the problem and put the leader back into production.diff --git a/solr/solr-ref-guide/src/user-managed-index-replication.adoc b/solr/solr-ref-guide/src/user-managed-index-replication.adocnew file mode 100644index 00000000000..f6127881e4a--- /dev/null+++ b/solr/solr-ref-guide/src/user-managed-index-replication.adoc@@ -0,0 +1,519 @@+= User-Managed Index Replication+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++User-Managed index replication distributes complete copies of a leader index to one or more follower replicas.+The leader continues to manage updates to the index.+All querying is handled by the followers.+This division of labor enables Solr to scale to provide adequate responsiveness to queries against large search volumes.++The figure below shows a Solr configuration using index replication.+The leader server's index is replicated on the followers.++.A Solr index can be replicated across multiple follower servers, which then process requests.+image::images/user-managed-index-replication/user-managed-replication.png[]+++== Index Replication in Solr++Solr includes index replication that works over HTTP:++* The configuration affecting replication is controlled by a single file, `solrconfig.xml`.+* Supports the replication of configuration files as well as index files.+* Works across platforms with same configuration.+* No reliance on OS-dependent file system features (e.g., hard links).+* Tightly integrated with Solr; an admin page offers fine-grained control of each aspect of replication.+* The Java-based replication feature is implemented as a request handler.+Configuring replication is therefore similar to any normal request handler.++.Replication In SolrCloud+[NOTE]+====+Although there is no explicit concept of leader or follower nodes in a <<cluster-types.adoc#solrcloud-mode,SolrCloud>> cluster, the `ReplicationHandler` discussed on this page is still used by SolrCloud as needed to support "shard recovery" – but this is done in a peer to peer manner.++When using SolrCloud, the `ReplicationHandler` must be available via the `/replication` path.+Solr does this implicitly unless overridden explicitly in your `solrconfig.xml`.+If you wish to override the default behavior, make certain that you do not set any of the "leader" or "follower" configuration options mentioned below, or they will interfere with normal SolrCloud operation.+====++== Configuring the ReplicationHandler++In addition to `ReplicationHandler` configuration options specific to the leader and follower roles described below, there are a few special configuration options that are generally supported (even when using SolrCloud).++* `maxNumberOfBackups` an integer value dictating the maximum number of backups this node will keep on disk as it receives `backup` commands.+* Similar to most other request handlers in Solr you may configure a set of <<requesthandlers-searchcomponents.adoc#search-handlers,defaults, invariants, and/or appends>> parameters corresponding with any request parameters supported by the `ReplicationHandler` when <<HTTP API Commands for the ReplicationHandler,processing commands>>.++=== Configuring a Leader Server++Before running a replication, you should set the following parameters on initialization of the handler:++`replicateAfter`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++String specifying action after which replication should occur.+Valid values are:+++* `commit`: Triggers replication whenever a commit is performed on the leader index.+* `optimize`: Triggers replication whenever the leader index is optimized.+* `startup`: Triggers replication whenever the leader index starts up.++++There can be multiple values for this parameter, as in:+++[source,xml]+----+<str name="replicateAfter">startup</str>+<str name="replicateAfter">commit</str>+<str name="replicateAfter">optimize</str>+----+++If you use `startup`, you should have a `commit` and/or `optimize` entry also if you want to trigger replication on future commits or optimizes.++`backupAfter`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++String specifying action after which a backup should occur.+Valid values are `commit`, `optimize`, or `startup`.+There can be multiple values for this parameter.+It is not required for replication, it just makes a backup.++`maxNumberOfBackups`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Integer specifying how many backups to keep.+This can be used to delete all but the most recent _N_ backups.++`confFiles`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The configuration files to replicate to the follower, separated by a comma.+These should be files such as the schema, stopwords, and similar configuration files that may change on the leader and need to be updated on the follower to use when serving queries.+++If `solrconfig.xml` should be copied, special care needs to be taken so the configuration of the leader does not overwrite the configuration of the follower.+A line like the following would ensure that the local configuration `solrconfig_follower.xml` will be saved as `solrconfig.xml` on the follower:+++[source,xml]+----+<str name="confFiles">solrconfig_follower.xml:solrconfig.xml,managed-schema,stopwords.txt</str>+----+++On the leader server, the file name of the follower configuration file can be anything, as long as the name is correctly identified in the `confFiles` string; then it will be saved as whatever file name appears after the colon ':'.+In the above example, all other files would be saved with their original names.++`commitReserveDuration`::+++[%autowidth,frame=none]+|===+|Optional |Default: `00:00:10`+|===+++If your commits are very frequent and your network is slow, you can tweak this parameter to increase the amount of time expected to be required to transfer data.+The default is `00:00:10`, or 10 seconds.++The example below shows a possible 'leader' configuration for the `ReplicationHandler`, including a fixed number of backups and an invariant setting for the `maxWriteMBPerSec` request parameter to prevent followers from saturating its network interface++[source,xml]+----+<requestHandler name="/replication" class="solr.ReplicationHandler">+ <lst name="leader">+ <str name="replicateAfter">optimize</str>+ <str name="backupAfter">optimize</str>+ <str name="confFiles">schema.xml,stopwords.txt,elevate.xml</str>+ </lst>+ <int name="maxNumberOfBackups">2</int>+ <str name="commitReserveDuration">00:00:10</str>+ <lst name="invariants">+ <str name="maxWriteMBPerSec">16</str>+ </lst>+</requestHandler>+----++=== Configuring a Follower Server++A follower configuration needs to be different from the leader because in this approach, the follower initiates the request to the leader for updated index and other files.++We use the following parameters:++`leaderUrl`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++A fully qualified url for the replication handler of leader.+++This parameter must be defined in order to fetch new index and configuration files from the leader, but it does not need to be defined in `solrconfig.xml`.+It can be passed as a request parameter for the <<HTTP API Commands for the ReplicationHandler,`fetchindex`>> command.++`pollInterval`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Interval in which the follower should poll leader.+Format is `HH:mm:ss`.+If this is absent follower does not poll automatically.+++If not configured, a fetchindex can be triggered from the Admin UI or the <<HTTP API Commands for the ReplicationHandler,HTTP API>>.++`compression`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++Enables compression while transferring the index files.+The possible values are `internal` or `external`.+If the value is `external` make sure that your leader Solr has the settings to honor the Accept-Encoding header.+If this is set to `internal` everything will be taken care of automatically.+++While this parameter may seem like a good idea for general use, it's usually only required if the bandwidth between leader and follower nodes is consistently low.++`httpConnTimeout`::+++[%autowidth,frame=none]+|===+|Optional |Default: `5000`+|===+++The length of time in milliseconds to wait for a connection to the leader.+This can usually be left to the default unless bandwidth is extremely low or if there is an extremely high latency.++`httpReadTimeout`::+++[%autowidth,frame=none]+|===+|Optional |Default: `10000`+|===+++The length of time in milliseconds to wait for reading the index files.+Like `httpConnTimeout`, this can usually be left to the default unless bandwidth is extremely low or if there is an extremely high latency.++`httpBasicAuthUser`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The username to use if the leader has been configured with HTTP Basic authentication.++`httpBasicAuthPassword`::+++[%autowidth,frame=none]+|===+|Optional |Default: none+|===+++The password to use if the leader has been configured with HTTP Basic authentication.++The following example shows a ReplicationHandler configuration on a follower:++[source,xml]+----+<requestHandler name="/replication" class="solr.ReplicationHandler">+ <lst name="follower">+ <str name="leaderUrl">http://remote_host:port/solr/core_name/replication</str>+ <str name="pollInterval">00:00:20</str>++ <!-- THE FOLLOWING PARAMETERS ARE USUALLY NOT REQUIRED-->+ <str name="compression">internal</str>+ <str name="httpConnTimeout">5000</str>+ <str name="httpReadTimeout">10000</str>+ <str name="httpBasicAuthUser">username</str>+ <str name="httpBasicAuthPassword">password</str>+ </lst>+</requestHandler>+----++=== Setting Up a Repeater with the ReplicationHandler++A leader may be able to serve only so many followers without affecting performance.+Some organizations have deployed follower servers across multiple data centers.+If each follower downloads the index from a remote data center, the resulting download may consume too much network bandwidth.+To avoid performance degradation in cases like this, you can configure one or more followers as repeaters.+A repeater is simply a node that acts as both a leader and a follower.++To configure a server as a repeater, the definition of the Replication `requestHandler` in the `solrconfig.xml` file must include file lists of use for both leaders and followers.++Be sure to set the `replicateAfter` parameter to `commit`, even if `replicateAfter` is set to `optimize` on the main leader.+This is because on a repeater (or any follower), a commit is called only after the index is downloaded.+The optimize command is never called on followers.++Optionally, one can configure the repeater to fetch compressed files from the leader through the `compression` parameter to reduce the index download time.++Here is an example of a ReplicationHandler configuration for a repeater:++[source,xml]+----+<requestHandler name="/replication" class="solr.ReplicationHandler">+ <lst name="leader">+ <str name="replicateAfter">commit</str>+ <str name="confFiles">schema.xml,stopwords.txt,synonyms.txt</str>+ </lst>+ <lst name="follower">+ <str name="leaderUrl">http://leader.solr.company.com:8983/solr/core_name/replication</str>+ <str name="pollInterval">00:00:60</str>+ </lst>+</requestHandler>+----++== Replication Screen++The Replication screen shows you the current replication state for the core you have specified.++This screen will only appear if you are not running Solr in SolrCloud mode.++If you are using Leader-Follower index replication, you can use this screen to:++. View the replicatable index state. (on a leader node)+. View the current replication status (on a follower node)+. Disable replication. (on a leader node)++image::images/user-managed-index-replication/replication.png[image,width=412,height=250]++== Follower Replication++The leader is totally unaware of the followers.++The follower continuously polls the leader (depending on the `pollInterval` parameter) to check the current index version of the leader.+If a follower discovers that the leader has a newer version of the index it initiates the replication process.++The steps are as follows:++* The follower issues a `filelist` command to get the list of the files.+This command returns the names of the files as well as some metadata (for example, size, a last modified timestamp, an alias if any, etc.).++* The follower checks if it has any of those files in the local index.+It then runs the `filecontent` command to download the missing files.+This uses a custom format (akin to the HTTP chunked encoding) to download the full content or a part of each file.+If the connection breaks in between, the download resumes from the point it failed.+At any point, the follower tries 5 times before giving up a replication altogether.++* The files are downloaded to a temp directory, so if either the follower or the leader crashes during the download process, no files will be corrupted.+Instead, the current replication will simply abort.++* After the download completes, the new files are moved to the live index directory and the file's timestamp is same as its counterpart on the leader.++* A commit command is issued on the follower by the follower's ReplicationHandler and the new index is loaded.++=== Replicating Configuration Files++To replicate configuration files, define them with the `confFiles` parameter.+Only files found in the `conf` directory of the leader's Solr instance will be replicated.++Solr replicates configuration files only when the index itself is replicated.+That means even if a configuration file is changed on the leader, that file will be replicated only after there is a new commit or optimize on the leader's index.++Unlike the index files, where the timestamp is good enough to figure out if they are identical, configuration files are compared against their checksum.+Schema files (on leader and follower) are judged to be identical if their checksums are identical.++As a precaution when replicating configuration files, Solr copies configuration files to a temporary directory before moving them into their ultimate location in the conf directory.+The old configuration files are then renamed and kept in the same `conf/` directory.+The ReplicationHandler does not automatically clean up these old files.++If a replication involved downloading of at least one configuration file, the ReplicationHandler issues a core-reload command instead of a commit command.++=== Resolving Corruption Issues on Follower Servers++If documents are added to the follower, then the follower is no longer in sync with its leader.+However, the follower will not undertake any action to put itself in sync until the leader has new index data.++When a commit operation takes place on the leader, the index version of the leader becomes different from that of the follower.+The follower then fetches the list of files and finds that some of the files present on the leader are also present in the local index but with different sizes and timestamps.+This means that the leader and follower have incompatible indexes.++To correct this problem, the follower then copies all the index files from leader to a new index directory and asks the core to load the fresh index from the new directory.++== HTTP API Commands for the ReplicationHandler++You can use the HTTP commands below to control the ReplicationHandler's operations.++`enablereplication`::+Enable replication on the "leader" for all its followers.+++[source,bash]+http://_leader_host:port_/solr/_core_name_/replication?command=enablereplication++`disablereplication`::+Disable replication on the leader for all its followers.+++[source,bash]+http://_leader_host:port_/solr/_core_name_/replication?command=disablereplication++`indexversion`::+Return the version of the latest replicatable index on the specified leader or follower.+++[source,bash]+http://_host:port_/solr/_core_name_/replication?command=indexversion++`fetchindex`::+Force the specified follower to fetch a copy of the index from its leader.+++[source.bash]+http://_follower_host:port_/solr/_core_name_/replication?command=fetchindex+++You can pass an extra attribute such as `leaderUrl` or `compression` (or any other parameter described in <<Configuring a Follower Server>>) to do a one time replication from a leader.+This removes the need for hard-coding the leader URL in the follower configuration.++`abortfetch`::+Abort copying an index from a leader to the specified follower.+++[source,bash]+http://_follower_host:port_/solr/_core_name_/replication?command=abortfetch++`enablepoll`::+Enable the specified follower to poll for changes on the leader.+++[source,bash]+http://_follower_host:port_/solr/_core_name_/replication?command=enablepoll++`disablepoll`::+Disable the specified follower from polling for changes on the leader.+++[source,bash]+http://_follower_host:port_/solr/_core_name_/replication?command=disablepoll++`details`::+Retrieve configuration details and current status.+++[source,bash]+http://_follower_host:port_/solr/_core_name_/replication?command=details++`filelist`::+Retrieve a list of Lucene files present in the specified host's index.+++[source,bash]+http://_host:port_/solr/_core_name_/replication?command=filelist&generation=<_generation-number_>+++You can discover the generation number of the index by running the `indexversion` command.++`backup`::+Create a backup on leader if there are committed index data in the server; otherwise, does nothing.+++[source,bash]+http://_leader_host:port_/solr/_core_name_/replication?command=backup+++This command is useful for making periodic backups. There are several supported request parameters:++`numberToKeep:`::: This can be used with the backup command unless the `maxNumberOfBackups` initialization parameter has been specified on the handler – in which case `maxNumberOfBackups` is always used and attempts to use the `numberToKeep` request parameter will cause an error.++`name`::: (optional) Backup name.+The snapshot will be created in a directory called `snapshot.<name>` within the data directory of the core.+By default the name is generated using date in `yyyyMMddHHmmssSSS` format.+If the `location` parameter is passed, that would be used instead of the data directory.++`repository`::: The name of the backup repository to use+ When not specified, it defaults to local file system.++`location`::: Backup location.+The value depends on the repository in use.+For file system repository, location defaults to the core's dataDir (data directory), and if specified, it needs to be within `SOLR_HOME`, `SOLR_DATA_HOME` or the paths specified by `solr.xml` `allowPaths` parameter.++`restore`::+Restore a backup from a backup repository.+++[source,bash]+http://_leader_host:port_/solr/_core_name_/replication?command=restore+++This command is used to restore a backup. There are several supported request parameters:++`name`::: (optional) Backup name.+The name of the backed up index snapshot to be restored.+If the name is not provided, it looks for backups with snapshot.<timestamp> format in the location directory.+It would choose the latest timestamp backup in that case.++`repository`::: The name of the backup repository where the backup resides.+When not specified, it defaults to local file system.++`location`::: Backup location. The value depends on the repository in use.+For file system repository, location defaults to core's dataDir (data directory), and if specified, it needs to be within `SOLR_HOME`, `SOLR_DATA_HOME` or the paths specified by `solr.xml` `allowPaths` parameter.++`restorestatus`::+Check the status of a running restore operation.+++[source,bash]+http://_leader_host:port_/solr/_core_name_/replication?command=restorestatus+++This command is used to check the status of a restore operation. This command takes no parameters.+++The status value can be "In Progress", "success", or "failed". If it failed then an "exception" will also be sent in the response.++`deletebackup`::+Delete any backup created using the `backup` command.+++[source,bash]+http://_leader_host:port_ /solr/_core_name_/replication?command=deletebackup+++There are two supported parameters:++`name`::: The name of the snapshot.+A snapshot with the name `snapshot._name_` must exist or an error will be returned.++`location`::: The location where the snapshot is created.+++== Optimizing Distributed Indexes++Optimizing an index is not something most users should generally worry about - but in particular users should be aware of the impacts of optimizing an index when using the `ReplicationHandler`.++The time required to optimize a leader index can vary dramatically.+A small index may be optimized in minutes.+A very large index may take hours.+The variables include the size of the index and the speed of the hardware.++Distributing a newly optimized index may take only a few minutes or up to an hour or more, again depending on the size of the index and the performance capabilities of network connections and disks.+During optimization the machine is under load and will not process queries very well.+Given a schedule of updates being driven a few times an hour to the followers, we cannot run an optimize with every committed snapshot.++Copying an optimized index means that the *entire* index will need to be transferred during the next `snappull`.+This is a large expense, but not nearly as huge as running the optimize everywhere.++Consider this example: on a three-follower one-leader configuration, distributing a newly-optimized index takes approximately 80 seconds _total_.+Rolling the change across a tier would require approximately ten minutes per machine (or machine group).+If this optimize were rolled across the query tier, and if each follower node being optimized were disabled and not receiving queries, a rollout would take at least twenty minutes and potentially as long as an hour and a half.+Additionally, the files would need to be synchronized so that _following_ the optimize, `snappull` would not think that the independently optimized files were different in any way.+This would also leave the door open to independent corruption of indexes instead of each being a perfect copy of the leader.++Optimizing on the leader allows for a straight-forward optimization operation.+No query followers need to be taken out of service.+The optimized index can be distributed in the background as queries are being normally serviced.+The optimization can occur at any time convenient to the application providing index updates.++While optimizing may have some benefits in some situations, a rapidly changing index will not retain those benefits for long, and since optimization is an intensive process, it may be better to consider other options, such as lowering the merge factor (discussed in the section on <<index-segments-merging.adoc#merge-factors,Index Configuration>>).++TIP: Do not elect to optimize your index unless you have tangible evidence that it will significantly improve your search performance.diff --git a/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc b/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adocdeleted file mode 100644index 70555182a3b..00000000000--- a/solr/solr-ref-guide/src/using-the-solr-administration-user-interface.adoc+++ /dev/null@@ -1,45 +0,0 @@-= Using the Solr Administration User Interface-:page-children: overview-of-the-solr-admin-ui, logging, cloud-screens, collections-core-admin, java-properties, thread-dump, collection-specific-tools, core-specific-tools-// Licensed to the Apache Software Foundation (ASF) under one-// or more contributor license agreements. See the NOTICE file-// distributed with this work for additional information-// regarding copyright ownership. The ASF licenses this file-// to you under the Apache License, Version 2.0 (the-// "License"); you may not use this file except in compliance-// with the License. You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing,-// software distributed under the License is distributed on an-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY-// KIND, either express or implied. See the License for the-// specific language governing permissions and limitations-// under the License.--This section discusses the Solr Administration User Interface ("Admin UI").--The <<overview-of-the-solr-admin-ui.adoc#,Overview of the Solr Admin UI>> explains the basic features of the user interface, what's on the initial Admin UI page, and how to configure the interface. In addition, there are pages describing each screen of the Admin UI:--* *<<logging.adoc#,Logging>>* shows recent messages logged by this Solr node and provides a way to change logging levels for specific classes.-* *<<cloud-screens.adoc#,Cloud Screens>>* display information about nodes when running in SolrCloud mode.-* *<<collections-core-admin.adoc#,Collections / Core Admin>>* explains how to get management information about each core.-* *<<java-properties.adoc#,Java Properties>>* shows the Java information about each core.-* *<<thread-dump.adoc#,Thread Dump>>* lets you see detailed information about each thread, along with state information.--* *<<collection-specific-tools.adoc#,Collection-Specific Tools>>* is a section explaining additional screens available for each collection.-// TODO: SOLR-10655 BEGIN: refactor this into a 'collection-screens-list.include.adoc' file for reuse-** <<analysis-screen.adoc#,Analysis>> - lets you analyze the data found in specific fields.-** <<documents-screen.adoc#,Documents>> - provides a simple form allowing you to execute various Solr indexing commands directly from the browser.-** <<files-screen.adoc#,Files>> - shows the current core configuration files such as `solrconfig.xml`.-** <<query-screen.adoc#,Query>> - lets you submit a structured query about various elements of a core.-** <<stream-screen.adoc#,Stream>> - allows you to submit streaming expressions and see results and parsing explanations.-** <<schema-browser-screen.adoc#,Schema Browser>> - displays schema data in a browser window.-// TODO: SOLR-10655 END-* *<<core-specific-tools.adoc#,Core-Specific Tools>>* is a section explaining additional screens available for each named core.-// TODO: SOLR-10655 BEGIN: refactor this into a 'core-screens-list.include.adoc' file for reuse-** <<ping.adoc#,Ping>> - lets you ping a named core and determine whether the core is active.-** <<plugins-stats-screen#plugins-stats-screen,Plugins/Stats>> - shows statistics for plugins and other installed components.-** <<replication-screen.adoc#,Replication>> - shows you the current replication status for the core, and lets you enable/disable replication.-** <<segments-info.adoc#,Segments Info>> - Provides a visualization of the underlying Lucene index segments.-// TODO: SOLR-10655 ENDdiff --git a/solr/solr-ref-guide/src/v2-api.adoc b/solr/solr-ref-guide/src/v2-api.adocindex 718687e349c..8f3484648e7 100644--- a/solr/solr-ref-guide/src/v2-api.adoc+++ b/solr/solr-ref-guide/src/v2-api.adoc@@ -17,13 +17,16 @@// under the License.[[top-v2-api]]-The v2 API is a modernized self-documenting API interface covering most current Solr APIs. It is anticipated that once the v2 API reaches full coverage, and Solr-internal API usages like SolrJ and the Admin UI have been converted from the old API to the v2 API, the old API will eventually be retired.+The v2 API is a modernized self-documenting API interface covering most current Solr APIs.+It is anticipated that once the v2 API reaches full coverage, and Solr-internal API usages like SolrJ and the Admin UI have been converted from the old API to the v2 API, the old API will eventually be retired.-For now the two API styles will coexist, and all the old APIs will continue to work without any change. You can disable all v2 API endpoints by starting your servers with this system property: `-Ddisable.v2.api=true`.+For now the two API styles will coexist, and all the old APIs will continue to work without any change.+You can disable all v2 API endpoints by starting your servers with this system property: `-Ddisable.v2.api=true`.The old API and the v2 API differ in three principle ways:-. Command format: The old API commands and associated parameters are provided through URL request parameters on HTTP GET requests, while in the v2 API most API commands are provided via a JSON body POST'ed to v2 API endpoints. The v2 API also supports HTTP methods GET and DELETE where appropriate.+. Command format: The old API commands and associated parameters are provided through URL request parameters on HTTP GET requests, while in the v2 API most API commands are provided via a JSON body POST'ed to v2 API endpoints.+The v2 API also supports HTTP methods GET and DELETE where appropriate.. Endpoint structure: The v2 API endpoint structure has been rationalized and regularized.. Documentation: The v2 APIs are self-documenting: append `/_introspect` to any valid v2 API path and the API specification will be returned in JSON format.@@ -59,7 +62,8 @@ To limit the introspect output to include just one particular HTTP method, add t`\http://localhost:8983/api/c/_introspect?method=POST`-Most endpoints support commands provided in a body sent via POST. To limit the introspect output to only one command, add the request parameter `command=_command-name_`.+Most endpoints support commands provided in a body sent via POST.+To limit the introspect output to only one command, add the request parameter `command=_command-name_`.`\http://localhost:8983/api/c/gettingstarted/_introspect?method=POST&command=modify`diff --git a/solr/solr-ref-guide/src/variables.adoc b/solr/solr-ref-guide/src/variables.adocindex 6cc2bd996d8..148f4d4fce1 100644--- a/solr/solr-ref-guide/src/variables.adoc+++ b/solr/solr-ref-guide/src/variables.adoc@@ -23,10 +23,12 @@ variables with math expressions.== The Let ExpressionThe `let` expression sets variables and returns-the value of the last variable by default. The output of any streaming expression or math expression can be set to a variable.+the value of the last variable by default.+The output of any streaming expression or math expression can be set to a variable.Below is a simple example setting three variables `a`, `b`,-and `c`. Variables `a` and `b` are set to arrays. The variable `c` is set+and `c`. Variables `a` and `b` are set to arrays.+The variable `c` is setto the output of the `ebeAdd` function which performs element-by-elementaddition of the two arrays.@@ -150,7 +152,8 @@ When this expression is sent to the `/stream` handler it responds with:The `let` expression can also include a `zplot` expression that can be used to visualize thevariables.-In the example below the variables `a` and `b` are set to arrays. The `zplot` function+In the example below the variables `a` and `b` are set to arrays.+The `zplot` functionoutputs the variables as `x` and `y` fields in the output.[source,text]@@ -188,13 +191,15 @@ When this expression is sent to the `/stream` handler it responds with:}-----Using this approach variables can by visualized using Zeppelin-Solr. In the example below+Using this approach variables can by visualized using Zeppelin-Solr.+In the example belowthe arrays are shown in table format.image::images/math-expressions/variables.png[]Once in table format we can plot the variables using one of the plotting or charting-visualizations. The example below shows variables plotted on a line chart:+visualizations.+The example below shows variables plotted on a line chart:image::images/math-expressions/variables1.png[]@@ -202,7 +207,9 @@ image::images/math-expressions/variables1.png[]== Caching VariablesVariables can be cached in-memory on the Solr node where the math expression-was run. A cached variable can then be used in future expressions. Any object+was run.+A cached variable can then be used in future expressions.+Any objectthat can be set to a variable, including data structures and mathematical models, canbe cached in-memory for future use.@@ -336,7 +343,8 @@ When this expression is sent to the `/stream` handler it responds with:----The `removeCache` function can be used to remove a key from a specific-workspace. The `removeCache` function removes the key from the cache+workspace.+The `removeCache` function removes the key from the cacheand returns the object that was removed.In the example below the array that was cached above is removed from the cache.diff --git a/solr/solr-ref-guide/src/vector-math.adoc b/solr/solr-ref-guide/src/vector-math.adocindex 2c3ee663bc4..504ec35fd50 100644--- a/solr/solr-ref-guide/src/vector-math.adoc+++ b/solr/solr-ref-guide/src/vector-math.adoc@@ -60,10 +60,12 @@ Let's first see what happens when we visualize the array function as a table.image::images/math-expressions/array.png[]-It appears as one row with a comma-delimited list of values. You'll find that you can't visualize this output+It appears as one row with a comma-delimited list of values.+You'll find that you can't visualize this outputusing any of the plotting tools.-To plot the array you need the `zplot` function. Let's first look at how `zplot` output looks like in JSON format.+To plot the array you need the `zplot` function.+Let's first look at how `zplot` output looks like in JSON format.[source,text]----@@ -145,7 +147,8 @@ image::images/math-expressions/line1.png[]Each chart has settings which can be explored by clicking on *settings*.-You can switch between chart types for different types of visualizations. Below is an example of+You can switch between chart types for different types of visualizations.+Below is an example ofa bar chart:image::images/math-expressions/bar.png[]@@ -426,8 +429,9 @@ When this expression is sent to the `/stream` handler it responds with:== Vector Summarizations and Norms-There are a set of functions that perform summarizations and return norms of arrays. These functions-operate over an array and return a single value. The following vector summarizations and norm functions are available:+There are a set of functions that perform summarizations and return norms of arrays.+These functions operate over an array and return a single value.+The following vector summarizations and norm functions are available:`mult`, `add`, `sumSq`, `mean`, `l1norm`, `l2norm`, `linfnorm`.The example below shows the `mult` function, which multiples all the values of an array.@@ -524,7 +528,8 @@ When this expression is sent to the `/stream` handler it responds with:== Element-By-Element Vector MathTwo vectors can be added, subtracted, multiplied and divided using element-by-element-vector math functions. The available element-by-element vector math functions are:+vector math functions.+The available element-by-element vector math functions are:`ebeAdd`, `ebeSubtract`, `ebeMultiply`, `ebeDivide`.The expression below performs the element-by-element subtraction of two arrays.@@ -560,7 +565,8 @@ When this expression is sent to the `/stream` handler it responds with:== Dot Product and Cosine SimilarityThe `dotProduct` and `cosineSimilarity` functions are often used as similarity measures between two-sparse vectors. The `dotProduct` is a measure of both angle and magnitude while `cosineSimilarity`+sparse vectors.+The `dotProduct` is a measure of both angle and magnitude while `cosineSimilarity`is a measure only of angle.Below is an example of the `dotProduct` function:diff --git a/solr/solr-ref-guide/src/zookeeper-access-control.adoc b/solr/solr-ref-guide/src/zookeeper-access-control.adocindex 9a904586e89..ac539d75538 100644--- a/solr/solr-ref-guide/src/zookeeper-access-control.adoc+++ b/solr/solr-ref-guide/src/zookeeper-access-control.adoc@@ -16,15 +16,20 @@// specific language governing permissions and limitations// under the License.-This section describes using ZooKeeper access control lists (ACLs) with Solr. For information about ZooKeeper ACLs, see the ZooKeeper documentation at http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperProgrammers.html#sc_ZooKeeperAccessControl.+This section describes using ZooKeeper access control lists (ACLs) with Solr.++For information about ZooKeeper ACLs, see the http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperProgrammers.html#sc_ZooKeeperAccessControl[ZooKeeper ACL documentation].== About ZooKeeper ACLsSolrCloud uses ZooKeeper for shared information and for coordination.-This section describes how to configure Solr to add more restrictive ACLs to the ZooKeeper content it creates, and how to tell Solr about the credentials required to access the content in ZooKeeper. If you want to use ACLs in your ZooKeeper nodes, you will have to activate this functionality; by default, Solr behavior is open-unsafe ACL everywhere and uses no credentials.+This section describes how to configure Solr to add more restrictive ACLs to the ZooKeeper content it creates, and how to tell Solr about the credentials required to access the content in ZooKeeper.+If you want to use ACLs in your ZooKeeper nodes, you will have to activate this functionality; by default, Solr behavior is open-unsafe ACL everywhere and uses no credentials.-Content stored in ZooKeeper is critical to the operation of a SolrCloud cluster. Open access to SolrCloud content on ZooKeeper could lead to a variety of problems. For example:+Content stored in ZooKeeper is critical to the operation of a SolrCloud cluster.+Open access to SolrCloud content on ZooKeeper could lead to a variety of problems.+For example:* Changing configuration might cause Solr to fail or behave in an unintended way.* Changing cluster state information into something wrong or inconsistent might very well make a SolrCloud cluster behave strangely.@@ -35,45 +40,67 @@ You may want to enable ZooKeeper ACLs with Solr if you grant access to your ZooK* Malware that found its way into your system.* Other systems using the same ZooKeeper ensemble (a "bad thing" might be done by accident).-You might even want to limit read-access, if you think there is stuff in ZooKeeper that not everyone should know about. Or you might just in general work on a need-to-know basis.+You might even want to limit read-access, if you think there is stuff in ZooKeeper that not everyone should know about.+Or you might just in general work on a need-to-know basis.-Protecting ZooKeeper itself could mean many different things. **This section is about protecting Solr content in ZooKeeper**. ZooKeeper content basically lives persisted on disk and (partly) in memory of the ZooKeeper processes. *This section is not about protecting ZooKeeper data at storage or ZooKeeper process levels* - that's for ZooKeeper to deal with.+Protecting ZooKeeper itself could mean many different things.+**This section is about protecting Solr content in ZooKeeper**.+ZooKeeper content basically lives persisted on disk and (partly) in memory of the ZooKeeper processes.+*This section is not about protecting ZooKeeper data at storage or ZooKeeper process levels* - that's for ZooKeeper to deal with.-But this content is also available to "the outside" via the ZooKeeper API. Outside processes can connect to ZooKeeper and create/update/delete/read content; for example, a Solr node in a SolrCloud cluster wants to create/update/delete/read, and a SolrJ client wants to read from the cluster. It is the responsibility of the outside processes that create/update content to setup ACLs on the content. ACLs describe who is allowed to read, update, delete, create, etc. Each piece of information (znode/content) in ZooKeeper has its own set of ACLs, and inheritance or sharing is not possible. The default behavior in Solr is to add one ACL on all the content it creates - one ACL that gives anyone the permission to do anything (in ZooKeeper terms this is called "the open-unsafe ACL").+But this content is also available to "the outside" via the ZooKeeper API.+Outside processes can connect to ZooKeeper and create/update/delete/read content.+For example, a Solr node in a SolrCloud cluster wants to create/update/delete/read, and a SolrJ client wants to read from the cluster.+It is the responsibility of the outside processes that create/update content to setup ACLs on the content.+ACLs describe who is allowed to read, update, delete, create, etc.+Each piece of information (znode/content) in ZooKeeper has its own set of ACLs, and inheritance or sharing is not possible.+The default behavior in Solr is to add one ACL on all the content it creates - one ACL that gives anyone the permission to do anything (in ZooKeeper terms this is called "the open-unsafe ACL").== How to Enable ACLsWe want to be able to:-. Control the credentials Solr uses for its ZooKeeper connections. The credentials are used to get permission to perform operations in ZooKeeper.+. Control the credentials Solr uses for its ZooKeeper connections.+The credentials are used to get permission to perform operations in ZooKeeper.. Control which ACLs Solr will add to znodes (ZooKeeper files/folders) it creates in ZooKeeper.. Control it "from the outside", so that you do not have to modify and/or recompile Solr code to turn this on.-Solr nodes, clients and tools (e.g., ZkCLI) always use a java class called {solr-javadocs}/solrj/org/apache/solr/common/cloud/SolrZkClient.html[`SolrZkClient`] to deal with their ZooKeeper stuff. The implementation of the solution described here is all about changing `SolrZkClient`. If you use `SolrZkClient` in your application, the descriptions below will be true for your application too.+Solr nodes, clients, and tools (e.g., ZkCLI) always use a java class called {solr-javadocs}/solrj/org/apache/solr/common/cloud/SolrZkClient.html[`SolrZkClient`] to deal with their ZooKeeper stuff.+The implementation of the solution described here is all about changing `SolrZkClient`.+If you use `SolrZkClient` in your application, the descriptions below will be true for your application too.=== Controlling Credentials-You control which credentials provider will be used by configuring the `zkCredentialsProvider` property in `solr.xml` 's `<solrcloud>` section to the name of a class (on the classpath) implementing the {solr-javadocs}/solrj/org/apache/solr/common/cloud/ZkCredentialsProvider.html[`ZkCredentialsProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkCredentialsProvider` such that it will take on the value of the same-named `zkCredentialsProvider` system property if it is defined (e.g., by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkCredentialsProvider` implementation.+You control which credentials provider will be used by configuring the `zkCredentialsProvider` property in the `<solrcloud>` section of <<configuring-solr-xml.adoc#,`solr.xml`>> to the name of a class (on the classpath) implementing the {solr-javadocs}/solrj/org/apache/solr/common/cloud/ZkCredentialsProvider.html[`ZkCredentialsProvider`] interface.++`solr.xml` defines the `zkCredentialsProvider` such that it will take on the value of the same-named `zkCredentialsProvider` system property if it is defined (in `solr.in.sh/.cmd` - see <<ZooKeeper ACLs in Solr Scripts,below>>), or if not, default to the `DefaultZkCredentialsProvider` implementation.==== Out of the Box Credential ImplementationsYou can always make you own implementation, but Solr comes with two implementations:* `org.apache.solr.common.cloud.DefaultZkCredentialsProvider`: Its `getCredentials()` returns a list of length zero, or "no credentials used". This is the default.-* `org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider`: This lets you define your credentials using system properties. It supports at most one set of credentials.-** The schema is "digest". The username and password are defined by system properties `zkDigestUsername` and `zkDigestPassword`. This set of credentials will be added to the list of credentials returned by `getCredentials()` if both username and password are provided.+* `org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider`: This lets you define your credentials using system properties.+It supports at most one set of credentials.+** The schema is "digest".+The username and password are defined by system properties `zkDigestUsername` and `zkDigestPassword`.+This set of credentials will be added to the list of credentials returned by `getCredentials()` if both username and password are provided.** If the one set of credentials above is not added to the list, this implementation will fall back to default behavior and use the (empty) credentials list from `DefaultZkCredentialsProvider`.=== Controlling ACLs-You control which ACLs will be added by configuring `zkACLProvider` property in `solr.xml` 's `<solrcloud>` section to the name of a class (on the classpath) implementing the {solr-javadocs}/solrj/org/apache/solr/common/cloud/ZkACLProvider.html[`ZkACLProvider`] interface. `server/solr/solr.xml` in the Solr distribution defines the `zkACLProvider` such that it will take on the value of the same-named `zkACLProvider` system property if it is defined (e.g., by uncommenting the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `solr.in.sh/.cmd` - see below), or if not, default to the `DefaultZkACLProvider` implementation.+You control which ACLs will be added by configuring `zkACLProvider` property in the `<solrcloud>` section of <<configuring-solr-xml.adoc#,`solr.xml`>> to the name of a class (on the classpath) implementing the {solr-javadocs}/solrj/org/apache/solr/common/cloud/ZkACLProvider.html[`ZkACLProvider`] interface.++`solr.xml` defines the `zkACLProvider` such that it will take on the value of the same-named `zkACLProvider` system property if it is defined (in `solr.in.sh/.cmd` - see <<ZooKeeper ACLs in Solr Scripts,below>>), or if not, default to the `DefaultZkACLProvider` implementation.==== Out of the Box ACL ImplementationsYou can always make you own implementation, but Solr comes with:-* `org.apache.solr.common.cloud.DefaultZkACLProvider`: It returns a list of length one for all `zNodePath`-s. The single ACL entry in the list is "open-unsafe". This is the default.-* `org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider`: This lets you define your ACLs using system properties. Its `getACLsToAdd()` implementation will apply only admin ACLs to pre-defined sensitive paths as defined by `SecurityAwareZkACLProvider` (`/security.json` and `/security/*`) and both admin and user ACLs to the rest of the contents. The two sets of roles will be defined as:+* `org.apache.solr.common.cloud.DefaultZkACLProvider`: It returns a list of length one for all `zNodePath`-s.+The single ACL entry in the list is "open-unsafe". This is the default.+* `org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider`: This lets you define your ACLs using system properties.+The `getACLsToAdd()` implementation will apply only admin ACLs to pre-defined sensitive paths as defined by `SecurityAwareZkACLProvider` (`/security.json` and `/security/*`) and both admin and user ACLs to the rest of the contents. The two sets of roles will be defined as:** A user that is allowed to do everything.*** The permission is `ALL` (corresponding to all of `CREATE`, `READ`, `WRITE`, `DELETE`, and `ADMIN`), and the schema is "digest".*** The username and password are defined by system properties `zkDigestUsername` and `zkDigestPassword`, respectively.@@ -82,13 +109,19 @@ You can always make you own implementation, but Solr comes with:*** The permission is `READ` and the schema is `digest`.*** The username and password are defined by system properties `zkDigestReadonlyUsername` and `zkDigestReadonlyPassword`, respectively.*** This ACL will not be added to the list of ACLs unless both username and password are provided.-* `org.apache.solr.common.cloud.SaslZkACLProvider`: Requires SASL authentication. Gives all permissions for the user specified in system property `solr.authorization.superuser` (default: `solr`) when using SASL, and gives read permissions for anyone else. Designed for a setup where configurations have already been set up and will not be modified, or where configuration changes are controlled via Solr APIs. This provider will be useful for administration in a kerberos environment. In such an environment, the administrator wants Solr to authenticate to ZooKeeper using SASL, since this is only way to authenticate with ZooKeeper via Kerberos.+* `org.apache.solr.common.cloud.SaslZkACLProvider`: Requires SASL authentication.+Gives all permissions for the user specified in system property `solr.authorization.superuser` (default: `solr`) when using SASL, and gives read permissions for anyone else.+Designed for a setup where configurations have already been set up and will not be modified, or where configuration changes are controlled via Solr APIs.+This provider will be useful for administration in a Kerberos environment.+In such an environment, the administrator wants Solr to authenticate to ZooKeeper using SASL, since this is only way to authenticate with ZooKeeper via Kerberos.If none of the above ACLs is added to the list, the (empty) ACL list of `DefaultZkACLProvider` will be used by default.-Notice the overlap in system property names with credentials provider `VMParamsSingleSetCredentialsDigestZkCredentialsProvider` (described above). This is to let the two providers collaborate in a nice and perhaps common way: we always protect access to content by limiting to two users - an admin-user and a readonly-user - AND we always connect with credentials corresponding to this same admin-user, basically so that we can do anything to the content/znodes we create ourselves.+Notice the overlap in system property names with credentials provider `VMParamsSingleSetCredentialsDigestZkCredentialsProvider` (described above).+This is to let the two providers collaborate in a nice and perhaps common way: we always protect access to content by limiting to two users - an admin-user and a readonly-user - AND we always connect with credentials corresponding to this same admin-user, basically so that we can do anything to the content/znodes we create ourselves.-You can give the readonly credentials to "clients" of your SolrCloud cluster - e.g., to be used by SolrJ clients. They will be able to read whatever is necessary to run a functioning SolrJ client, but they will not be able to modify any content in ZooKeeper.+You can give the readonly credentials to "clients" of your SolrCloud cluster - e.g., to be used by SolrJ clients.+They will be able to read whatever is necessary to run a functioning SolrJ client, but they will not be able to modify any content in ZooKeeper.=== ZooKeeper ACLs in Solr Scripts@@ -98,9 +131,11 @@ There are two scripts that impact ZooKeeper ACLs:* For Windows systems: `bin/solr.cmd` & `server/scripts/cloud-scripts/zkcli.bat`[IMPORTANT]-Both the solr.in.* and the zkcli.* files will need to be updated with the same password for everything to work. The contents may appear redundant, but the scripts will not consult each other during operations.+Both the solr.in.* and the zkcli.* files will need to be updated with the same password for everything to work.+The contents may appear redundant, but the scripts will not consult each other during operations.-These Solr scripts can enable use of ZooKeeper ACLs by setting the appropriate system properties: uncomment the following and replace the passwords with ones you choose to enable the above-described VM parameters ACL and credentials providers in the following files:+These Solr scripts can enable use of ZooKeeper ACLs by setting the appropriate system properties.+Uncomment the following and replace the passwords with ones you choose to enable the VM parameters and ACL credentials providers in the following files:[.dynamic-tabs]--@@ -159,13 +194,43 @@ REM -DzkDigestReadonlyUsername=readonly-user -DzkDigestReadonlyPassword=CHANGEM== Changing ACL Schemes-Over the lifetime of operating your Solr cluster, you may decide to move from an unsecured ZooKeeper to a secured instance. Changing the configured `zkACLProvider` in `solr.xml` will ensure that newly created nodes are secure, but will not protect the already existing data. To modify all existing ACLs, you can use the `updateacls` command with Solr's ZkCLI. First uncomment the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `server/scripts/cloud-scripts/zkcli.sh` (or `zkcli.bat` on Windows) and fill in the passwords for the admin-user and the readonly-user - see above - then run `server/scripts/cloud-scripts/zkcli.sh -cmd updateacls /zk-path`, or on Windows run `server\scripts\cloud-scripts\zkcli.bat cmd updateacls /zk-path`.+Over the lifetime of operating your Solr cluster, you may decide to move from an unsecured ZooKeeper to a secured instance.+Changing the configured `zkACLProvider` in `solr.xml` will ensure that newly created nodes are secure, but will not protect the already existing data.++To modify all existing ACLs, you can use the `updateacls` command with Solr's ZkCLI.+First uncomment the `SOLR_ZK_CREDS_AND_ACLS` environment variable definition in `server/scripts/cloud-scripts/zkcli.sh` (or `zkcli.bat` on Windows) and fill in the passwords for the admin-user and the readonly-user as described above in <<ZooKeeper ACLs in Solr Scripts>>.++Then run the command below appropriate for your operating system:++[.dynamic-tabs]+--+[example.tab-pane#updateacls-unix]+====+[.tab-label]**nix*+[source,bash]+----+$ ./server/scripts/cloud-scripts/zkcli.sh -cmd updateacls /zk-path+----+====++[example.tab-pane#updateacls-windows]+====+[.tab-label]*Windows*+[source,powershell]+----+C:\\ server\scripts\cloud-scripts\zkcli.bat cmd updateacls /zk-path+----+====+---Changing ACLs in ZK should only be done while your SolrCloud cluster is stopped. Attempting to do so while Solr is running may result in inconsistent state and some nodes becoming inaccessible.+Changing ACLs in ZooKeeper should only be done while your SolrCloud cluster is stopped.+Attempting to do so while Solr is running may result in inconsistent state and some nodes becoming inaccessible.The VM properties `zkACLProvider` and `zkCredentialsProvider`, included in the `SOLR_ZK_CREDS_AND_ACLS` environment variable in `zkcli.sh/.bat`, control the conversion:-* The Credentials Provider must be one that has current admin privileges on the nodes. When omitted, the process will use no credentials (suitable for an unsecure configuration).-* The ACL Provider will be used to compute the new ACLs. When omitted, the process will set all permissions to all users, removing any security present.+* The Credentials Provider must be one that has current admin privileges on the nodes.+When omitted, the process will use no credentials (suitable for an unsecure configuration).+* The ACL Provider will be used to compute the new ACLs.+When omitted, the process will set all permissions to all users, removing any security present.The uncommented `SOLR_ZK_CREDS_AND_ACLS` environment variable in `zkcli.sh/.bat` sets the credentials and ACL providers to the `VMParamsSingleSetCredentialsDigestZkCredentialsProvider` and `VMParamsAllAndReadonlyDigestZkACLProvider` implementations, described earlier in the page.diff --git a/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc b/solr/solr-ref-guide/src/zookeeper-ensemble.adocsimilarity index 73%rename from solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adocrename to solr/solr-ref-guide/src/zookeeper-ensemble.adocindex 4a3220978b2..316ea984cb5 100644--- a/solr/solr-ref-guide/src/setting-up-an-external-zookeeper-ensemble.adoc+++ b/solr/solr-ref-guide/src/zookeeper-ensemble.adoc@@ -1,4 +1,4 @@-= Setting Up an External ZooKeeper Ensemble+= ZooKeeper Ensemble Configuration// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -27,37 +27,46 @@ The solution to this problem is to set up an external ZooKeeper _ensemble_, whicThe first question to answer is the number of ZooKeeper nodes you will run in your ensemble.-When planning how many ZooKeeper nodes to configure, keep in mind that the main principle for a ZooKeeper ensemble is maintaining a majority of servers to serve requests. This majority is called a _quorum_.+When planning how many ZooKeeper nodes to configure, keep in mind that the main principle for a ZooKeeper ensemble is maintaining a majority of servers to serve requests.+This majority is called a _quorum_.[quote,ZooKeeper Administrator's Guide,http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperAdmin.html]____-"For a ZooKeeper service to be active, there must be a majority of non-failing machines that can communicate with each other. *To create a deployment that can tolerate the failure of F machines, you should count on deploying 2xF+1 machines*.+"For a ZooKeeper service to be active, there must be a majority of non-failing machines that can communicate with each other.+*To create a deployment that can tolerate the failure of F machines, you should count on deploying 2xF+1 machines*.____To properly maintain a quorum, it's highly recommended to have an odd number of ZooKeeper servers in your ensemble, so a majority is maintained.-To explain why, think about this scenario: If you have two ZooKeeper nodes and one goes down, this means only 50% of available servers are available. Since this is not a majority, ZooKeeper will no longer serve requests.+To explain why, think about this scenario: If you have two ZooKeeper nodes and one goes down, this means only 50% of available servers are available.+Since this is not a majority, ZooKeeper will no longer serve requests.-However, if you have three ZooKeeper nodes and one goes down, you have 66% of your servers available and ZooKeeper will continue normally while you repair the one down node. If you have 5 nodes, you could continue operating with two down nodes if necessary.+However, if you have three ZooKeeper nodes and one goes down, you have 66% of your servers available and ZooKeeper will continue normally while you repair the one down node.+If you have 5 nodes, you could continue operating with two down nodes if necessary.-It's not generally recommended to go above 5 nodes. While it may seem that more nodes provide greater fault-tolerance and availability, in practice it becomes less efficient because of the amount of inter-node coordination that occurs. Unless you have a truly massive Solr cluster (on the scale of 1,000s of nodes), try to stay to 3 as a general rule, or maybe 5 if you have a larger cluster.+It's not generally recommended to go above 5 nodes.+While it may seem that more nodes provide greater fault-tolerance and availability, in practice it becomes less efficient because of the amount of inter-node coordination that occurs.+Unless you have a truly massive Solr cluster (on the scale of 1,000s of nodes), try to stay to 3 as a general rule, or maybe 5 if you have a larger cluster.More information on ZooKeeper clusters is available from the ZooKeeper documentation at http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperAdmin.html#sc_zkMulitServerSetup.== Download Apache ZooKeeper-The first step in setting up Apache ZooKeeper is, of course, to download the software. It's available from http://zookeeper.apache.org/releases.html.+The first step in setting up Apache ZooKeeper is, of course, to download the software.+It's available from http://zookeeper.apache.org/releases.html.Solr currently uses Apache ZooKeeper v{ivy-zookeeper-version}.[WARNING]====-When using an external ZooKeeper ensemble, you will need need to keep your local installation up-to-date with the latest version distributed with Solr. Since it is a stand-alone application in this scenario, it does not get upgraded as part of a standard Solr upgrade.+When using an external ZooKeeper ensemble, you will need need to keep your local installation up-to-date with the latest version distributed with Solr.+Since it is a stand-alone application in this scenario, it does not get upgraded as part of a standard Solr upgrade.====== ZooKeeper Installation-Installation consists of extracting the files into a specific target directory where you'd like to have ZooKeeper store its internal data. The actual directory itself doesn't matter, as long as you know where it is.+Installation consists of extracting the files into a specific target directory where you'd like to have ZooKeeper store its internal data.+The actual directory itself doesn't matter, as long as you know where it is.The command to unpack the ZooKeeper package is:@@ -90,11 +99,13 @@ The parameters are as follows:`tickTime`:: Part of what ZooKeeper does is determine which servers are up and running at any given time, and the minimum session time out is defined as two "ticks". The `tickTime` parameter specifies in milliseconds how long each tick should be.-`dataDir`:: This is the directory in which ZooKeeper will store data about the cluster. This directory must be empty before starting ZooKeeper for the first time.+`dataDir`:: This is the directory in which ZooKeeper will store data about the cluster.+This directory must be empty before starting ZooKeeper for the first time.`clientPort`:: This is the port on which Solr will access ZooKeeper.-`4lw.commands.whitelist`:: This allows the Solr admin UI to query ZooKeeper. Optionally use "*" to enable all "4 letter words", the three listed will enable the admin UI.+`4lw.commands.whitelist`:: This allows the Solr admin UI to query ZooKeeper.+Optionally use "*" to enable all "4 letter words", the three listed will enable the admin UI.These are the basic parameters that need to be in use on each ZooKeeper node, so this file must be copied to or created on each node.@@ -127,13 +138,18 @@ autopurge.purgeInterval=1We've added these parameters to the three we had already:-`initLimit`:: Amount of time, in ticks, to allow followers to connect and sync to a leader. In this case, you have 5 ticks, each of which is 2000 milliseconds long, so the server will wait as long as 10 seconds to connect and sync with the leader.+`initLimit`:: Amount of time, in ticks, to allow followers to connect and sync to a leader.+In this case, you have 5 ticks, each of which is 2000 milliseconds long, so the server will wait as long as 10 seconds to connect and sync with the leader.-`syncLimit`:: Amount of time, in ticks, to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they will be dropped.+`syncLimit`:: Amount of time, in ticks, to allow followers to sync with ZooKeeper.+If followers fall too far behind a leader, they will be dropped.-`server._X_`:: These are the server IDs (the `_X_` part), hostnames (or IP addresses) and ports for all servers in the ensemble. The IDs differentiate each node of the ensemble, and allow each node to know where each of the other node is located. The ports can be any ports you choose; ZooKeeper's default ports are `2888:3888`.+`server._X_`:: These are the server IDs (the `_X_` part), hostnames (or IP addresses) and ports for all servers in the ensemble.+The IDs differentiate each node of the ensemble, and allow each node to know where each of the other node is located.+The ports can be any ports you choose; ZooKeeper's default ports are `2888:3888`.+-Since we've assigned server IDs to specific hosts/ports, we must also define which server in the list this node is. We do this with a `myid` file stored in the data directory (defined by the `dataDir` parameter). The contents of the `myid` file is only the server ID.+Since we've assigned server IDs to specific hosts/ports, we must also define which server in the list this node is.+We do this with a `myid` file stored in the data directory (defined by the `dataDir` parameter). The contents of the `myid` file is only the server ID.+In the case of the configuration example above, you would create the file `/var/lib/zookeeper/1/myid` with the content "1" (without quotes), as in this example:+@@ -142,11 +158,17 @@ In the case of the configuration example above, you would create the file `/var/`autopurge.snapRetainCount`:: The number of snapshots and corresponding transaction logs to retain when purging old snapshots and transaction logs.+-ZooKeeper automatically keeps a transaction log and writes to it as changes are made. A snapshot of the current state is taken periodically, and this snapshot supersedes transaction logs older than the snapshot. However, ZooKeeper never cleans up either the old snapshots or the old transaction logs; over time they will silently fill available disk space on each server.+ZooKeeper automatically keeps a transaction log and writes to it as changes are made.+A snapshot of the current state is taken periodically, and this snapshot supersedes transaction logs older than the snapshot.+However, ZooKeeper never cleans up either the old snapshots or the old transaction logs; over time they will silently fill available disk space on each server.+-To avoid this, set the `autopurge.snapRetainCount` and `autopurge.purgeInterval` parameters to enable an automatic clean up (purge) to occur at regular intervals. The `autopurge.snapRetainCount` parameter will keep the set number of snapshots and transaction logs when a clean up occurs. This parameter can be configured higher than `3`, but cannot be set lower than 3.+To avoid this, set the `autopurge.snapRetainCount` and `autopurge.purgeInterval` parameters to enable an automatic clean up (purge) to occur at regular intervals.+The `autopurge.snapRetainCount` parameter will keep the set number of snapshots and transaction logs when a clean up occurs.+This parameter can be configured higher than `3`, but cannot be set lower than 3.-`autopurge.purgeInterval`:: The time in hours between purge tasks. The default for this parameter is `0`, so must be set to `1` or higher to enable automatic clean up of snapshots and transaction logs. Setting it as high as `24`, for once a day, is acceptable if preferred.+`autopurge.purgeInterval`:: The time in hours between purge tasks.+The default for this parameter is `0`, so must be set to `1` or higher to enable automatic clean up of snapshots and transaction logs.+Setting it as high as `24`, for once a day, is acceptable if preferred.We'll repeat this configuration on each node.@@ -217,21 +239,27 @@ ZOO_LOG4J_PROP="INFO,ROLLINGFILE"SERVER_JVMFLAGS="-Xms2048m -Xmx2048m -verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -Xloggc:$ZOO_LOG_DIR/zookeeper_gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=9 -XX:GCLogFileSize=20M"----+-The property `ZOO_LOG_DIR` defines the location on the server where ZooKeeper will print its logs. `ZOO_LOG4J_PROP` sets the logging level and log appenders.+The property `ZOO_LOG_DIR` defines the location on the server where ZooKeeper will print its logs.+`ZOO_LOG4J_PROP` sets the logging level and log appenders.+-With `SERVER_JVMFLAGS`, we've defined several parameters for garbage collection and logging GC-related events. One of the system parameters is `-Xloggc:$ZOO_LOG_DIR/zookeeper_gc.log`, which will put the garbage collection logs in the same directory we've defined for ZooKeeper logs, in a file named `zookeeper_gc.log`.+With `SERVER_JVMFLAGS`, we've defined several parameters for garbage collection and logging GC-related events.+One of the system parameters is `-Xloggc:$ZOO_LOG_DIR/zookeeper_gc.log`, which will put the garbage collection logs in the same directory we've defined for ZooKeeper logs, in a file named `zookeeper_gc.log`.-. Review the default settings in `<ZOOKEEPER_HOME>/conf/log4j.properties`, especially the `log4j.appender.ROLLINGFILE.MaxFileSize` parameter. This sets the size at which log files will be rolled over, and by default it is 10MB.+. Review the default settings in `<ZOOKEEPER_HOME>/conf/log4j.properties`, especially the `log4j.appender.ROLLINGFILE.MaxFileSize` parameter.+This sets the size at which log files will be rolled over, and by default it is 10MB.. Copy `zookeeper-env.sh` and any changes to `log4j.properties` to each server in the ensemble.-NOTE: The above instructions are for Linux servers only. The default `zkServer.sh` script includes support for a `zookeeper-env.sh` file but the Windows version of the script, `zkServer.cmd`, does not. To make the same configuration on a Windows server, the changes would need to be made directly in the `zkServer.cmd`.+NOTE: The above instructions are for Linux servers only.+The default `zkServer.sh` script includes support for a `zookeeper-env.sh` file but the Windows version of the script, `zkServer.cmd`, does not.+To make the same configuration on a Windows server, the changes would need to be made directly in the `zkServer.cmd`.At this point, you are ready to start your ZooKeeper ensemble.=== More Information about ZooKeeper-ZooKeeper provides a great deal of power through additional configurations, but delving into them is beyond the scope of Solr's documentation. For more information, see the http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}[ZooKeeper documentation].+ZooKeeper provides a great deal of power through additional configurations, but delving into them is beyond the scope of Solr's documentation.+For more information, see the http://zookeeper.apache.org/doc/r{ivy-zookeeper-version}[ZooKeeper documentation].== Starting and Stopping ZooKeeper@@ -249,7 +277,8 @@ zkServer.cmd startThis command needs to be run on each server that will run ZooKeeper.-TIP: You should see the ZooKeeper logs in the directory where you defined to store them. However, immediately after startup, you may not see the `zookeeper_gc.log` yet, as it likely will not appear until garbage collection has happened the first time.+TIP: You should see the ZooKeeper logs in the directory where you defined to store them.+However, immediately after startup, you may not see the `zookeeper_gc.log` yet, as it likely will not appear until garbage collection has happened the first time.=== Shut Down ZooKeeper@@ -265,11 +294,15 @@ zkServer.cmd stop== Solr Configuration-When starting Solr, you must provide an address for ZooKeeper or Solr won't know how to use it. This can be done in two ways: by defining the _connect string_, a list of servers where ZooKeeper is running, at every startup on every node of the Solr cluster, or by editing Solr's include file as a permanent system parameter. Both approaches are described below.+When starting Solr, you must provide an address for ZooKeeper or Solr won't know how to use it.+This can be done in two ways: by defining the _connect string_, a list of servers where ZooKeeper is running, at every startup on every node of the Solr cluster, or by editing Solr's include file as a permanent system parameter.+Both approaches are described below.-When referring to the location of ZooKeeper within Solr, it's best to use the addresses of all the servers in the ensemble. If one happens to be down, Solr will automatically be able to send its request to another server in the list.+When referring to the location of ZooKeeper within Solr, it's best to use the addresses of all the servers in the ensemble.+If one happens to be down, Solr will automatically be able to send its request to another server in the list.-ZooKeeper version 3.5 and later supports dynamic reconfiguration of server addresses and roles. But note that Solr will only be able to talk to the servers listed in the static ZooKeeper connect string.+ZooKeeper version 3.5 and later supports dynamic reconfiguration of server addresses and roles.+But note that Solr will only be able to talk to the servers listed in the static ZooKeeper connect string.=== Using a chroot@@ -359,13 +392,18 @@ Now you will not have to enter the connection string when starting Solr.== Increasing the File Size Limit-ZooKeeper is designed to hold small files, on the order of kilobytes. By default, ZooKeeper's file size limit is 1MB. Attempting to write or read files larger than this will cause errors.+ZooKeeper is designed to hold small files, on the order of kilobytes.+By default, ZooKeeper's file size limit is 1MB.+Attempting to write or read files larger than this will cause errors.-Some Solr features, e.g., text analysis synonyms, LTR, and OpenNLP named entity recognition, require configuration resources that can be larger than the default limit. ZooKeeper can be configured, via Java system property https://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperAdmin.html#Unsafe+Options[`jute.maxbuffer`], to increase this limit. Note that this configuration, which is required both for ZooKeeper server(s) and for all clients that connect to the server(s), must be the same everywhere it is specified.+Some Solr features, e.g., text analysis synonyms, LTR, and OpenNLP named entity recognition, require configuration resources that can be larger than the default limit.+ZooKeeper can be configured, via Java system property https://zookeeper.apache.org/doc/r{ivy-zookeeper-version}/zookeeperAdmin.html#Unsafe+Options[`jute.maxbuffer`], to increase this limit.+Note that this configuration, which is required both for ZooKeeper server(s) and for all clients that connect to the server(s), must be the same everywhere it is specified.=== Configuring jute.maxbuffer on ZooKeeper Nodes-`jute.maxbuffer` must be configured on each external ZooKeeper node. This can be achieved in any of the following ways; note though that only the first option works on Windows:+`jute.maxbuffer` must be configured on each external ZooKeeper node.+This can be achieved in any of the following ways; note though that only the first option works on Windows:. In `<ZOOKEEPER_HOME>/conf/zoo.cfg`, e.g., to increase the file size limit to one byte less than 10MB, add this line:+@@ -382,7 +420,8 @@ JVMFLAGS="$JVMFLAGS -Djute.maxbuffer=5000000"=== Configuring jute.maxbuffer for ZooKeeper Clients-The `bin/solr` script invokes Java programs that act as ZooKeeper clients. When you use Solr's bundled ZooKeeper server instead of setting up an external ZooKeeper ensemble, the configuration described below will also configure the ZooKeeper server.+The `bin/solr` script invokes Java programs that act as ZooKeeper clients.+When you use Solr's bundled ZooKeeper server instead of setting up an external ZooKeeper ensemble, the configuration described below will also configure the ZooKeeper server.Add the setting to the `SOLR_OPTS` environment variable in Solr's include file (`bin/solr.in.sh` or `solr.in.cmd`):diff --git a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc b/solr/solr-ref-guide/src/zookeeper-file-management.adocsimilarity index 93%rename from solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adocrename to solr/solr-ref-guide/src/zookeeper-file-management.adocindex 44c103a59f7..47a486f4ee4 100644--- a/solr/solr-ref-guide/src/using-zookeeper-to-manage-configuration-files.adoc+++ b/solr/solr-ref-guide/src/zookeeper-file-management.adoc@@ -1,4 +1,4 @@-= Using ZooKeeper to Manage Configuration Files+= ZooKeeper File Management:solr-root-path: ../../:example-source-dir: {solr-root-path}solrj/src/test/org/apache/solr/client/ref_guide_examples/// Licensed to the Apache Software Foundation (ASF) under one@@ -30,7 +30,7 @@ These files are uploaded in either of the following cases:When you try SolrCloud for the first time using the `bin/solr -e cloud`, the related configset gets uploaded to ZooKeeper automatically and is linked with the newly created collection.-The below command would start SolrCloud with the default collection name (gettingstarted) and default configset (_default) uploaded and linked to it.+The below command would start SolrCloud with the default collection name (`gettingstarted`) and default configset (`_default`) uploaded and linked to it.[source,bash]----@@ -93,4 +93,4 @@ If you for example would like to keep your `solr.xml` in ZooKeeper to avoid havibin/solr zk cp file:local/file/path/to/solr.xml zk:/solr.xml -z localhost:2181-----NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<setting-up-an-external-zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.+NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see <<zookeeper-ensemble#updating-solr-include-files,instructions>>) you can omit `-z <zk host string>` from the above command.diff --git a/solr/solr-ref-guide/src/command-line-utilities.adoc b/solr/solr-ref-guide/src/zookeeper-utilities.adocsimilarity index 90%rename from solr/solr-ref-guide/src/command-line-utilities.adocrename to solr/solr-ref-guide/src/zookeeper-utilities.adocindex 80680e4546d..3bf28301d8a 100644--- a/solr/solr-ref-guide/src/command-line-utilities.adoc+++ b/solr/solr-ref-guide/src/zookeeper-utilities.adoc@@ -1,4 +1,4 @@-= Command Line Utilities+= ZooKeeper Utilities// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information@@ -18,11 +18,11 @@A ZooKeeper Command Line Interface (CLI) script is available to allow you to interact directly with Solr configuration files stored in ZooKeeper.-While Solr's Administration UI includes pages dedicated to the state of your SolrCloud cluster, it does not allow you to download or modify related configuration files.+While Solr's Admin UI includes pages dedicated to the state of your SolrCloud cluster, it does not allow you to download or modify related configuration files.TIP: See the section <<cloud-screens.adoc#,Cloud Screens>> for more information about using the Admin UI screens.-The ZooKeeper CLI scripts found in `server/scripts/cloud-scripts` let you upload configuration information to ZooKeeper, in the same ways shown in the examples in <<parameter-reference.adoc#,Parameter Reference>>. It also provides a few other commands that let you link collection sets to collections, make ZooKeeper paths or clear them, and download configurations from ZooKeeper to the local filesystem.+The ZooKeeper CLI script found in `server/scripts/cloud-scripts` let you upload configuration information to ZooKeeper. It also provides a few other commands that let you link collection sets to collections, make ZooKeeper paths or clear them, and download configurations from ZooKeeper to the local filesystem.Many of the functions provided by the zkCli.sh script are also provided by the <<solr-control-script-reference.adoc#,Solr Control Script>>, which may be more familiar as the start script ZooKeeper maintenance commands are very similar to Unix commands.