Apache Solr open-source search software
SOLR-18197: Add root document query shortcut support to NestPathField (#4512)
Abhishek Umarjikar83422bbb870f
diff --git a/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.yml b/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.ymlnew file mode 100644index 00000000000..646fe64ae83--- /dev/null+++ b/changelog/unreleased/SOLR_18197_nest_path_easy_of_use.yml@@ -0,0 +1,10 @@+title: Add root document query shortcut support to NestPathField+type: added+authors:+ - name: Abhishek Umarjikar+ nick: abumarjikar+ - name: David Smiley+ nick: dsmiley+links:+ - name: SOLR-18197+ url: https://issues.apache.org/jira/browse/SOLR-18197diff --git a/solr/core/src/java/org/apache/solr/schema/NestPathField.java b/solr/core/src/java/org/apache/solr/schema/NestPathField.javaindex d34e532abbb..bac7fa74997 100644--- a/solr/core/src/java/org/apache/solr/schema/NestPathField.java+++ b/solr/core/src/java/org/apache/solr/schema/NestPathField.java@@ -17,13 +17,18 @@package org.apache.solr.schema;-import java.io.IOException;+import java.util.List;import java.util.Map;-import org.apache.lucene.analysis.core.KeywordTokenizerFactory;-import org.apache.lucene.analysis.custom.CustomAnalyzer;-import org.apache.lucene.analysis.pattern.PatternReplaceFilterFactory;-import org.apache.solr.analysis.TokenizerChain;+import java.util.regex.Pattern;+import org.apache.lucene.document.SortedDocValuesField;+import org.apache.lucene.index.IndexableField;+import org.apache.lucene.search.BooleanClause;+import org.apache.lucene.search.BooleanQuery;+import org.apache.lucene.search.MatchAllDocsQuery;+import org.apache.lucene.search.Query;+import org.apache.lucene.util.BytesRef;import org.apache.solr.common.SolrException;+import org.apache.solr.search.QParser;/*** To be used for field {@link IndexSchema#NEST_PATH_FIELD_NAME} for enhanced nested doc@@ -34,32 +39,82 @@ import org.apache.solr.common.SolrException;* @see org.apache.solr.update.processor.NestedUpdateProcessorFactory* @since 8.0*/-public class NestPathField extends SortableTextField {+public class NestPathField extends StrField {++ private static final Pattern ORDINAL_PATTERN = Pattern.compile("#\\d*");@Overridepublic void setArgs(IndexSchema schema, Map<String, String> args) {- args.putIfAbsent("stored", "false");- args.putIfAbsent("multiValued", "false");- args.putIfAbsent("omitTermFreqAndPositions", "true");- args.putIfAbsent("omitNorms", "true");- args.putIfAbsent("maxCharsForDocValues", "-1");+ args.putIfAbsent("stored", "false"); // flip a default+ args.putIfAbsent("docValues", "true"); // flip a default; necessary for old schemas+ args.putIfAbsent("multiValued", "false"); // flip a default; necessary for old schemas+ args.putIfAbsent("uninvertible", "false"); // flip a default; necessary for old schemassuper.setArgs(schema, args);+ // Doesn't support these flags; perhaps others too.+ // note: we could support STORED if truly useful but why bother given docValues.+ restrictProps(STORED | MULTIVALUED | UNINVERTIBLE | TOKENIZED);+ }++ /**+ * {@inheritDoc} Overridden to manipulate the indexed form (AKA terms), but not the DV form. The+ * value may look something like {@code /childA#0/childB#23} and we want to strip out the+ * pound-digits aspects.+ *+ * <p>The terms index is used for block-join and docValues is used for returning nested docs in+ * the same structure as given.+ *+ * @see #toInternal(String)+ */+ @Override+ public List<IndexableField> createFields(SchemaField field, Object value) {+ // indexed or docValues or both may be false... albeit we strongly recommend both enabled and+ // the testing of disabling either may be non-existent.+ final IndexableField termField = createField(field, value); // calls toInternal++ if (!field.hasDocValues()) { // note: strongly recommended, however+ return termField == null ? List.of() : List.of(termField);+ }++ final BytesRef bytes = getBytesRef(value); // unmodified, unlike the indexed term+ final IndexableField dvField = new SortedDocValuesField(field.getName(), bytes);++ return termField == null ? List.of(dvField) : List.of(termField, dvField);+ }++ @Override+ public String toInternal(String val) {+ return ORDINAL_PATTERN.matcher(val).replaceAll("");+ }++ @Override+ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {+ if (externalVal == null) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Field " + field.getName() + " missing value. Forgot `v` local-param?");+ }- // CustomAnalyzer is easy to use- CustomAnalyzer customAnalyzer;- try {- customAnalyzer =- CustomAnalyzer.builder(schema.getResourceLoader())- .withDefaultMatchVersion(schema.getDefaultLuceneMatchVersion())- .withTokenizer(KeywordTokenizerFactory.class)- .addTokenFilter(- PatternReplaceFilterFactory.class, "pattern", "#\\d*", "replace", "all")- .build();- } catch (IOException e) {- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); // impossible?+ if (externalVal.contains("\\")) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Field " + field.getName() + " query value contains backslashes ('\\').");}- // Solr HTTP Schema APIs don't know about CustomAnalyzer so use TokenizerChain instead- setIndexAnalyzer(new TokenizerChain(customAnalyzer));- // leave queryAnalyzer as literal++ if (externalVal.isEmpty() || "/".equals(externalVal)) {+ return new BooleanQuery.Builder()+ .add(MatchAllDocsQuery.INSTANCE, BooleanClause.Occur.MUST)+ .add(field.getType().getExistenceQuery(parser, field), BooleanClause.Occur.MUST_NOT)+ .build();+ }++ if (!externalVal.startsWith("/") || externalVal.endsWith("/")) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Field "+ + field.getName()+ + " query value must start with a forward slash ('/') and cannot contain a trailing slash.");+ }++ return super.getFieldQuery(parser, field, externalVal);}}diff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.javaindex 0d7383dfc4f..b53d7d81680 100644--- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java+++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinChildQParser.java@@ -90,7 +90,7 @@ public class BlockJoinChildQParser extends BlockJoinParentQParser {}// allParents filter: (*:* -{!prefix f="_nest_path_" v="<parentPath>/"})- // For root: (*:* -_nest_path_:*)+ // For root: {!field f=_nest_path_ v='/'}final Query allParentsFilter = buildAllParentsFilterFromPath(parentPath);// constrain the parent query to only match docs at exactly parentPathdiff --git a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.javaindex 6e7dbaf4c63..6cd34424b95 100644--- a/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java+++ b/solr/core/src/java/org/apache/solr/search/join/BlockJoinParentQParser.java@@ -171,11 +171,11 @@ public class BlockJoinParentQParser extends FiltersQParser {if (parsedChildQuery.clauses().isEmpty()) { // i.e. all children// no block-join needed; just return all "parent" docs at this level- return wrapWithParentPathConstraint(parentPath, new MatchAllDocsQuery());+ return wrapWithParentPathConstraint(parentPath, MatchAllDocsQuery.INSTANCE);}// allParents filter: (*:* -{!prefix f="_nest_path_" v="<parentPath>/"})- // For root: (*:* -_nest_path_:*)+ // For root: {!field f=_nest_path_ v='/'}final Query allParentsFilter = buildAllParentsFilterFromPath(parentPath);// constrain child query: (+<original_child> +{!prefix f="_nest_path_" v="<parentPath>/"})@@ -206,18 +206,19 @@ public class BlockJoinParentQParser extends FiltersQParser {* </ul>** <p>Equivalent to: {@code (*:* -{!prefix f="_nest_path_" v="<parentPath>/"})} For root ({@code- * /}): {@code (*:* -_nest_path_:*)}+ * /}): {@code {!field f=_nest_path_ v='/'}}*/protected Query buildAllParentsFilterFromPath(String parentPath) {- final Query excludeQuery;- if (parentPath.equals("/")) {- excludeQuery = newNestPathExistsQuery();- } else {- excludeQuery = new PrefixQuery(new Term(IndexSchema.NEST_PATH_FIELD_NAME, parentPath + "/"));+ if (parentPath.equals("/") || parentPath.isEmpty()) {+ final SchemaField nestPathField = req.getSchema().getField(IndexSchema.NEST_PATH_FIELD_NAME);+ return nestPathField.getType().getFieldQuery(this, nestPathField, parentPath);}+return new BooleanQuery.Builder()- .add(new MatchAllDocsQuery(), Occur.MUST)- .add(excludeQuery, Occur.MUST_NOT)+ .add(MatchAllDocsQuery.INSTANCE, Occur.MUST)+ .add(+ new PrefixQuery(new Term(IndexSchema.NEST_PATH_FIELD_NAME, parentPath + "/")),+ Occur.MUST_NOT).build();}diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.javaindex ecad83f39de..b8895fe7b18 100644--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java@@ -24,6 +24,7 @@ import org.apache.lucene.index.Term;import org.apache.lucene.search.BooleanClause;import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.FuzzyQuery;+import org.apache.lucene.search.MatchAllDocsQuery;import org.apache.lucene.search.NamedMatches;import org.apache.lucene.search.Query;import org.apache.lucene.search.TermInSetQuery;@@ -2000,6 +2001,29 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {"{!hash_range l='107347968' u='214695935' f='x_id'}");}+ @Test+ public void testNestPathRootShortcut() throws Exception {+ try (SolrQueryRequest req = req("df", "_nest_path_")) {+ Query parsedQ =+ assertQueryEqualsAndReturn(+ null, req, "{!field f=_nest_path_ v=''}", "{!field f=_nest_path_}/");++ var schemaField = req.getSchema().getField("_nest_path_");+ Query expectedQ =+ new BooleanQuery.Builder()+ .add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST)+ .add(+ schemaField.getType().getExistenceQuery(null, schemaField),+ BooleanClause.Occur.MUST_NOT)+ .build();++ assertEquals(+ "The root shortcut query did not form the expected match-all minus existence structure",+ expectedQ,+ parsedQ);+ }+ }+// Override req to add df parampublic static SolrQueryRequest req(String... q) {return SolrTestCaseJ4.req(q, "df", "text");diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.javaindex fca48d5765a..d5482e52c3f 100644--- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java+++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java@@ -690,8 +690,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 {"child_q", "(+" + inner_child_query + " +_nest_path_:*)");} else {return params(- "q",- "{!parent which='(*:* -_nest_path_:*)'}(+" + inner_child_query + " +_nest_path_:*)");+ "q", "{!parent which=_nest_path_:\\/}(+" + inner_child_query + " +_nest_path_:*)");}} // else...@@ -786,7 +785,9 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 {} else {return params("q",- "{!child of='(*:* -_nest_path_:*)'}(+" + inner_parent_query + " -_nest_path_:*)");+ "{!child of='{!field f=_nest_path_ v=/}'}(+"+ + inner_parent_query+ + " -_nest_path_:*)");}} // else...diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adocindex fd9a916a88a..c91a362f740 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc@@ -539,7 +539,7 @@ Here is an example of a `knn` search using a `childrenOf`:[source,text]?q={!knn f=vector topK=3 childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0]-&allParents=*:* -_nest_path_:*+&allParents={!field f=_nest_path_ v='/'}The search results retrieved are the k=3 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, each of them with a different parent. The 'childrenOf' parameter must return all valid parents to guarantee the correct functioning of the query.@@ -560,7 +560,7 @@ Here is an example of a `knn` search using a `parents.preFilter`:[source,text]?q={!knn f=vector topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0]-&allParents=*:* -_nest_path_:*+&allParents={!field f=_nest_path_ v='/'}&someParents=color_s:REDThe search results retrieved are the k=3 nearest documents to the vector in input `[1.0, 2.0, 3.0, 4.0]`, each of them with a different parent. Only the documents with a parent that satisfy the 'color_s:RED' condition are considered candidates for the ANN search.@@ -605,7 +605,7 @@ So you should query a multivalued vector fields following the same syntax:[source,text]?q={!parent which=$allParents score=max v=$children.q}&children.q={!knn f=vector_multivalued topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0]-&allParents=*:* -_nest_path_:*+&allParents={!field f=_nest_path_ v='/'}&someParents=color_s:REDIn terms of rendering the results, you need the child transformer if you want to output them flat (you can choose to only return the best vector per result or all vectors):diff --git a/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc b/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adocindex 81220d51318..879663b17ac 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/searching-nested-documents.adoc@@ -274,6 +274,21 @@ $ curl 'http://localhost:8983/solr/gettingstarted/select' -d 'omitHeader=true' -}}----+=== Root Document Query Shortcut++When working with hierarchical nested documents, you may frequently need to isolate or filter your search results exclusively to top-level "root" documents (documents that do not belong to a parent path layout).++Instead of using a verbose negative field existence filter (like `*:* -_nest_path_:*`), the `NestPathField` type supports an explicit root document query shortcut. Passing either a single forward slash (`/`) or an empty string (`''`) automatically constructs a query matching all documents while excluding any nested child paths:++[source,text]+----+# Targets only top-level root documents via the slash shortcut+fq={!field f=_nest_path_}/++# Alternative equivalent syntax using an empty parameter string+fq={!field f=_nest_path_ v=''}+----+=== Nested Vectors search through Block Join Query Parsers and Child Doc Transformer@@ -293,7 +308,7 @@ An example:[source,text]?q={!parent which=$allParents score=max v=$children.q}&children.q={!knn f=vector topK=3 parents.preFilter=$someParents childrenOf=$allParents}[1.0, 2.0, 3.0, 4.0]&-allParents=*:* -_nest_path_:*&+allParents={!field f=_nest_path_ v=''}&someParents=color_s:RED&fl=id,score,vectors,vector,[child fl=vector childFilter=$children.q]