Apache Solr open-source search software
SOLR-5707: Lucene Expressions via ExpressionValueSourceParser (#1244)
Kevin Risdenc3b5f57cd068
diff --git a/gradle/documentation/pull-lucene-javadocs.gradle b/gradle/documentation/pull-lucene-javadocs.gradleindex e3fcaecef25..aa6990ebf8b 100644--- a/gradle/documentation/pull-lucene-javadocs.gradle+++ b/gradle/documentation/pull-lucene-javadocs.gradle@@ -41,11 +41,12 @@ configure(project(":solr:documentation")) {// - For now this list is focused solely on the javadocs needed for ref-guide link validation.// - If/when additional links are added from the ref-guide to additional lucene modules not listed here,// they can be added.- // - If/when we need the lucene javadocs for "all" lucene depdencies in Solr (ie: to do link checking- // from all Solr javadocs?) then perhaps we can find a way to build this list programatically?- // - If these javadocs are (only every) consumed by the ref guide only, then these deps & associated tasks+ // - If/when we need the lucene javadocs for "all" lucene dependencies in Solr (ie: to do link checking+ // from all Solr javadocs?) then perhaps we can find a way to build this list programmatically?+ // - If these javadocs are only consumed by the ref guide, then these deps & associated tasks// should just be moved to the ref-guide build.gradlejavadocs variantOf(libs.apache.lucene.core) { classifier 'javadoc' }+ javadocs variantOf(libs.apache.lucene.expressions) { classifier 'javadoc' }javadocs variantOf(libs.apache.lucene.analysis.common) { classifier 'javadoc' }javadocs variantOf(libs.apache.lucene.analysis.stempel) { classifier 'javadoc' }javadocs variantOf(libs.apache.lucene.queryparser) { classifier 'javadoc' }@@ -65,7 +66,7 @@ configure(project(":solr:documentation")) {def resolved = configurations.javadocs.resolvedConfigurationresolved.resolvedArtifacts.each { artifact ->def id = artifact.moduleVersion.id- // This mimics the directory stucture used on lucene.apache.org for the javadocs of all modules.+ // This mimics the directory structure used on lucene.apache.org for the javadocs of all modules.//// HACK: the lucene.apache.org javadocs are organized to match the module directory structure in the repo,// not the "flat" artifact names -- so there is no one size fits all way to determine the directory name.diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex dfad1fcea13..85b4be69d12 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -223,10 +223,6 @@ Other Changes================== 9.9.0 ==================New Features----------------------* SOLR-17582: The CLUSTERSTATUS API will now stream each collection's status to the response,- fetching and computing it on the fly. To avoid a backwards compatibility concern, this won't work- for wt=javabin. (Matthew Biscocho, David Smiley)-* SOLR-17626: Add RawTFSimilarityFactory class. (Christine Poerschke)* SOLR-17656: New 'skipLeaderRecovery' replica property allows PULL replicas with existing indexes to immediately become ACTIVE (hossman)@@ -243,6 +239,10 @@ New Features* SOLR-17749: Added linear function support for RankField via RankQParserPlugin. (Christine Poerschke)+* SOLR-5707: New ExpressionValueSourceParser that allows custom function queries / VSPs to be defined in a+ subset of JavaScript, pre-compiled, and that which can access the score and fields. It's powered by+ the Lucene Expressions module. (hossman, David Smiley, Ryan Ernst, Kevin Risden)+Improvements---------------------* SOLR-15751: The v2 API now has parity with the v1 "COLSTATUS" and "segments" APIs, which can be used to fetch detailed information about@@ -291,6 +291,10 @@ ImprovementsOptimizations---------------------+* SOLR-17582: The CLUSTERSTATUS API will now stream each collection's status to the response,+ fetching and computing it on the fly. To avoid a backwards compatibility concern, this won't work+ for wt=javabin. (Matthew Biscocho, David Smiley)+* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)* SOLR-17669: Reduced memory usage in SolrJ getBeans() method when handling dynamic fields with wildcards. (Martin Anzinger)diff --git a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.javaindex 636212bc185..21d1434b64e 100644--- a/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java+++ b/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java@@ -22,9 +22,11 @@ import java.util.List;import java.util.Map;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.ReaderUtil;+import org.apache.lucene.internal.hppc.IntFloatHashMap;import org.apache.lucene.internal.hppc.IntObjectHashMap;import org.apache.lucene.queries.function.FunctionValues;import org.apache.lucene.queries.function.ValueSource;+import org.apache.lucene.search.Scorable;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrException;import org.apache.solr.response.ResultContext;@@ -68,20 +70,44 @@ public class ValueSourceAugmenter extends DocTransformer {fcontext = ValueSource.newContext(searcher);this.valueSource.createWeight(fcontext, searcher);final var docList = context.getDocList();- if (docList == null) {+ final int prefetchSize = docList == null ? 0 : Math.min(docList.size(), maxPrefetchSize);+ if (prefetchSize == 0) {return;}- final int prefetchSize = Math.min(docList.size(), maxPrefetchSize);+ // Check if scores are wanted and initialize the Scorable if so+ final MutableScorable scorable; // stored in fcontext (when not null)+ final IntFloatHashMap docToScoreMap;+ if (context.wantsScores()) { // TODO switch to ValueSource.needsScores once it exists+ docToScoreMap = new IntFloatHashMap(prefetchSize);+ scorable =+ new MutableScorable() {+ @Override+ public float score() throws IOException {+ return docToScoreMap.get(docBase + localDocId);+ }+ };+ fcontext.put("scorer", scorable);+ } else {+ scorable = null;+ docToScoreMap = null;+ }++ // Get the IDs and scoresfinal int[] ids = new int[prefetchSize];int i = 0;var iter = docList.iterator();while (iter.hasNext() && i < prefetchSize) {- ids[i++] = iter.nextDoc();+ ids[i] = iter.nextDoc();+ if (docToScoreMap != null) {+ docToScoreMap.put(ids[i], iter.score());+ }+ i++;}Arrays.sort(ids);- cachedValuesById = new IntObjectHashMap<>(ids.length);+ // Get the values in docId order. Store in cachedValuesById+ cachedValuesById = new IntObjectHashMap<>(ids.length);FunctionValues values = null;int docBase = -1;int nextDocBase = 0; // i.e. this segment's maxDoc@@ -95,9 +121,16 @@ public class ValueSourceAugmenter extends DocTransformer {}int localId = docid - docBase;- var value = values.objectVal(localId);++ if (scorable != null) {+ scorable.docBase = docBase;+ scorable.localDocId = localId;+ }+ var value = values.objectVal(localId); // note: might use the Scorable+cachedValuesById.put(docid, value != null ? value : NULL_SENTINEL);}+ fcontext.remove("scorer"); // remove ours; it was there only for prefetching} catch (IOException e) {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "exception for valuesource " + valueSource, e);@@ -119,8 +152,13 @@ public class ValueSourceAugmenter extends DocTransformer {try {int idx = ReaderUtil.subIndex(docid, readerContexts);LeafReaderContext rcontext = readerContexts.get(idx);- FunctionValues values = valueSource.getValues(fcontext, rcontext);int localId = docid - rcontext.docBase;++ if (context.wantsScores()) {+ fcontext.put("scorer", new ScoreAndDoc(localId, docIterationInfo.score()));+ }++ FunctionValues values = valueSource.getValues(fcontext, rcontext);setValue(doc, values.objectVal(localId));} catch (IOException e) {throw new SolrException(@@ -131,6 +169,17 @@ public class ValueSourceAugmenter extends DocTransformer {}}+ private abstract static class MutableScorable extends Scorable {++ int docBase;+ int localDocId;++ @Override+ public int docID() {+ return localDocId;+ }+ }+/** Always returns true */@Overridepublic boolean needsSolrIndexSearcher() {@@ -142,4 +191,25 @@ public class ValueSourceAugmenter extends DocTransformer {doc.setField(name, val);}}++ /** Fake scorer for a single document */+ protected static class ScoreAndDoc extends Scorable {+ final int docid;+ final float score;++ ScoreAndDoc(int docid, float score) {+ this.docid = docid;+ this.score = score;+ }++ @Override+ public int docID() {+ return docid;+ }++ @Override+ public float score() throws IOException {+ return score;+ }+ }}diff --git a/solr/core/src/java/org/apache/solr/search/ExpressionValueSourceParser.java b/solr/core/src/java/org/apache/solr/search/ExpressionValueSourceParser.javanew file mode 100644index 00000000000..3812357c3d9--- /dev/null+++ b/solr/core/src/java/org/apache/solr/search/ExpressionValueSourceParser.java@@ -0,0 +1,155 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;++import java.text.ParseException;+import java.util.ArrayList;+import java.util.List;+import java.util.Objects;+import java.util.Optional;+import java.util.regex.Matcher;+import java.util.regex.Pattern;+import org.apache.lucene.expressions.Bindings;+import org.apache.lucene.expressions.Expression;+import org.apache.lucene.expressions.js.JavascriptCompiler;+import org.apache.lucene.queries.function.ValueSource;+import org.apache.lucene.search.DoubleValuesSource;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.schema.IndexSchema;+import org.apache.solr.schema.SchemaField;++/**+ * A ValueSource parser configured with a pre-compiled expression that can then be evaluated at+ * request time. It's powered by the Lucene Expressions module, which is a subset of JavaScript.+ */+public class ExpressionValueSourceParser extends ValueSourceParser {++ public static final String SCORE_KEY = "score-name"; // TODO get rid of this? Why have it?+ public static final String EXPRESSION_KEY = "expression";++ private Expression expression;+ private String scoreKey;+ private int numPositionalArgs = 0; // Number of positional arguments in the expression++ @Override+ public void init(NamedList<?> args) {+ initConfiguredExpression(args);+ initScoreKey(args);+ super.init(args);+ }++ /** Checks for optional scoreKey override */+ private void initScoreKey(NamedList<?> args) {+ scoreKey = Optional.ofNullable((String) args.remove(SCORE_KEY)).orElse(SolrReturnFields.SCORE);+ }++ /** Parses the pre-configured expression */+ private void initConfiguredExpression(NamedList<?> args) {+ String expressionStr =+ Optional.ofNullable((String) args.remove(EXPRESSION_KEY))+ .orElseThrow(+ () ->+ new SolrException(+ SERVER_ERROR, EXPRESSION_KEY + " must be configured with an expression"));++ // Find the highest positional argument in the expression+ Pattern pattern = Pattern.compile("\\$(\\d+)");+ Matcher matcher = pattern.matcher(expressionStr);+ while (matcher.find()) {+ int argNum = Integer.parseInt(matcher.group(1));+ numPositionalArgs = Math.max(numPositionalArgs, argNum);+ }++ // TODO add way to register additional functions+ try {+ this.expression = JavascriptCompiler.compile(expressionStr);+ } catch (ParseException e) {+ throw new SolrException(+ SERVER_ERROR, "Unable to parse javascript expression: " + expressionStr, e);+ }+ }++ // TODO: support dynamic expressions: expr("foo * bar / 32") ??++ @Override+ public ValueSource parse(FunctionQParser fp) throws SyntaxError {+ assert null != fp;++ // Parse positional arguments if any+ List<DoubleValuesSource> positionalArgs = new ArrayList<>();+ for (int i = 0; i < numPositionalArgs; i++) {+ ValueSource vs = fp.parseValueSource();+ positionalArgs.add(vs.asDoubleValuesSource());+ }++ IndexSchema schema = fp.getReq().getSchema();+ SolrBindings b = new SolrBindings(scoreKey, schema, positionalArgs);+ return ValueSource.fromDoubleValuesSource(expression.getDoubleValuesSource(b));+ }++ /**+ * A bindings class that uses schema fields to resolve variables.+ *+ * @lucene.internal+ */+ public static class SolrBindings extends Bindings {+ private final String scoreKey;+ private final IndexSchema schema;+ private final List<DoubleValuesSource> positionalArgs;++ /**+ * @param scoreKey The binding name that should be used to represent the score, may be null+ * @param schema IndexSchema for field bindings+ * @param positionalArgs List of positional arguments+ */+ public SolrBindings(+ String scoreKey, IndexSchema schema, List<DoubleValuesSource> positionalArgs) {+ this.scoreKey = scoreKey;+ this.schema = schema;+ this.positionalArgs = positionalArgs != null ? positionalArgs : new ArrayList<>();+ }++ @Override+ public DoubleValuesSource getDoubleValuesSource(String key) {+ assert null != key;++ if (Objects.equals(scoreKey, key)) {+ return DoubleValuesSource.SCORES;+ }++ // Check for positional arguments like $1, $2, etc.+ if (key.startsWith("$")) {+ try {+ int position = Integer.parseInt(key.substring(1));+ return positionalArgs.get(position - 1); // Convert to 0-based index+ } catch (RuntimeException e) {+ throw new IllegalArgumentException("Not a valid positional argument: " + key, e);+ }+ }++ SchemaField field = schema.getFieldOrNull(key);+ if (null != field) {+ return field.getType().getValueSource(field, null).asDoubleValuesSource();+ }++ throw new IllegalArgumentException("No binding or schema field for key: " + key);+ }+ }+}diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-expressions-vs.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-expressions-vs.xmlnew file mode 100644index 00000000000..9081d634eff--- /dev/null+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-expressions-vs.xml@@ -0,0 +1,76 @@+<?xml version="1.0" ?>++<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->++<config>+ <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>++ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>+ <schemaFactory class="ClassicIndexSchemaFactory"/>++ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>++ <requestHandler name="standard" class="solr.StandardRequestHandler"/>+ <requestHandler name="/update" class="solr.UpdateRequestHandler" />++ <valueSourceParser name="sin1" class="solr.ExpressionValueSourceParser">+ <str name="expression">sin(1)</str>+ </valueSourceParser>++ <valueSourceParser name="cos_sin1" class="solr.ExpressionValueSourceParser">+ <str name="expression">cos(sin(1))</str>+ </valueSourceParser>++ <valueSourceParser name="sqrt_int1_i" class="solr.ExpressionValueSourceParser">+ <str name="expression">sqrt(int1_i)</str>+ </valueSourceParser>++ <valueSourceParser name="sqrt_double1_d" class="solr.ExpressionValueSourceParser">+ <str name="expression">sqrt(double1_d)</str>+ </valueSourceParser>++ <valueSourceParser name="date1_dt_minus_1990" class="solr.ExpressionValueSourceParser">+ <str name="expression">date1_dt - 631036800000</str>+ </valueSourceParser>++ <valueSourceParser name="one_plus_score" class="solr.ExpressionValueSourceParser">+ <str name="expression">1 + score</str>+ </valueSourceParser>++ <valueSourceParser name="two_plus_score" class="solr.ExpressionValueSourceParser">+ <str name="expression">1 + score + 1</str>+ </valueSourceParser>++ <valueSourceParser name="sqrt_int1_i_plus_one_plus_score" class="solr.ExpressionValueSourceParser">+ <str name="expression">sqrt(int1_i) + 1 + score</str>+ </valueSourceParser>++ <valueSourceParser name="mixed_expr" class="solr.ExpressionValueSourceParser">+ <str name="expression">(1 + score)*ln(cos(sin(1)))</str>+ </valueSourceParser>++ <valueSourceParser name="expr_ssccoorree" class="solr.ExpressionValueSourceParser">+ <!-- unclear if anyone needs this capability -->+ <str name="score-name">ssccoorree</str>+ <str name="expression">1 + ssccoorree</str>+ </valueSourceParser>++ <valueSourceParser name="positional_args" class="solr.ExpressionValueSourceParser">+ <str name="expression">$1 + $2 + $3</str>+ </valueSourceParser>+</config>diff --git a/solr/core/src/test/org/apache/solr/search/ExpressionValueSourceParserTest.java b/solr/core/src/test/org/apache/solr/search/ExpressionValueSourceParserTest.javanew file mode 100644index 00000000000..97f9c4d7418--- /dev/null+++ b/solr/core/src/test/org/apache/solr/search/ExpressionValueSourceParserTest.java@@ -0,0 +1,346 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.search;++import java.text.ParseException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.queries.function.ValueSource;+import org.apache.lucene.search.DoubleValuesSource;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.schema.IndexSchema;+import org.apache.solr.search.ExpressionValueSourceParser.SolrBindings;+import org.apache.solr.util.DateMathParser;+import org.junit.BeforeClass;++public class ExpressionValueSourceParserTest extends SolrTestCaseJ4 {++ private final List<DoubleValuesSource> positionalArgs = new ArrayList<>();++ @BeforeClass+ public static void beforeTests() throws Exception {+ initCore("solrconfig-expressions-vs.xml", "schema15.xml");++ assertU(+ adoc("id", "1", "int1_i", "50", "double1_d", "-2.5", "date1_dt", "1996-12-19T16:39:57Z"));+ assertU(+ adoc("id", "2", "int1_i", "-30", "double1_d", "10.3", "date1_dt", "1999-12-19T16:39:57Z"));+ assertU(+ adoc("id", "3", "int1_i", "10", "double1_d", "500.3", "date1_dt", "1995-12-19T16:39:57Z"));+ assertU(adoc("id", "4", "int1_i", "40", "double1_d", "-1", "date1_dt", "1994-12-19T16:39:57Z"));+ assertU(+ adoc("id", "5", "int1_i", "20", "double1_d", "2.1", "date1_dt", "1997-12-19T16:39:57Z"));++ assertU(commit());+ }++ public void testValidBindings() throws ParseException {+ IndexSchema schema = h.getCore().getLatestSchema();+ SolrBindings bindings = new SolrBindings("ScOrE", schema, this.positionalArgs);++ assertEquals(+ "foo_i from bindings is wrong",+ schema+ .getFieldType("foo_i")+ .getValueSource(schema.getField("foo_i"), null)+ .asDoubleValuesSource(),+ bindings.getDoubleValuesSource("foo_i"));+ ValueSource scoreBind =+ ValueSource.fromDoubleValuesSource(bindings.getDoubleValuesSource("ScOrE"));+ assertNotNull("ScOrE bindings failed", scoreBind);++ try {+ bindings.getDoubleValuesSource("not_a_expr_and_not_in_schema");+ fail("no exception from bogus binding");+ } catch (IllegalArgumentException e) {+ assertTrue(+ "wrong exception message: " + e.getMessage(),+ e.getMessage().contains("not_a_expr_and_not_in_schema"));+ }++ // change things up a bit+ bindings = new SolrBindings(null, schema, this.positionalArgs);+ try {+ bindings.getDoubleValuesSource("ScOrE");+ fail("ScOrE should not have bindings");+ } catch (IllegalArgumentException e) {+ // NOOP+ }+ }++ public void testBogusBindings() throws ParseException {+ IndexSchema schema = h.getCore().getLatestSchema();+ SolrBindings bindings = new SolrBindings("ScOrE", schema, this.positionalArgs);++ try {+ bindings.getDoubleValuesSource("yak");+ fail("sanity check failed: yak has a binding?");+ } catch (IllegalArgumentException e) {+ // NOOP+ }++ // change things up a bit+ bindings = new SolrBindings(null, schema, this.positionalArgs);+ try {+ bindings.getDoubleValuesSource("ScOrE");+ fail("ScOrE should not have bindings");+ } catch (IllegalArgumentException e) {+ // NOOP+ }+ try {+ bindings.getDoubleValuesSource("score");+ fail("score should not have bindings");+ } catch (IllegalArgumentException e) {+ // NOOP+ }+ }++ /** tests an expression referring to a score field using an overridden score binding */+ public void testSortSsccoorree() {+ assertQ(+ "sort",+ req(+ "fl", "id",+ "q", "{!func}field(int1_i)",+ "sort", "expr_ssccoorree() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='1']",+ "//result/doc[2]/str[@name='id'][.='4']",+ "//result/doc[3]/str[@name='id'][.='5']",+ "//result/doc[4]/str[@name='id'][.='3']",+ "//result/doc[5]/str[@name='id'][.='2']");+ }++ /** tests a constant expression */+ public void testSortConstant() {+ assertQ(+ "sort",+ req("fl", "id", "q", "*:*", "sort", "sin1() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='1']",+ "//result/doc[2]/str[@name='id'][.='2']",+ "//result/doc[3]/str[@name='id'][.='3']",+ "//result/doc[4]/str[@name='id'][.='4']",+ "//result/doc[5]/str[@name='id'][.='5']");+ }++ // Removed testSortExpression as expressions can no longer reference other expressions++ /** tests an expression referring to an int field */+ public void testSortInt() {+ assertQ(+ "sort",+ req("fl", "id", "q", "*:*", "sort", "sqrt_int1_i() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='2']", // NaN+ "//result/doc[2]/str[@name='id'][.='1']",+ "//result/doc[3]/str[@name='id'][.='4']",+ "//result/doc[4]/str[@name='id'][.='5']",+ "//result/doc[5]/str[@name='id'][.='3']");+ }++ /** tests an expression referring to a double field */+ public void testSortDouble() {+ assertQ(+ "sort",+ req("fl", "id", "q", "*:*", "sort", "sqrt_double1_d() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='1']", // NaN+ "//result/doc[2]/str[@name='id'][.='4']", // NaN+ "//result/doc[3]/str[@name='id'][.='3']",+ "//result/doc[4]/str[@name='id'][.='2']",+ "//result/doc[5]/str[@name='id'][.='5']");+ }++ /** tests an expression referring to a date field */+ public void testSortDate() {+ assertQ(+ "sort",+ req("fl", "id", "q", "*:*", "sort", "date1_dt_minus_1990() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='2']",+ "//result/doc[2]/str[@name='id'][.='5']",+ "//result/doc[3]/str[@name='id'][.='1']",+ "//result/doc[4]/str[@name='id'][.='3']",+ "//result/doc[5]/str[@name='id'][.='4']");+ }++ /** tests an expression referring to a score field */+ public void testSortScore() {+ assertQ(+ "sort",+ req("fl", "id", "q", "{!func}field(int1_i)", "sort", "one_plus_score() desc,id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/str[@name='id'][.='1']",+ "//result/doc[2]/str[@name='id'][.='4']",+ "//result/doc[3]/str[@name='id'][.='5']",+ "//result/doc[4]/str[@name='id'][.='3']",+ "//result/doc[5]/str[@name='id'][.='2']");+ }++ /** tests a constant expression */+ public void testReturnConstant() {+ final float expected = (float) Math.sin(1);+ assertQ(+ "return",+ req("fl", "sin1:sin1()", "q", "*:*", "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='sin1'][.='" + expected + "']",+ "//result/doc[2]/float[@name='sin1'][.='" + expected + "']",+ "//result/doc[3]/float[@name='sin1'][.='" + expected + "']",+ "//result/doc[4]/float[@name='sin1'][.='" + expected + "']",+ "//result/doc[5]/float[@name='sin1'][.='" + expected + "']");+ }++ // Removed testReturnExpression as expressions can no longer reference other expressions++ /** tests an expression referring to an int field */+ public void testReturnInt() {+ assertQ(+ "return",+ req("fl", "foo:sqrt_int1_i()", "q", "*:*", "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='foo'][.=" + (float) Math.sqrt(50) + "]",+ "//result/doc[2]/float[@name='foo'][.='NaN']",+ "//result/doc[3]/float[@name='foo'][.=" + (float) Math.sqrt(10) + "]",+ "//result/doc[4]/float[@name='foo'][.=" + (float) Math.sqrt(40) + "]",+ "//result/doc[5]/float[@name='foo'][.=" + (float) Math.sqrt(20) + "]");+ }++ /** tests an expression referring to a double field */+ public void testReturnDouble() {+ assertQ(+ "return",+ req("fl", "bar:sqrt_double1_d()", "q", "*:*", "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='bar'][.='NaN']",+ "//result/doc[2]/float[@name='bar'][.=" + (float) Math.sqrt(10.3d) + "]",+ "//result/doc[3]/float[@name='bar'][.=" + (float) Math.sqrt(500.3d) + "]",+ "//result/doc[4]/float[@name='bar'][.='NaN']",+ "//result/doc[5]/float[@name='bar'][.=" + (float) Math.sqrt(2.1d) + "]");+ }++ /** tests an expression referring to a date field */+ public void testReturnDate() {+ assertQ(+ "return",+ req("fl", "date1_dt_minus_1990:date1_dt_minus_1990()", "q", "*:*", "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='date1_dt_minus_1990'][.='"+ + (float)+ (DateMathParser.parseMath(null, "1996-12-19T16:39:57Z").getTime() - 631036800000D)+ + "']",+ "//result/doc[2]/float[@name='date1_dt_minus_1990'][.='"+ + (float)+ (DateMathParser.parseMath(null, "1999-12-19T16:39:57Z").getTime() - 631036800000D)+ + "']",+ "//result/doc[3]/float[@name='date1_dt_minus_1990'][.='"+ + (float)+ (DateMathParser.parseMath(null, "1995-12-19T16:39:57Z").getTime() - 631036800000D)+ + "']",+ "//result/doc[4]/float[@name='date1_dt_minus_1990'][.='"+ + (float)+ (DateMathParser.parseMath(null, "1994-12-19T16:39:57Z").getTime() - 631036800000D)+ + "']",+ "//result/doc[5]/float[@name='date1_dt_minus_1990'][.='"+ + (float)+ (DateMathParser.parseMath(null, "1997-12-19T16:39:57Z").getTime() - 631036800000D)+ + "']");+ }++ /** tests an expression referring to score */+ public void testReturnScores() {+ assertQ(+ "return",+ // unfortunately, need to add fl=score for ValueSourceAugmenter to access it+ req(+ "fl", "one_plus_score(),score",+ "q", "{!func}field(int1_i)",+ "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='one_plus_score()'][.='51.0']",+ "//result/doc[2]/float[@name='one_plus_score()'][.='1.0']",+ "//result/doc[3]/float[@name='one_plus_score()'][.='11.0']",+ "//result/doc[4]/float[@name='one_plus_score()'][.='41.0']",+ "//result/doc[5]/float[@name='one_plus_score()'][.='21.0']");+ }++ public void testReturnScores2() {+ assertQ(+ "return",+ // unfortunately, need to add fl=score for ValueSourceAugmenter to access it+ req(+ "fl", "two_plus_score:two_plus_score(),score",+ "q", "{!func}field(int1_i)",+ "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='two_plus_score'][.='52.0']",+ "//result/doc[2]/float[@name='two_plus_score'][.='2.0']",+ "//result/doc[3]/float[@name='two_plus_score'][.='12.0']",+ "//result/doc[4]/float[@name='two_plus_score'][.='42.0']",+ "//result/doc[5]/float[@name='two_plus_score'][.='22.0']");+ }++ public void testReturnScores3() {+ assertQ(+ "return",+ // unfortunately, need to add fl=score for ValueSourceAugmenter to access it+ req(+ "fl", "foo:sqrt_int1_i_plus_one_plus_score(),score",+ "q", "{!func}field(int1_i)",+ "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='foo'][.='" + (float) (Math.sqrt(50) + 1 + 50) + "']",+ "//result/doc[2]/float[@name='foo'][.='NaN']",+ "//result/doc[3]/float[@name='foo'][.='" + (float) (Math.sqrt(10) + 1 + 10) + "']",+ "//result/doc[4]/float[@name='foo'][.='" + (float) (Math.sqrt(40) + 1 + 40) + "']",+ "//result/doc[5]/float[@name='foo'][.='" + (float) (Math.sqrt(20) + 1 + 20) + "']");+ }++ /** tests an expression with positional arguments */+ public void testPositionalArgs() {+ assertQ(+ "return",+ req(+ "fl", "sum:positional_args(10,20,30)",+ "q", "*:*",+ "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='sum'][.='60.0']",+ "//result/doc[2]/float[@name='sum'][.='60.0']",+ "//result/doc[3]/float[@name='sum'][.='60.0']",+ "//result/doc[4]/float[@name='sum'][.='60.0']",+ "//result/doc[5]/float[@name='sum'][.='60.0']");+ }++ /** tests embedding an expression in another value source */+ public void testEmbeddedExpression() {+ final float expected = (float) Math.sin(1);+ assertQ(+ "return",+ req(+ "fl", "embedded:sum(sin1(),5)",+ "q", "*:*",+ "sort", "id asc"),+ "//*[@numFound='5']",+ "//result/doc[1]/float[@name='embedded'][.='" + (expected + 5.0f) + "']",+ "//result/doc[2]/float[@name='embedded'][.='" + (expected + 5.0f) + "']",+ "//result/doc[3]/float[@name='embedded'][.='" + (expected + 5.0f) + "']",+ "//result/doc[4]/float[@name='embedded'][.='" + (expected + 5.0f) + "']",+ "//result/doc[5]/float[@name='embedded'][.='" + (expected + 5.0f) + "']");+ }+}diff --git a/solr/documentation/gradle.lockfile b/solr/documentation/gradle.lockfileindex 64bcc86b277..29100635f9f 100644--- a/solr/documentation/gradle.lockfile+++ b/solr/documentation/gradle.lockfile@@ -4,6 +4,7 @@org.apache.lucene:lucene-analysis-common:9.12.2=javadocsorg.apache.lucene:lucene-analysis-stempel:9.12.2=javadocsorg.apache.lucene:lucene-core:9.12.2=javadocs+org.apache.lucene:lucene-expressions:9.12.2=javadocsorg.apache.lucene:lucene-queries:9.12.2=javadocsorg.apache.lucene:lucene-queryparser:9.12.2=javadocsorg.apache.lucene:lucene-spatial-extras:9.12.2=javadocsdiff --git a/solr/solr-ref-guide/gradle.lockfile b/solr/solr-ref-guide/gradle.lockfileindex 83242de546c..9c18839eb18 100644--- a/solr/solr-ref-guide/gradle.lockfile+++ b/solr/solr-ref-guide/gradle.lockfile@@ -89,7 +89,7 @@ org.apache.lucene:lucene-backward-codecs:9.12.2=testRuntimeClasspathorg.apache.lucene:lucene-classification:9.12.2=testRuntimeClasspathorg.apache.lucene:lucene-codecs:9.12.2=testRuntimeClasspathorg.apache.lucene:lucene-core:9.12.2=localJavadocs,testCompileClasspath,testRuntimeClasspath-org.apache.lucene:lucene-expressions:9.12.2=testRuntimeClasspath+org.apache.lucene:lucene-expressions:9.12.2=localJavadocs,testRuntimeClasspathorg.apache.lucene:lucene-facet:9.12.2=testRuntimeClasspathorg.apache.lucene:lucene-grouping:9.12.2=testRuntimeClasspathorg.apache.lucene:lucene-highlighter:9.12.2=testRuntimeClasspathdiff --git a/solr/solr-ref-guide/modules/query-guide/pages/expression-value-source-parser.adoc b/solr/solr-ref-guide/modules/query-guide/pages/expression-value-source-parser.adocnew file mode 100644index 00000000000..ae9a9989a4e--- /dev/null+++ b/solr/solr-ref-guide/modules/query-guide/pages/expression-value-source-parser.adoc@@ -0,0 +1,56 @@+= Expression Value Source Parser+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++The `ExpressionValueSourceParser` allows you to implement a custom valueSource merely by adding a concise JavaScript expression to your `solrconfig.xml`.+The expression is precompiled and offers competitive performance to those written in Java.+The syntax is a limited subset of JavaScript that is purely numerically oriented, and only certain built-in functions can be called.+The implementation is based on the Lucene Expressions module, which you can learn more about in the {lucene-javadocs}/expressions/org/apache/lucene/expressions/js/package-summary.html[Lucene Expressions documentation].++== Examples++Expressions can reference field values directly by field name, the document's score using the special name `score`, and positional arguments as `$1`, `$2`, etc.+The arguments might be constants, fields, or other functions supplying a result.++Here are some example definitions designed to illustrate these features:++[source,xml]+----+<valueSourceParser name="sqrt_popularity" class="solr.ExpressionValueSourceParser">+ <str name="expression">sqrt(popularity)</str>+</valueSourceParser>++<valueSourceParser name="weighted_sum" class="solr.ExpressionValueSourceParser">+ <str name="expression">$1 * 0.8 + $2 * 0.2</str>+</valueSourceParser>++<valueSourceParser name="complex_score" class="solr.ExpressionValueSourceParser">+ <str name="expression">log(sum(popularity,recency)) * max(score,0.1)</str>+</valueSourceParser>+----++Here is one unrealistic query using multiple function queries to illustrate its features:++[source,text]+----+&q=my query+&fq={!frange l=1000}sqrt_popularity()+&sort=complex_score() desc+&fl=id,weighted_sum(field1,field2)+----++Using this VSP to boost/manipulate scores is more efficient than using the `query()` function query, since the latter would execute the underlying `q` an additional time.diff --git a/solr/solr-ref-guide/modules/query-guide/pages/function-queries.adoc b/solr/solr-ref-guide/modules/query-guide/pages/function-queries.adocindex 7c6f1a9d0ea..05abe1a114f 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/function-queries.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/function-queries.adoc@@ -86,6 +86,8 @@ Only functions with fast random access are recommended.The table below summarizes the functions available for function queries.+Additionally, you can write a custom one using a tiny bit of JavaScript with the xref:expression-value-source-parser.adoc[Expression Value Source Parser].+=== abs FunctionReturns the absolute value of the specified value or function.diff --git a/solr/solr-ref-guide/modules/query-guide/querying-nav.adoc b/solr/solr-ref-guide/modules/query-guide/querying-nav.adocindex 973bd80f005..c66b1f85016 100644--- a/solr/solr-ref-guide/modules/query-guide/querying-nav.adoc+++ b/solr/solr-ref-guide/modules/query-guide/querying-nav.adoc@@ -23,6 +23,7 @@** xref:dismax-query-parser.adoc[]** xref:edismax-query-parser.adoc[]** xref:function-queries.adoc[]+*** xref:expression-value-source-parser.adoc[]** xref:local-params.adoc[]** xref:json-request-api.adoc[]*** xref:json-query-dsl.adoc[]diff --git a/solr/solrj-streaming/gradle.lockfile b/solr/solrj-streaming/gradle.lockfileindex 141544a9f1b..943508f2eca 100644--- a/solr/solrj-streaming/gradle.lockfile+++ b/solr/solrj-streaming/gradle.lockfile@@ -177,7 +177,7 @@ org.ow2.asm:asm:9.7.1=jarValidation,testRuntimeClasspathorg.pcollections:pcollections:4.0.1=annotationProcessor,errorprone,testAnnotationProcessororg.semver4j:semver4j:5.3.0=jarValidation,permitTestUsedUndeclared,runtimeClasspath,testRuntimeClasspathorg.slf4j:jcl-over-slf4j:2.0.16=jarValidation,permitTestUsedUndeclared,runtimeClasspath,testRuntimeClasspath-org.slf4j:slf4j-api:1.7.32=permitTestUnusedDeclared+org.slf4j:slf4j-api:1.7.30=permitTestUnusedDeclaredorg.slf4j:slf4j-api:2.0.16=compileClasspath,jarValidation,permitTestUsedUndeclared,runtimeClasspath,testCompileClasspath,testRuntimeClasspathorg.xerial.snappy:snappy-java:1.1.10.5=jarValidation,testRuntimeClasspathempty=apiHelper,apiHelperTest,compileOnlyHelper,compileOnlyHelperTest,missingdoclet,permitAggregatorUse,permitTestAggregatorUse,permitUnusedDeclared,permitUsedUndeclared,signaturesdiff --git a/solr/solrj/gradle.lockfile b/solr/solrj/gradle.lockfileindex e12274f0a74..cccf1fc6802 100644--- a/solr/solrj/gradle.lockfile+++ b/solr/solrj/gradle.lockfile@@ -170,7 +170,7 @@ org.ow2.asm:asm:9.7.1=jarValidation,testRuntimeClasspathorg.pcollections:pcollections:4.0.1=annotationProcessor,errorprone,testAnnotationProcessororg.semver4j:semver4j:5.3.0=apiHelper,jarValidation,runtimeClasspath,testRuntimeClasspathorg.slf4j:jcl-over-slf4j:2.0.16=jarValidation,runtimeClasspath,testRuntimeClasspath-org.slf4j:slf4j-api:1.7.32=permitTestUnusedDeclared+org.slf4j:slf4j-api:1.7.30=permitTestUnusedDeclaredorg.slf4j:slf4j-api:2.0.16=apiHelper,compileClasspath,jarValidation,runtimeClasspath,testCompileClasspath,testRuntimeClasspathorg.xerial.snappy:snappy-java:1.1.10.5=jarValidation,testRuntimeClasspathempty=apiHelperTest,compileOnlyHelperTest,missingdoclet,openApiSpecFile,permitAggregatorUse,permitTestAggregatorUse,permitTestUsedUndeclared,permitUnusedDeclared,permitUsedUndeclared,signatures