Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15753: Ensure v2 API requests are logged (#1738)

Jason Gerlowski8784f70f6755
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index 9406ce2df51..83f48e2f223 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -79,6 +79,7 @@ dependencies {
implementation 'org.glassfish.hk2:hk2-api'
implementation 'org.glassfish.hk2.external:jakarta.inject'
implementation 'jakarta.ws.rs:jakarta.ws.rs-api'
+ implementation 'jakarta.annotation:jakarta.annotation-api'
// Non-API below; although there are exceptions
diff --git a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java
index 0bed6862b49..bd4283c2dcf 100644
--- a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java
+++ b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java
@@ -58,7 +58,8 @@ public class JerseyApplications {
register(MessageBodyWriters.XmlMessageBodyWriter.class);
register(MessageBodyWriters.CsvMessageBodyWriter.class);
register(MessageBodyWriters.RawMessageBodyWriter.class);
- register(JacksonJsonProvider.class);
+ register(JacksonJsonProvider.class, 5);
+ register(MessageBodyReaders.CachingJsonMessageBodyReader.class, 10);
register(SolrJacksonMapper.class);
// Request lifecycle logic
@@ -68,6 +69,7 @@ public class JerseyApplications {
register(RequestMetricHandling.PreRequestMetricsFilter.class);
register(RequestMetricHandling.PostRequestMetricsFilter.class);
register(PostRequestDecorationFilter.class);
+ register(PostRequestLoggingFilter.class);
register(
new AbstractBinder() {
@Override
diff --git a/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.java b/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.java
new file mode 100644
index 00000000000..e2ac55fc914
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.jersey;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.annotation.Annotation;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Type;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.MessageBodyReader;
+import javax.ws.rs.ext.Provider;
+import org.glassfish.hk2.api.ServiceLocator;
+import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** A collection point for various {@link MessageBodyReader} implementations. */
+public class MessageBodyReaders {
+
+ /**
+ * A JSON {@link MessageBodyReader} that caches request bodies for use later in the request
+ * lifecycle.
+ *
+ * @see CachingDelegatingMessageBodyReader
+ */
+ @Provider
+ @Consumes(MediaType.APPLICATION_JSON)
+ public static class CachingJsonMessageBodyReader extends CachingDelegatingMessageBodyReader
+ implements MessageBodyReader<Object> {
+ @Override
+ public MessageBodyReader<Object> getDelegate() {
+ return new JacksonJsonProvider();
+ }
+ }
+
+ /**
+ * Caches the deserialized request body in the {@link ContainerRequestContext} for use later in
+ * the request lifecycle.
+ *
+ * <p>This makes the request body accessible to any Jersey response filters or interceptors.
+ *
+ * @see PostRequestLoggingFilter
+ */
+ public abstract static class CachingDelegatingMessageBodyReader
+ implements MessageBodyReader<Object> {
+ public static final String DESERIALIZED_REQUEST_BODY_KEY = "request-body";
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Context ServiceLocator serviceLocator;
+ private final MessageBodyReader<Object> delegate;
+
+ public CachingDelegatingMessageBodyReader() {
+ this.delegate = getDelegate();
+ }
+
+ public abstract MessageBodyReader<Object> getDelegate();
+
+ @Override
+ public boolean isReadable(
+ Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {
+ return delegate.isReadable(type, genericType, annotations, mediaType);
+ }
+
+ @Override
+ public Object readFrom(
+ Class<Object> type,
+ Type genericType,
+ Annotation[] annotations,
+ MediaType mediaType,
+ MultivaluedMap<String, String> httpHeaders,
+ InputStream entityStream)
+ throws IOException, WebApplicationException {
+ final ContainerRequestContext requestContext = getRequestContext();
+ final Object object =
+ delegate.readFrom(type, genericType, annotations, mediaType, httpHeaders, entityStream);
+ if (requestContext != null) {
+ log.info("object is {}", object);
+ requestContext.setProperty(DESERIALIZED_REQUEST_BODY_KEY, object);
+ }
+ return object;
+ }
+
+ private ContainerRequestContext getRequestContext() {
+ return serviceLocator.getService(ContainerRequestContext.class);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java b/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java
index 8735368f879..b7ecd64720c 100644
--- a/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java
+++ b/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java
@@ -17,10 +17,12 @@
package org.apache.solr.jersey;
+import static org.apache.solr.jersey.PostRequestDecorationFilter.PRIORITY;
import static org.apache.solr.jersey.RequestContextKeys.SOLR_QUERY_REQUEST;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import javax.annotation.Priority;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerResponseContext;
import javax.ws.rs.container.ContainerResponseFilter;
@@ -37,7 +39,9 @@ import org.slf4j.LoggerFactory;
*
* @see SolrCore#postDecorateResponse(SolrRequestHandler, SolrQueryRequest, SolrQueryResponse)
*/
+@Priority(PRIORITY)
public class PostRequestDecorationFilter implements ContainerResponseFilter {
+ public static final int PRIORITY = 10;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
diff --git a/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.java b/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.java
new file mode 100644
index 00000000000..f2d35e93573
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.jersey;
+
+import static org.apache.solr.common.params.CommonParams.LOG_PARAMS_LIST;
+import static org.apache.solr.jersey.MessageBodyReaders.CachingDelegatingMessageBodyReader.DESERIALIZED_REQUEST_BODY_KEY;
+import static org.apache.solr.jersey.PostRequestLoggingFilter.PRIORITY;
+import static org.apache.solr.jersey.RequestContextKeys.SOLR_QUERY_REQUEST;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Priority;
+import javax.ws.rs.Path;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MultivaluedMap;
+import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.servlet.HttpSolrCall;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MarkerFactory;
+
+@Priority(PRIORITY)
+public class PostRequestLoggingFilter implements ContainerResponseFilter {
+
+ // Ensures that this filter runs AFTER response decoration, so that we can assume
+ // QTime, etc. have been populated on the response.
+ public static final int PRIORITY = PostRequestDecorationFilter.PRIORITY / 2;
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ // Use SolrCore and HttpSolrCall request loggers to maintain compatibility with logging dashboards
+ // built for v1 APIs
+ private static final Logger coreRequestLogger =
+ LoggerFactory.getLogger(SolrCore.class.getName() + ".Request");
+ private static final Logger slowCoreRequestLogger =
+ LoggerFactory.getLogger(SolrCore.class.getName() + ".SlowRequest");
+ private static final Logger nonCoreRequestLogger =
+ LoggerFactory.getLogger(HttpSolrCall.class.getName());
+
+ @Context private ResourceInfo resourceInfo;
+
+ @Override
+ public void filter(
+ ContainerRequestContext requestContext, ContainerResponseContext responseContext)
+ throws IOException {
+ if (requestContext.getPropertyNames().contains(RequestContextKeys.NOT_FOUND_FLAG)) {
+ return;
+ }
+ if (!responseContext.hasEntity()
+ || !SolrJerseyResponse.class.isInstance(responseContext.getEntity())) {
+ log.debug("Skipping v2 API logging because response is of an unexpected type");
+ return;
+ }
+ final SolrJerseyResponse response = (SolrJerseyResponse) responseContext.getEntity();
+ final SolrQueryRequest solrQueryRequest =
+ (SolrQueryRequest) requestContext.getProperty(SOLR_QUERY_REQUEST);
+ final var solrConfig =
+ (solrQueryRequest.getCore() != null) ? solrQueryRequest.getCore().getSolrConfig() : null;
+
+ final Logger requestLogger = (solrConfig != null) ? coreRequestLogger : nonCoreRequestLogger;
+ final String templatedPath =
+ buildTemplatedPath(requestContext.getUriInfo().getAbsolutePath().getPath());
+ final String bodyVal = buildRequestBodyString(requestContext);
+ requestLogger.info(
+ MarkerFactory.getMarker(templatedPath),
+ "method={} path={} query-params={{}} entity={} status={} QTime={}",
+ requestContext.getMethod(),
+ templatedPath,
+ filterAndStringifyQueryParameters(requestContext.getUriInfo().getQueryParameters()),
+ bodyVal,
+ response.responseHeader.status,
+ response.responseHeader.qTime);
+
+ /* slowQueryThresholdMillis defaults to -1 in SolrConfig -- not enabled.*/
+ if (log.isWarnEnabled()
+ && solrConfig != null
+ && solrConfig.slowQueryThresholdMillis >= 0
+ && response.responseHeader.qTime >= solrConfig.slowQueryThresholdMillis) {
+ slowCoreRequestLogger.warn(
+ MarkerFactory.getMarker(templatedPath),
+ "method={} path={} query-params={{}} entity={} status={} QTime={}",
+ requestContext.getMethod(),
+ templatedPath,
+ filterAndStringifyQueryParameters(requestContext.getUriInfo().getQueryParameters()),
+ response.responseHeader.status,
+ response.responseHeader.qTime);
+ }
+ }
+
+ private String buildTemplatedPath(String fallbackPath) {
+ // We won't have a resource class or method in the case of a 404, so don't try to template out
+ // the path-variables
+ if (resourceInfo == null
+ || resourceInfo.getResourceClass() == null
+ || resourceInfo.getResourceMethod() == null) {
+ return fallbackPath;
+ }
+
+ final var classPathAnnotation = resourceInfo.getResourceClass().getAnnotation(Path.class);
+ final var classPathAnnotationVal =
+ (classPathAnnotation != null) ? classPathAnnotation.value() : "";
+ final var methodPathAnnotation = resourceInfo.getResourceMethod().getAnnotation(Path.class);
+ final var methodPathAnnotationVal =
+ (methodPathAnnotation != null) ? methodPathAnnotation.value() : "";
+
+ return String.format(Locale.ROOT, "%s%s", classPathAnnotationVal, methodPathAnnotationVal)
+ .replaceAll("//", "/");
+ }
+
+ public static String buildRequestBodyString(ContainerRequestContext requestContext) {
+ if (requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY) == null) {
+ return "{}";
+ }
+
+ if (!(requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)
+ instanceof JacksonReflectMapWriter)) {
+ log.warn(
+ "Encountered unexpected request-body type {} for request {}; only {} expected.",
+ requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY).getClass().getName(),
+ requestContext.getUriInfo().getPath(),
+ JacksonReflectMapWriter.class.getName());
+ return "{}";
+ }
+
+ return ((JacksonReflectMapWriter) requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY))
+ .jsonStr()
+ .replace("\n", "");
+ }
+
+ public static String filterAndStringifyQueryParameters(
+ MultivaluedMap<String, String> unfilteredParams) {
+ final var paramNamesToLog = getParamNamesToLog(unfilteredParams);
+ final StringBuilder sb = new StringBuilder(128);
+ unfilteredParams.entrySet().stream()
+ .sorted(Map.Entry.comparingByKey())
+ .forEachOrdered(
+ entry -> {
+ final String name = entry.getKey();
+ if (!paramNamesToLog.contains(name)) return;
+
+ for (String val : entry.getValue()) {
+ if (sb.length() != 0) sb.append('&');
+ StrUtils.partialURLEncodeVal(sb, name);
+ sb.append('=');
+ StrUtils.partialURLEncodeVal(sb, val);
+ }
+ });
+ return sb.toString();
+ }
+
+ private static Set<String> getParamNamesToLog(MultivaluedMap<String, String> queryParameters) {
+ if (CollectionUtil.isEmpty(queryParameters.get(LOG_PARAMS_LIST))) {
+ return queryParameters.keySet();
+ }
+
+ final var paramsToLogStr = queryParameters.getFirst(LOG_PARAMS_LIST);
+ if (StrUtils.isBlank(paramsToLogStr)) {
+ return new HashSet<>(); // A value-less param means that no parameters should be logged
+ }
+
+ return Arrays.stream(paramsToLogStr.split(",")).collect(Collectors.toSet());
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.java b/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.java
new file mode 100644
index 00000000000..c0770c4912e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.jersey;
+
+import static org.apache.solr.jersey.MessageBodyReaders.CachingDelegatingMessageBodyReader.DESERIALIZED_REQUEST_BODY_KEY;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.UriInfo;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.handler.admin.api.CreateReplicaAPI;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link PostRequestLoggingFilter}
+ *
+ * <p>Tests primarily focus on exercising the string operations used to flatten the request into a
+ * log message.
+ */
+public class PostRequestLoggingFilterTest extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void ensureWorkingMockito() {
+ assumeWorkingMockito();
+ }
+
+ @Test
+ public void testBuildQueryParameterString_Simple() {
+ final var queryParams = new MultivaluedHashMap<String, String>();
+ queryParams.putSingle("paramName1", "paramValue1");
+ queryParams.putSingle("paramName2", "paramValue2");
+
+ final var queryParamStr =
+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);
+
+ assertEquals("paramName1=paramValue1&paramName2=paramValue2", queryParamStr);
+ }
+
+ @Test
+ public void testBuildQueryParameterString_DuplicateParams() {
+ final var queryParams = new MultivaluedHashMap<String, String>();
+ queryParams.putSingle("paramName1", "paramValue1");
+ queryParams.put("paramName2", List.of("paramValue2a", "paramValue2b"));
+
+ final var queryParamStr =
+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);
+
+ assertEquals(
+ "paramName1=paramValue1&paramName2=paramValue2a&paramName2=paramValue2b", queryParamStr);
+ }
+
+ @Test
+ public void testBuildQueryParameterString_SpecialChars() {
+ final var queryParams = new MultivaluedHashMap<String, String>();
+ queryParams.putSingle("paramName1", "paramValue1");
+ queryParams.putSingle(
+ "paramName=2", "paramValue=2"); // The name and value themselves contain an equals sign
+
+ final var queryParamStr =
+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);
+
+ assertEquals("paramName1=paramValue1&paramName%3D2=paramValue%3D2", queryParamStr);
+ }
+
+ @Test
+ public void testBuildQueryParameterString_ExcludesFilteredParameters() {
+ final var queryParams = new MultivaluedHashMap<String, String>();
+ queryParams.putSingle("paramName1", "paramValue1");
+ queryParams.putSingle("paramName2", "paramValue2");
+ queryParams.putSingle("hiddenParam1", "hiddenValue1");
+ queryParams.putSingle("hiddenParam2", "hiddenValue2");
+ queryParams.putSingle("logParamsList", "paramName1,paramName2");
+
+ final var queryParamStr =
+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);
+
+ assertEquals("paramName1=paramValue1&paramName2=paramValue2", queryParamStr);
+ }
+
+ @Test
+ public void testRequestBodyStringIsEmptyIfNoRequestBodyFound() {
+ // NOTE: no request body is set on the context.
+ final var mockContext = mock(ContainerRequestContext.class);
+
+ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);
+
+ assertEquals("{}", requestBodyStr);
+ }
+
+ @Test
+ public void testRequestBodyStringIsEmptyIfRequestBodyWasUnexpectedType() {
+ // NOTE: Request body is set, but of an unexpected type (i.e. not a JacksonReflectMapWriter)
+ final var mockContext = mock(ContainerRequestContext.class);
+ final var mockUriInfo = mock(UriInfo.class);
+ when(mockUriInfo.getPath()).thenReturn("/somepath");
+ when(mockContext.getUriInfo()).thenReturn(mockUriInfo);
+ when(mockContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)).thenReturn("unexpectedType");
+
+ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);
+
+ assertEquals("{}", requestBodyStr);
+ }
+
+ @Test
+ public void testRequestBodyRepresentedAsJsonWhenFound() {
+ final var requestBody = new CreateReplicaAPI.AddReplicaRequestBody();
+ requestBody.name = "someReplicaName";
+ requestBody.type = "NRT";
+ requestBody.asyncId = "someAsyncId";
+ final var mockContext = mock(ContainerRequestContext.class);
+ when(mockContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)).thenReturn(requestBody);
+
+ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);
+
+ assertEquals(
+ "{ \"name\":\"someReplicaName\", \"type\":\"NRT\", \"async\":\"someAsyncId\"}",
+ requestBodyStr);
+ }
+}
diff --git a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
index 594e4683535..29fdf0f17e0 100644
--- a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
+++ b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc
@@ -352,10 +352,13 @@ If you do not define the `wt` parameter in your queries, JSON will be returned a
== logParamsList Parameter
-By default, Solr logs all parameters of requests.
-Set this parameter to restrict which parameters of a request are logged.
+By default, Solr logs all query parameters on each request.
+This parameter allows users to override this behavior, by specifying a comma-separated "allowlist" of parameter names that should be logged.
This may help control logging to only those parameters considered important to your organization.
+NOTE: `logParamsList` only governs the logging of query parameters.
+It does not apply to parameters specified in the request path, body, etc.
+
For example, you could define this like:
`logParamsList=q,fq`