Apache Solr open-source search software
SOLR-15753: Ensure v2 API requests are logged (#1738)
Jason Gerlowski8784f70f6755
diff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 9406ce2df51..83f48e2f223 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -79,6 +79,7 @@ dependencies {implementation 'org.glassfish.hk2:hk2-api'implementation 'org.glassfish.hk2.external:jakarta.inject'implementation 'jakarta.ws.rs:jakarta.ws.rs-api'+ implementation 'jakarta.annotation:jakarta.annotation-api'// Non-API below; although there are exceptionsdiff --git a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.javaindex 0bed6862b49..bd4283c2dcf 100644--- a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java+++ b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java@@ -58,7 +58,8 @@ public class JerseyApplications {register(MessageBodyWriters.XmlMessageBodyWriter.class);register(MessageBodyWriters.CsvMessageBodyWriter.class);register(MessageBodyWriters.RawMessageBodyWriter.class);- register(JacksonJsonProvider.class);+ register(JacksonJsonProvider.class, 5);+ register(MessageBodyReaders.CachingJsonMessageBodyReader.class, 10);register(SolrJacksonMapper.class);// Request lifecycle logic@@ -68,6 +69,7 @@ public class JerseyApplications {register(RequestMetricHandling.PreRequestMetricsFilter.class);register(RequestMetricHandling.PostRequestMetricsFilter.class);register(PostRequestDecorationFilter.class);+ register(PostRequestLoggingFilter.class);register(new AbstractBinder() {@Overridediff --git a/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.java b/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.javanew file mode 100644index 00000000000..e2ac55fc914--- /dev/null+++ b/solr/core/src/java/org/apache/solr/jersey/MessageBodyReaders.java@@ -0,0 +1,109 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.jersey;++import java.io.IOException;+import java.io.InputStream;+import java.lang.annotation.Annotation;+import java.lang.invoke.MethodHandles;+import java.lang.reflect.Type;+import javax.ws.rs.Consumes;+import javax.ws.rs.WebApplicationException;+import javax.ws.rs.container.ContainerRequestContext;+import javax.ws.rs.core.Context;+import javax.ws.rs.core.MediaType;+import javax.ws.rs.core.MultivaluedMap;+import javax.ws.rs.ext.MessageBodyReader;+import javax.ws.rs.ext.Provider;+import org.glassfish.hk2.api.ServiceLocator;+import org.glassfish.jersey.jackson.internal.jackson.jaxrs.json.JacksonJsonProvider;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/** A collection point for various {@link MessageBodyReader} implementations. */+public class MessageBodyReaders {++ /**+ * A JSON {@link MessageBodyReader} that caches request bodies for use later in the request+ * lifecycle.+ *+ * @see CachingDelegatingMessageBodyReader+ */+ @Provider+ @Consumes(MediaType.APPLICATION_JSON)+ public static class CachingJsonMessageBodyReader extends CachingDelegatingMessageBodyReader+ implements MessageBodyReader<Object> {+ @Override+ public MessageBodyReader<Object> getDelegate() {+ return new JacksonJsonProvider();+ }+ }++ /**+ * Caches the deserialized request body in the {@link ContainerRequestContext} for use later in+ * the request lifecycle.+ *+ * <p>This makes the request body accessible to any Jersey response filters or interceptors.+ *+ * @see PostRequestLoggingFilter+ */+ public abstract static class CachingDelegatingMessageBodyReader+ implements MessageBodyReader<Object> {+ public static final String DESERIALIZED_REQUEST_BODY_KEY = "request-body";++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ @Context ServiceLocator serviceLocator;+ private final MessageBodyReader<Object> delegate;++ public CachingDelegatingMessageBodyReader() {+ this.delegate = getDelegate();+ }++ public abstract MessageBodyReader<Object> getDelegate();++ @Override+ public boolean isReadable(+ Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) {+ return delegate.isReadable(type, genericType, annotations, mediaType);+ }++ @Override+ public Object readFrom(+ Class<Object> type,+ Type genericType,+ Annotation[] annotations,+ MediaType mediaType,+ MultivaluedMap<String, String> httpHeaders,+ InputStream entityStream)+ throws IOException, WebApplicationException {+ final ContainerRequestContext requestContext = getRequestContext();+ final Object object =+ delegate.readFrom(type, genericType, annotations, mediaType, httpHeaders, entityStream);+ if (requestContext != null) {+ log.info("object is {}", object);+ requestContext.setProperty(DESERIALIZED_REQUEST_BODY_KEY, object);+ }+ return object;+ }++ private ContainerRequestContext getRequestContext() {+ return serviceLocator.getService(ContainerRequestContext.class);+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java b/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.javaindex 8735368f879..b7ecd64720c 100644--- a/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java+++ b/solr/core/src/java/org/apache/solr/jersey/PostRequestDecorationFilter.java@@ -17,10 +17,12 @@package org.apache.solr.jersey;+import static org.apache.solr.jersey.PostRequestDecorationFilter.PRIORITY;import static org.apache.solr.jersey.RequestContextKeys.SOLR_QUERY_REQUEST;import java.io.IOException;import java.lang.invoke.MethodHandles;+import javax.annotation.Priority;import javax.ws.rs.container.ContainerRequestContext;import javax.ws.rs.container.ContainerResponseContext;import javax.ws.rs.container.ContainerResponseFilter;@@ -37,7 +39,9 @@ import org.slf4j.LoggerFactory;** @see SolrCore#postDecorateResponse(SolrRequestHandler, SolrQueryRequest, SolrQueryResponse)*/+@Priority(PRIORITY)public class PostRequestDecorationFilter implements ContainerResponseFilter {+ public static final int PRIORITY = 10;private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());diff --git a/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.java b/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.javanew file mode 100644index 00000000000..f2d35e93573--- /dev/null+++ b/solr/core/src/java/org/apache/solr/jersey/PostRequestLoggingFilter.java@@ -0,0 +1,191 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.jersey;++import static org.apache.solr.common.params.CommonParams.LOG_PARAMS_LIST;+import static org.apache.solr.jersey.MessageBodyReaders.CachingDelegatingMessageBodyReader.DESERIALIZED_REQUEST_BODY_KEY;+import static org.apache.solr.jersey.PostRequestLoggingFilter.PRIORITY;+import static org.apache.solr.jersey.RequestContextKeys.SOLR_QUERY_REQUEST;++import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.Arrays;+import java.util.HashSet;+import java.util.Locale;+import java.util.Map;+import java.util.Set;+import java.util.stream.Collectors;+import javax.annotation.Priority;+import javax.ws.rs.Path;+import javax.ws.rs.container.ContainerRequestContext;+import javax.ws.rs.container.ContainerResponseContext;+import javax.ws.rs.container.ContainerResponseFilter;+import javax.ws.rs.container.ResourceInfo;+import javax.ws.rs.core.Context;+import javax.ws.rs.core.MultivaluedMap;+import org.apache.solr.common.util.CollectionUtil;+import org.apache.solr.common.util.StrUtils;+import org.apache.solr.core.SolrCore;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.servlet.HttpSolrCall;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;+import org.slf4j.MarkerFactory;++@Priority(PRIORITY)+public class PostRequestLoggingFilter implements ContainerResponseFilter {++ // Ensures that this filter runs AFTER response decoration, so that we can assume+ // QTime, etc. have been populated on the response.+ public static final int PRIORITY = PostRequestDecorationFilter.PRIORITY / 2;++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ // Use SolrCore and HttpSolrCall request loggers to maintain compatibility with logging dashboards+ // built for v1 APIs+ private static final Logger coreRequestLogger =+ LoggerFactory.getLogger(SolrCore.class.getName() + ".Request");+ private static final Logger slowCoreRequestLogger =+ LoggerFactory.getLogger(SolrCore.class.getName() + ".SlowRequest");+ private static final Logger nonCoreRequestLogger =+ LoggerFactory.getLogger(HttpSolrCall.class.getName());++ @Context private ResourceInfo resourceInfo;++ @Override+ public void filter(+ ContainerRequestContext requestContext, ContainerResponseContext responseContext)+ throws IOException {+ if (requestContext.getPropertyNames().contains(RequestContextKeys.NOT_FOUND_FLAG)) {+ return;+ }+ if (!responseContext.hasEntity()+ || !SolrJerseyResponse.class.isInstance(responseContext.getEntity())) {+ log.debug("Skipping v2 API logging because response is of an unexpected type");+ return;+ }+ final SolrJerseyResponse response = (SolrJerseyResponse) responseContext.getEntity();+ final SolrQueryRequest solrQueryRequest =+ (SolrQueryRequest) requestContext.getProperty(SOLR_QUERY_REQUEST);+ final var solrConfig =+ (solrQueryRequest.getCore() != null) ? solrQueryRequest.getCore().getSolrConfig() : null;++ final Logger requestLogger = (solrConfig != null) ? coreRequestLogger : nonCoreRequestLogger;+ final String templatedPath =+ buildTemplatedPath(requestContext.getUriInfo().getAbsolutePath().getPath());+ final String bodyVal = buildRequestBodyString(requestContext);+ requestLogger.info(+ MarkerFactory.getMarker(templatedPath),+ "method={} path={} query-params={{}} entity={} status={} QTime={}",+ requestContext.getMethod(),+ templatedPath,+ filterAndStringifyQueryParameters(requestContext.getUriInfo().getQueryParameters()),+ bodyVal,+ response.responseHeader.status,+ response.responseHeader.qTime);++ /* slowQueryThresholdMillis defaults to -1 in SolrConfig -- not enabled.*/+ if (log.isWarnEnabled()+ && solrConfig != null+ && solrConfig.slowQueryThresholdMillis >= 0+ && response.responseHeader.qTime >= solrConfig.slowQueryThresholdMillis) {+ slowCoreRequestLogger.warn(+ MarkerFactory.getMarker(templatedPath),+ "method={} path={} query-params={{}} entity={} status={} QTime={}",+ requestContext.getMethod(),+ templatedPath,+ filterAndStringifyQueryParameters(requestContext.getUriInfo().getQueryParameters()),+ response.responseHeader.status,+ response.responseHeader.qTime);+ }+ }++ private String buildTemplatedPath(String fallbackPath) {+ // We won't have a resource class or method in the case of a 404, so don't try to template out+ // the path-variables+ if (resourceInfo == null+ || resourceInfo.getResourceClass() == null+ || resourceInfo.getResourceMethod() == null) {+ return fallbackPath;+ }++ final var classPathAnnotation = resourceInfo.getResourceClass().getAnnotation(Path.class);+ final var classPathAnnotationVal =+ (classPathAnnotation != null) ? classPathAnnotation.value() : "";+ final var methodPathAnnotation = resourceInfo.getResourceMethod().getAnnotation(Path.class);+ final var methodPathAnnotationVal =+ (methodPathAnnotation != null) ? methodPathAnnotation.value() : "";++ return String.format(Locale.ROOT, "%s%s", classPathAnnotationVal, methodPathAnnotationVal)+ .replaceAll("//", "/");+ }++ public static String buildRequestBodyString(ContainerRequestContext requestContext) {+ if (requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY) == null) {+ return "{}";+ }++ if (!(requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)+ instanceof JacksonReflectMapWriter)) {+ log.warn(+ "Encountered unexpected request-body type {} for request {}; only {} expected.",+ requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY).getClass().getName(),+ requestContext.getUriInfo().getPath(),+ JacksonReflectMapWriter.class.getName());+ return "{}";+ }++ return ((JacksonReflectMapWriter) requestContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY))+ .jsonStr()+ .replace("\n", "");+ }++ public static String filterAndStringifyQueryParameters(+ MultivaluedMap<String, String> unfilteredParams) {+ final var paramNamesToLog = getParamNamesToLog(unfilteredParams);+ final StringBuilder sb = new StringBuilder(128);+ unfilteredParams.entrySet().stream()+ .sorted(Map.Entry.comparingByKey())+ .forEachOrdered(+ entry -> {+ final String name = entry.getKey();+ if (!paramNamesToLog.contains(name)) return;++ for (String val : entry.getValue()) {+ if (sb.length() != 0) sb.append('&');+ StrUtils.partialURLEncodeVal(sb, name);+ sb.append('=');+ StrUtils.partialURLEncodeVal(sb, val);+ }+ });+ return sb.toString();+ }++ private static Set<String> getParamNamesToLog(MultivaluedMap<String, String> queryParameters) {+ if (CollectionUtil.isEmpty(queryParameters.get(LOG_PARAMS_LIST))) {+ return queryParameters.keySet();+ }++ final var paramsToLogStr = queryParameters.getFirst(LOG_PARAMS_LIST);+ if (StrUtils.isBlank(paramsToLogStr)) {+ return new HashSet<>(); // A value-less param means that no parameters should be logged+ }++ return Arrays.stream(paramsToLogStr.split(",")).collect(Collectors.toSet());+ }+}diff --git a/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.java b/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.javanew file mode 100644index 00000000000..c0770c4912e--- /dev/null+++ b/solr/core/src/test/org/apache/solr/jersey/PostRequestLoggingFilterTest.java@@ -0,0 +1,138 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.jersey;++import static org.apache.solr.jersey.MessageBodyReaders.CachingDelegatingMessageBodyReader.DESERIALIZED_REQUEST_BODY_KEY;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++import java.util.List;+import javax.ws.rs.container.ContainerRequestContext;+import javax.ws.rs.core.MultivaluedHashMap;+import javax.ws.rs.core.UriInfo;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.handler.admin.api.CreateReplicaAPI;+import org.junit.BeforeClass;+import org.junit.Test;++/**+ * Unit tests for {@link PostRequestLoggingFilter}+ *+ * <p>Tests primarily focus on exercising the string operations used to flatten the request into a+ * log message.+ */+public class PostRequestLoggingFilterTest extends SolrTestCaseJ4 {++ @BeforeClass+ public static void ensureWorkingMockito() {+ assumeWorkingMockito();+ }++ @Test+ public void testBuildQueryParameterString_Simple() {+ final var queryParams = new MultivaluedHashMap<String, String>();+ queryParams.putSingle("paramName1", "paramValue1");+ queryParams.putSingle("paramName2", "paramValue2");++ final var queryParamStr =+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);++ assertEquals("paramName1=paramValue1¶mName2=paramValue2", queryParamStr);+ }++ @Test+ public void testBuildQueryParameterString_DuplicateParams() {+ final var queryParams = new MultivaluedHashMap<String, String>();+ queryParams.putSingle("paramName1", "paramValue1");+ queryParams.put("paramName2", List.of("paramValue2a", "paramValue2b"));++ final var queryParamStr =+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);++ assertEquals(+ "paramName1=paramValue1¶mName2=paramValue2a¶mName2=paramValue2b", queryParamStr);+ }++ @Test+ public void testBuildQueryParameterString_SpecialChars() {+ final var queryParams = new MultivaluedHashMap<String, String>();+ queryParams.putSingle("paramName1", "paramValue1");+ queryParams.putSingle(+ "paramName=2", "paramValue=2"); // The name and value themselves contain an equals sign++ final var queryParamStr =+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);++ assertEquals("paramName1=paramValue1¶mName%3D2=paramValue%3D2", queryParamStr);+ }++ @Test+ public void testBuildQueryParameterString_ExcludesFilteredParameters() {+ final var queryParams = new MultivaluedHashMap<String, String>();+ queryParams.putSingle("paramName1", "paramValue1");+ queryParams.putSingle("paramName2", "paramValue2");+ queryParams.putSingle("hiddenParam1", "hiddenValue1");+ queryParams.putSingle("hiddenParam2", "hiddenValue2");+ queryParams.putSingle("logParamsList", "paramName1,paramName2");++ final var queryParamStr =+ PostRequestLoggingFilter.filterAndStringifyQueryParameters(queryParams);++ assertEquals("paramName1=paramValue1¶mName2=paramValue2", queryParamStr);+ }++ @Test+ public void testRequestBodyStringIsEmptyIfNoRequestBodyFound() {+ // NOTE: no request body is set on the context.+ final var mockContext = mock(ContainerRequestContext.class);++ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);++ assertEquals("{}", requestBodyStr);+ }++ @Test+ public void testRequestBodyStringIsEmptyIfRequestBodyWasUnexpectedType() {+ // NOTE: Request body is set, but of an unexpected type (i.e. not a JacksonReflectMapWriter)+ final var mockContext = mock(ContainerRequestContext.class);+ final var mockUriInfo = mock(UriInfo.class);+ when(mockUriInfo.getPath()).thenReturn("/somepath");+ when(mockContext.getUriInfo()).thenReturn(mockUriInfo);+ when(mockContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)).thenReturn("unexpectedType");++ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);++ assertEquals("{}", requestBodyStr);+ }++ @Test+ public void testRequestBodyRepresentedAsJsonWhenFound() {+ final var requestBody = new CreateReplicaAPI.AddReplicaRequestBody();+ requestBody.name = "someReplicaName";+ requestBody.type = "NRT";+ requestBody.asyncId = "someAsyncId";+ final var mockContext = mock(ContainerRequestContext.class);+ when(mockContext.getProperty(DESERIALIZED_REQUEST_BODY_KEY)).thenReturn(requestBody);++ final var requestBodyStr = PostRequestLoggingFilter.buildRequestBodyString(mockContext);++ assertEquals(+ "{ \"name\":\"someReplicaName\", \"type\":\"NRT\", \"async\":\"someAsyncId\"}",+ requestBodyStr);+ }+}diff --git a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adocindex 594e4683535..29fdf0f17e0 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/common-query-parameters.adoc@@ -352,10 +352,13 @@ If you do not define the `wt` parameter in your queries, JSON will be returned a== logParamsList Parameter-By default, Solr logs all parameters of requests.-Set this parameter to restrict which parameters of a request are logged.+By default, Solr logs all query parameters on each request.+This parameter allows users to override this behavior, by specifying a comma-separated "allowlist" of parameter names that should be logged.This may help control logging to only those parameters considered important to your organization.+NOTE: `logParamsList` only governs the logging of query parameters.+It does not apply to parameters specified in the request path, body, etc.+For example, you could define this like:`logParamsList=q,fq`