Apache Solr open-source search software
SOLR-15965 Use proper signatures for SolrAuth (#614)
Mike Drob1df6634b2a59
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 2b0cf47bda8..7c5bf1ad7e0 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -299,6 +299,8 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)* SOLR-15556: Migrate the Ref Guide to be built with Antora, enabling many new features (Cassandra Targett, Houston Putman, Mike Drob)+* SOLR-15965: Use better signatures for the PKI Authentication plugin. (Mike Drob)+Build---------------------* LUCENE-9077 LUCENE-9433: Support Gradle build, remove Ant support from trunk (Dawid Weiss, Erick Erickson, Uwe Schindler et.al.)diff --git a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.javaindex 8a1e1e056a6..049ae38a62d 100644--- a/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java+++ b/solr/core/src/java/org/apache/solr/security/PKIAuthenticationPlugin.java@@ -22,12 +22,16 @@ import javax.servlet.http.HttpServletResponse;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.nio.ByteBuffer;+import java.security.InvalidKeyException;import java.security.Principal;import java.security.PublicKey;+import java.security.SignatureException;+import java.time.Instant;import java.util.Base64;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;import com.google.common.annotations.VisibleForTesting;@@ -60,6 +64,9 @@ import static java.nio.charset.StandardCharsets.UTF_8;public class PKIAuthenticationPlugin extends AuthenticationPlugin implements HttpClientBuilderPlugin {+ public static final String ACCEPT_VERSIONS = "solr.pki.acceptVersions";+ public static final String SEND_VERSION = "solr.pki.sendVersion";+/*** Mark the current thread as a server thread and set a flag in SolrRequestInfo to indicate you want* to send a request as the server identity instead of as the authenticated user.@@ -88,11 +95,13 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Httprivate final Map<String, PublicKey> keyCache = new ConcurrentHashMap<>();private final PublicKeyHandler publicKeyHandler;private final CoreContainer cores;- private final int MAX_VALIDITY = Integer.parseInt(System.getProperty("pkiauth.ttl", "15000"));+ private static final int MAX_VALIDITY = Integer.getInteger("pkiauth.ttl", 5000);private final String myNodeName;private final HttpHeaderClientInterceptor interceptor = new HttpHeaderClientInterceptor();private boolean interceptorRegistered = false;+ private boolean acceptPkiV1 = false;+public boolean isInterceptorRegistered(){return interceptorRegistered;}@@ -101,6 +110,20 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Httthis.publicKeyHandler = publicKeyHandler;this.cores = cores;myNodeName = nodeName;++ Set<String> knownPkiVersions = Set.of("v1", "v2");+ // We always accept v2 even if it is not specified+ String[] versions = System.getProperty(ACCEPT_VERSIONS, "v2").split(",");+ for (String version : versions) {+ if (knownPkiVersions.contains(version) == false) {+ log.warn("Unknown protocol version [{}] specified in {}", version, ACCEPT_VERSIONS);+ }+ if ("v1".equals(version)) {+ log.warn("System setting {} includes the deprecated v1, which should only be used for compatibility during rolling upgrades. " ++ "After all servers have been upgraded, consider disabling this compatability layer.", ACCEPT_VERSIONS);+ acceptPkiV1 = true;+ }+ }}@Override@@ -123,56 +146,122 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Httreturn true;}- String header = request.getHeader(HEADER);- assert header != null : "Should have been checked by SolrDispatchFilter.authenticateRequest";+ PKIHeaderData headerData = null;+ String headerV2 = request.getHeader(HEADER_V2);+ String headerV1 = request.getHeader(HEADER);+ if (headerV2 != null) {+ // Try V2 first+ int nodeNameEnd = headerV2.indexOf(' ');+ if (nodeNameEnd <= 0) {+ // Do not log the value as it is likely gibberish+ return sendError(response, true, "Could not parse node name from SolrAuthV2 header.");+ }- List<String> authInfo = StrUtils.splitWS(header, false);- if (authInfo.size() != 2) {- numErrors.mark();- log.error("Invalid SolrAuth header: {}", header);- response.setHeader(HttpHeaders.WWW_AUTHENTICATE, HEADER);- response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Invalid SolrAuth header");- return false;+ headerData = decipherHeaderV2(headerV2, headerV2.substring(0, nodeNameEnd));+ } else if (headerV1 != null && acceptPkiV1) {+ List<String> authInfo = StrUtils.splitWS(headerV1, false);+ if (authInfo.size() != 2) {+ // We really shouldn't be logging and returning this, but we did it before so keep that+ return sendError(response, false, "Invalid SolrAuth header: " + headerV1);+ }+ headerData = decipherHeader(authInfo.get(0), authInfo.get(1));}- String nodeName = authInfo.get(0);- String cipher = authInfo.get(1);-- PKIHeaderData decipher = decipherHeader(nodeName, cipher);- if (decipher == null) {- numMissingCredentials.inc();- log.error("Could not load principal from SolrAuth header.");- response.setHeader(HttpHeaders.WWW_AUTHENTICATE, HEADER);- response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Could not load principal from SolrAuth header.");- return false;+ if (headerData == null) {+ return sendError(response, true, "Could not load principal from SolrAuthV2 header.");}- long elapsed = receivedTime - decipher.timestamp;+ long elapsed = receivedTime - headerData.timestamp;if (elapsed > MAX_VALIDITY) {- numErrors.mark();- log.error("Expired key request timestamp, elapsed={}, TTL={}", elapsed, MAX_VALIDITY);- response.setHeader(HttpHeaders.WWW_AUTHENTICATE, HEADER);- response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Expired key request timestamp");- return false;+ return sendError(response, true, "Expired key request timestamp, elapsed=" + elapsed);}- final Principal principal = "$".equals(decipher.userName) ?+ final Principal principal = "$".equals(headerData.userName) ?SU :- new BasicUserPrincipal(decipher.userName);+ new BasicUserPrincipal(headerData.userName);numAuthenticated.inc();filterChain.doFilter(wrapWithPrincipal(request, principal), response);return true;}+ /**+ * Set the response header errors, possibly log something and return false for failed authentication+ * @param response the response to set error status with+ * @param v2 whether this authentication used the v1 or v2 header (true if v2)+ * @param message the message to log and send back to client. do not include anyhting sensitive here about server state+ * @return false to chain with calls from authenticate+ */+ private boolean sendError(HttpServletResponse response, boolean v2, String message) throws IOException {+ numErrors.mark();+ log.error(message);+ response.setHeader(HttpHeaders.WWW_AUTHENTICATE, v2 ? HEADER_V2 : HEADER);+ response.sendError(HttpServletResponse.SC_UNAUTHORIZED, message);+ return false;+ }+public static class PKIHeaderData {String userName;long timestamp;++ @Override+ public String toString() {+ return "PKIHeaderData{" ++ "userName='" + userName + '\'' ++ ", timestamp=" + timestamp ++ '}';+ }+ }++ private PKIHeaderData decipherHeaderV2(String header, String nodeName) {+ PublicKey key = keyCache.get(nodeName);+ if (key == null) {+ log.debug("No key available for node: {} fetching now ", nodeName);+ key = getRemotePublicKey(nodeName);+ log.debug("public key obtained {} ", key);+ }++ int sigStart = header.lastIndexOf(' ');++ String data = header.substring(0, sigStart);+ byte[] sig = Base64.getDecoder().decode(header.substring(sigStart + 1));+ PKIHeaderData rv = validateSignature(data, sig, key);+ if (rv == null) {+ log.warn("Failed to verify signature, trying after refreshing the key ");+ key = getRemotePublicKey(nodeName);+ rv = validateSignature(data, sig, key);+ }++ return rv;+ }++ private PKIHeaderData validateSignature(String data, byte[] sig, PublicKey key) {+ try {+ if (CryptoKeys.verifySha256(data.getBytes(UTF_8), sig, key)) {+ int timestampStart = data.lastIndexOf(' ');+ PKIHeaderData rv = new PKIHeaderData();+ String ts = data.substring(timestampStart + 1);+ try {+ rv.timestamp = Long.parseLong(ts);+ } catch (NumberFormatException e) {+ log.error("SolrAuthV2 header error, cannot parse {} as timestamp", ts);+ return null;+ }+ rv.userName = data.substring(data.indexOf(' ') + 1, timestampStart);+ return rv;+ } else {+ log.warn("Signature verification failed, signature or checksum does not match");+ return null;+ }+ } catch (InvalidKeyException | SignatureException e) {+ log.error("Signature validation failed, likely key error");+ return null;+ }}private PKIHeaderData decipherHeader(String nodeName, String cipherBase64) {PublicKey key = keyCache.get(nodeName);if (key == null) {- log.debug("No key available for node : {} fetching now ", nodeName);+ log.debug("No key available for node: {} fetching now ", nodeName);key = getRemotePublicKey(nodeName);log.debug("public key obtained {} ", key);}@@ -215,13 +304,18 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt}}+ /**+ * Fetch the public key for a remote Solr node and store it in our key cache, replacing any existing entries.+ * @param nodename the node to fetch a key from+ * @return the public key+ */PublicKey getRemotePublicKey(String nodename) {if (!cores.getZkController().getZkStateReader().getClusterState().getLiveNodes().contains(nodename)) return null;String url = cores.getZkController().getZkStateReader().getBaseUrlForNodeName(nodename);HttpEntity entity = null;try {String uri = url + PublicKeyHandler.PATH + "?wt=json&omitHeader=true";- log.debug("Fetching fresh public key from : {}",uri);+ log.debug("Fetching fresh public key from: {}", uri);HttpResponse rsp = cores.getUpdateShardHandler().getDefaultHttpClient().execute(new HttpGet(uri), HttpClientUtil.createNewHttpClientRequestContext());entity = rsp.getEntity();@@ -229,16 +323,16 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements HttMap<?, ?> m = (Map<?, ?>) Utils.fromJSON(bytes);String key = (String) m.get("key");if (key == null) {- log.error("No key available from {} {}", url, PublicKeyHandler.PATH);+ log.error("No key available from {}{}", url, PublicKeyHandler.PATH);return null;} else {- log.info("New Key obtained from node: {} / {}", nodename, key);+ log.info("New key obtained from node={}, key={}", nodename, key);}PublicKey pubKey = CryptoKeys.deserializeX509PublicKey(key);keyCache.put(nodename, pubKey);return pubKey;} catch (Exception e) {- log.error("Exception trying to get public key from : {}", url, e);+ log.error("Exception trying to get public key from: {}", url, e);return null;} finally {Utils.consumeFully(entity);@@ -260,7 +354,11 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Httif (log.isDebugEnabled()) {log.debug("{} secures this internode request", this.getClass().getSimpleName());}- generateToken().ifPresent(s -> request.header(HEADER, myNodeName + " " + s));+ if ("v1".equals(System.getProperty(SEND_VERSION))) {+ generateToken().ifPresent(s -> request.header(HEADER, s));+ } else {+ generateTokenV2().ifPresent(s -> request.header(HEADER_V2, s));+ }} else {if (log.isDebugEnabled()) {log.debug("{} secures this internode request", cores.getAuthenticationPlugin().getClass().getSimpleName());@@ -305,42 +403,66 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements Htt}}- @SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header")- private Optional<String> generateToken() {+ private String getUser() {SolrRequestInfo reqInfo = getRequestInfo();- String usr;if (reqInfo != null && !reqInfo.useServerToken()) {Principal principal = reqInfo.getUserPrincipal();if (principal == null) {log.debug("generateToken: principal is null");//this had a request but not authenticated//so we don't not need to set a principal- return Optional.empty();+ return null;} else {- usr = principal.getName();+ assert principal.getName() != null;+ return principal.getName();}} else {if (!isSolrThread()) {//if this is not running inside a Solr threadpool (as in testcases)// then no need to add any headerlog.debug("generateToken: not a solr (server) thread");- return Optional.empty();+ return null;}//this request seems to be originated from Solr itself- usr = "$"; //special name to denote the user is the node itself+ return "$"; //special name to denote the user is the node itself}+ }- String s = usr + " " + System.currentTimeMillis();+ @SuppressForbidden(reason = "Needs currentTimeMillis to set current time in header")+ private Optional<String> generateToken() {+ String usr = getUser();+ if (usr == null) {+ return Optional.empty();+ }+ String s = usr + " " + System.currentTimeMillis();byte[] payload = s.getBytes(UTF_8);byte[] payloadCipher = publicKeyHandler.keyPair.encrypt(ByteBuffer.wrap(payload));String base64Cipher = Base64.getEncoder().encodeToString(payloadCipher);log.trace("generateToken: usr={} token={}", usr, base64Cipher);- return Optional.of(base64Cipher);+ return Optional.of(myNodeName + " " + base64Cipher);+ }++ private Optional<String> generateTokenV2() {+ String user = getUser();+ if (user == null) {+ return Optional.empty();+ }++ String s = myNodeName + " " + user + " " + Instant.now().toEpochMilli();++ byte[] payload = s.getBytes(UTF_8);+ byte[] signature = publicKeyHandler.keyPair.signSha256(payload);+ String base64Signature = Base64.getEncoder().encodeToString(signature);+ return Optional.of(s + " " + base64Signature);}void setHeader(HttpRequest httpRequest) {- generateToken().ifPresent(s -> httpRequest.setHeader(HEADER, myNodeName + " " + s));+ if ("v1".equals(System.getProperty(SEND_VERSION))) {+ generateToken().ifPresent(s -> httpRequest.setHeader(HEADER, s));+ } else {+ generateTokenV2().ifPresent(s -> httpRequest.setHeader(HEADER_V2, s));+ }}boolean isSolrThread() {@@ -357,11 +479,13 @@ public class PKIAuthenticationPlugin extends AuthenticationPlugin implements HttinterceptorRegistered = false;}+ @VisibleForTestingpublic String getPublicKey() {return publicKeyHandler.getPublicKey();}public static final String HEADER = "SolrAuth";+ public static final String HEADER_V2 = "SolrAuthV2";public static final String NODE_IS_USER = "$";// special principal to denote the cluster memberprivate static final Principal SU = new BasicUserPrincipal("$");diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.javaindex 7f6ac16ea54..2c9cb69d972 100644--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java@@ -290,7 +290,8 @@ public class SolrDispatchFilter extends BaseSolrFilter implements PathExcluder {return;}String header = request.getHeader(PKIAuthenticationPlugin.HEADER);- if (header != null && cores.getPkiAuthenticationSecurityBuilder() != null)+ String headerV2 = request.getHeader(PKIAuthenticationPlugin.HEADER_V2);+ if ((header != null || headerV2 != null) && cores.getPkiAuthenticationSecurityBuilder() != null)authenticationPlugin = cores.getPkiAuthenticationSecurityBuilder();try {if (log.isDebugEnabled()) {diff --git a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java b/solr/core/src/java/org/apache/solr/util/CryptoKeys.javaindex c43feff8334..d83c32b6c86 100644--- a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java+++ b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java@@ -190,6 +190,17 @@ public final class CryptoKeys {return rsaCipher.doFinal(buffer, 0, buffer.length);}+ public static boolean verifySha256(byte[] data, byte[] sig, PublicKey key) throws SignatureException, InvalidKeyException {+ try {+ Signature signature = Signature.getInstance("SHA256withRSA");+ signature.initVerify(key);+ signature.update(data);+ return signature.verify(sig);+ } catch (NoSuchAlgorithmException e) {+ throw new InternalError("SHA256withRSA must be supported by the JVM.");+ }+ }+/*** Tries for find X509 certificates in the input stream in DER or PEM format.* Supports multiple certs in same stream if multiple PEM certs are concatenated.@@ -276,6 +287,7 @@ public final class CryptoKeys {}}+public String getPublicKeyStr() {return pubKeyStr;}@@ -295,18 +307,21 @@ public final class CryptoKeys {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);}}- public byte[] signSha256(byte[] bytes) throws InvalidKeyException, SignatureException {++ public byte[] signSha256(byte[] bytes) {Signature dsa = null;try {dsa = Signature.getInstance("SHA256withRSA");} catch (NoSuchAlgorithmException e) {- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ throw new InternalError("SHA256withRSA is required to be supported by the JVM.", e);+ }+ try {+ dsa.initSign(privateKey);+ dsa.update(bytes,0, bytes.length);+ return dsa.sign();+ } catch (InvalidKeyException | SignatureException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error generating PKI Signature", e);}- dsa.initSign(privateKey);- dsa.update(bytes,0,bytes.length);- return dsa.sign();-}-}}diff --git a/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java b/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.javaindex a742e54334e..8044b8fb590 100644--- a/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java+++ b/solr/core/src/test/org/apache/solr/security/TestPKIAuthenticationPlugin.java@@ -19,6 +19,7 @@ package org.apache.solr.security;import javax.servlet.FilterChain;import javax.servlet.ServletRequest;import javax.servlet.http.HttpServletRequest;+import javax.servlet.http.HttpServletResponse;import java.nio.ByteBuffer;import java.security.Principal;import java.security.PublicKey;@@ -27,30 +28,34 @@ import java.util.Base64;import java.util.concurrent.atomic.AtomicReference;import org.apache.http.Header;+import org.apache.http.HttpHeaders;import org.apache.http.auth.BasicUserPrincipal;import org.apache.http.message.BasicHttpRequest;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.core.CoreContainer;import org.apache.solr.request.LocalSolrQueryRequest;import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.util.CryptoKeys;+import org.junit.Test;+import org.mockito.ArgumentMatchers;import static java.nio.charset.StandardCharsets.UTF_8;+import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.any;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {- static class MockPKIAuthenticationPlugin extends PKIAuthenticationPlugin {+ private static class MockPKIAuthenticationPlugin extends PKIAuthenticationPlugin {SolrRequestInfo solrRequestInfo;final PublicKey myKey;- public MockPKIAuthenticationPlugin(CoreContainer cores, String node) {- super(cores, node, new PublicKeyHandler());+ public MockPKIAuthenticationPlugin(String node) {+ super(null, node, new PublicKeyHandler());myKey = CryptoKeys.deserializeX509PublicKey(getPublicKey());}@@ -86,6 +91,7 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {}};+ String headerKey;HttpServletRequest mockReq;MockPKIAuthenticationPlugin mock;BasicHttpRequest request;@@ -99,8 +105,18 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {header.set(null);wrappedRequestByFilter.set(null);+ if (random().nextBoolean()) {+ headerKey = PKIAuthenticationPlugin.HEADER_V2;+ System.setProperty(PKIAuthenticationPlugin.SEND_VERSION, "v2");+ System.setProperty(PKIAuthenticationPlugin.ACCEPT_VERSIONS, "v2");+ } else {+ headerKey = PKIAuthenticationPlugin.HEADER;+ System.setProperty(PKIAuthenticationPlugin.SEND_VERSION, "v1");+ System.setProperty(PKIAuthenticationPlugin.ACCEPT_VERSIONS, "v1,v2");+ }+mockReq = createMockRequest(header);- mock = new MockPKIAuthenticationPlugin(null, nodeName);+ mock = new MockPKIAuthenticationPlugin(nodeName);request = new BasicHttpRequest("GET", "http://localhost:56565");}@@ -117,10 +133,10 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {principal.set(new BasicUserPrincipal(username));mock.solrRequestInfo = new SolrRequestInfo(localSolrQueryRequest, new SolrQueryResponse());mock.setHeader(request);- header.set(request.getFirstHeader(PKIAuthenticationPlugin.HEADER));+ header.set(request.getFirstHeader(headerKey));assertNotNull(header.get());assertTrue(header.get().getValue().startsWith(nodeName));- mock.authenticate(mockReq, null, filterChain);+ assertTrue(mock.authenticate(mockReq, null, filterChain));assertNotNull(wrappedRequestByFilter.get());assertNotNull(((HttpServletRequest) wrappedRequestByFilter.get()).getUserPrincipal());@@ -130,7 +146,7 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {public void testSuperUser() throws Exception {// Simulate the restart of a node, this will return a different key on subsequent invocations.// Create it in advance because it can take some time and should be done before header is set- MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(null, nodeName) {+ MockPKIAuthenticationPlugin mock1 = new MockPKIAuthenticationPlugin(nodeName) {boolean firstCall = true;@Override@@ -146,21 +162,41 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {// Setup regular superuser requestmock.solrRequestInfo = null;mock.setHeader(request);- header.set(request.getFirstHeader(PKIAuthenticationPlugin.HEADER));+ header.set(request.getFirstHeader(headerKey));assertNotNull(header.get());assertTrue(header.get().getValue().startsWith(nodeName));- mock.authenticate(mockReq, null, filterChain);+ assertTrue(mock.authenticate(mockReq, null, filterChain));assertNotNull(wrappedRequestByFilter.get());assertEquals("$", ((HttpServletRequest) wrappedRequestByFilter.get()).getUserPrincipal().getName());// With the simulated restart- mock1.authenticate(mockReq, null,filterChain );+ assertTrue(mock1.authenticate(mockReq, null, filterChain));assertNotNull(wrappedRequestByFilter.get());assertEquals("$", ((HttpServletRequest) wrappedRequestByFilter.get()).getUserPrincipal().getName());mock1.close();}+ @Test+ public void testProtocolMismatch() throws Exception {+ System.setProperty(PKIAuthenticationPlugin.SEND_VERSION, "v1");+ System.setProperty(PKIAuthenticationPlugin.ACCEPT_VERSIONS, "v2");+ mock = new MockPKIAuthenticationPlugin(nodeName);++ principal.set(new BasicUserPrincipal("solr"));+ mock.solrRequestInfo = new SolrRequestInfo(localSolrQueryRequest, new SolrQueryResponse());+ mock.setHeader(request);++ HttpServletResponse response = mock(HttpServletResponse.class);+ // This will fail in the same way that a missing header would fail+ assertFalse("Should have failed authentication", mock.authenticate(mockReq, response, filterChain));++ verify(response).setHeader(HttpHeaders.WWW_AUTHENTICATE, PKIAuthenticationPlugin.HEADER_V2);+ verify(response).sendError(ArgumentMatchers.eq(401), anyString());++ assertNull("Should not have proceeded after authentication failure", wrappedRequestByFilter.get());+ }+public void testParseCipher() {for (String validUser: new String[]{"user1", "$", "some user","some 123"}) {for (long validTimestamp: new long[]{Instant.now().toEpochMilli(), 99999999999L, 9999999999999L}) {@@ -234,7 +270,7 @@ public class TestPKIAuthenticationPlugin extends SolrTestCaseJ4 {private HttpServletRequest createMockRequest(final AtomicReference<Header> header) {HttpServletRequest mockReq = mock(HttpServletRequest.class);when(mockReq.getHeader(any(String.class))).then(invocation -> {- if (PKIAuthenticationPlugin.HEADER.equals(invocation.getArgument(0))) {+ if (headerKey.equals(invocation.getArgument(0))) {if (header.get() == null) return null;return header.get().getValue();} else return null;diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/authentication-and-authorization-plugins.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/authentication-and-authorization-plugins.adocindex edbf2ee55ac..d6d5d934e8a 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/authentication-and-authorization-plugins.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/authentication-and-authorization-plugins.adoc@@ -202,7 +202,7 @@ All operations supported by Admin UI can be performed through Solr's APIs.There are a lot of requests that originate from the Solr nodes itself.For example, requests from overseer to nodes, recovery threads, etc.We call these 'inter-node' requests.-Solr has a built-in `PKIAuthenticationPlugin` (described below) that is always available to secure inter-node traffic.+Solr has a built-in `PKIAuthenticationPlugin` (described below) that is always available to secure inter-node traffic.Each Authentication plugin may also decide to secure inter-node requests on its own.They may do this through the so-called `HttpClientBuilder` mechanism, or they may alternatively choose on a per-request basis whether to delegate to PKI or not by overriding a `interceptInternodeRequest()` method from the base class, where any HTTP headers can be set.@@ -212,13 +212,23 @@ They may do this through the so-called `HttpClientBuilder` mechanism, or they maThe `PKIAuthenticationPlugin` provides a built-in authentication mechanism where each Solr node is a super user and is fully trusted by other Solr nodes through the use of Public Key Infrastructure (PKI).Each Authentication plugin may choose to delegate all or some inter-node traffic to the PKI plugin.-For each outgoing request `PKIAuthenticationPlugin` adds a special header `'SolrAuth'` which carries the timestamp and principal encrypted using the private key of that node.-The public key is exposed through an API so that any node can read it whenever it needs it.-Any node who gets the request with that header, would get the public key from the sender and decrypt the information.-If it is able to decrypt the data, the request trusted.-It is invalid if the timestamp is more than 5 seconds old.-This assumes that the clocks of different nodes in the cluster are synchronized.-Only traffic from other Solr nodes registered with ZooKeeper is trusted.+There are currently two versions of the PKI Authentication protocol available in Solr. For each outgoing request `PKIAuthenticationPlugin` adds a special header which carries the request timestamp and user principal.+When a node receives a request with this special header, it will verify to message using the corresponding source node's public key.+Message validation is only attempted for incoming traffic from other Solr nodes registered in ZooKeeper.+If the request passes PKI validation and the timestamp is less than 5 seconds old, then the request will be trusted.++[NOTE]+====+Note: Because the PKI Authentication Plugin relies on relatively short timestamp expiration to validate requests, the clocks on separate nodes in the cluster must be synchronized.+====++Version 2 of the protocol is the default version. In this version, the `SolrAuthV2` header contains: the source node name, user principal, request timestamp, and a base64-encoded RSA signature. All nodes will attempt to validate this header first.++To support rolling restarts from older versions, Solr can be configured to accept and validate PKI authentication using protocol v1. This is enabled by setting the system properties `solr.pki.sendVersion=v1` and `solr.pki.acceptVersions=v1,v2`. When enabled, requests will contain a `SolrAuth` header which will contain the user principal and timestamp encrypted using the sender's private key.++If the `SolrAuthV2` header is present but fails validation, then Solr will not fall back to checking `SolrAuth`. The legacy authentication headers will only be consulted when the newest headers are not present.++Unkown values for `solr.pki.acceptVersion` will emit a warning log message but will not cause errors to more smoothly support future protocol revisions.The timeout is configurable through a system property called `pkiauth.ttl`.-For example, if you wish to increase the time-to-live to 10 seconds (10000 milliseconds), start each node with a property `'-Dpkiauth.ttl=10000'`.+For example, if you wish to increase the time-to-live to 10 seconds (10,000 milliseconds), start each node with a property `'-Dpkiauth.ttl=10000'`.diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex 0593c56e194..973354eb521 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -53,7 +53,16 @@ SolrCloud doesn't support that but may sometime.* SOLR-14702: All references to "master" and "slave" were replaced in the code with "leader" and "follower".This includes API calls for the replication handler and metrics.-For rolling upgrades into 9.0, you need to be on Solr version 8.7 or greater. Some metrics also changed, alerts and monitors on Solr KPIs that mention "master" or "slave" will also now be "leader" and "follower"+For rolling upgrades into 9.0, you need to be on Solr version 8.7 or greater. Some metrics also changed, alerts and monitors on Solr KPIs that mention "master" or "slave" will also now be "leader" and "follower"++* SOLR-15965: Internode communication secured by PKI Authentication has changed formats. For detailed information, see+ xref:deployment-guide:authentication-and-authorization-plugins.adoc#pkiauthenticationplugin[PKI Authentication Plugin].+ A rolling upgrade from Solr 8 to Solr 9 requires the following multiple restart sequence:+** Either:+*** Upgrade to Solr 8.11.2 or newer+*** Upgrade to Solr 9 and set system properties: `solr.pki.sendVersion=v1` and `solr.pki.acceptVersions=v1,v2`+** Upgrade to Solr 9 with `solr.pki.acceptVersions=v1,v2`. If you had already upgraded to Solr 9 in the previous step, this will be a rolling restart, not upgrade.+** (Optional) A rolling restart with system property `solr.pki.acceptVersions=v2` to prevent outdated nodes from connecting to your cluster.=== Reindexing After Upgrade