Apache Solr open-source search software
SOLR-15599: Fix Security Policies for S3 Repository
Houston Putman93be49b65bd5
diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policyindex b9137a81adb..6aabbb66e7f 100644--- a/gradle/testing/randomization/policies/solr-tests.policy+++ b/gradle/testing/randomization/policies/solr-tests.policy@@ -140,7 +140,7 @@ grant {permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";permission java.security.SecurityPermission "insertProvider";- // Needed by JWT integration tests+ // Needed by JWT integration tests & S3 testspermission java.lang.RuntimePermission "setFactory";permission javax.xml.bind.JAXBPermission "setDatatypeConverter";@@ -202,6 +202,11 @@ grant {permission java.io.FilePermission "${log4j.configurationFile}", "read,write,delete,readlink";+ // Credentials for S3 Repository+ permission java.io.FilePermission "${aws.sharedCredentialsFile}", "read,readlink";+ permission java.io.FilePermission "${aws.configFile}", "read,readlink";+ permission java.io.FilePermission "${user.home}${/}.aws${/}-", "read,readlink";+// expanded to a wildcard if set, allows all networking everywherepermission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve";};diff --git a/solr/contrib/s3-repository/build.gradle b/solr/contrib/s3-repository/build.gradleindex 91e3dc73da6..8e1ee3a14a4 100644--- a/solr/contrib/s3-repository/build.gradle+++ b/solr/contrib/s3-repository/build.gradle@@ -62,6 +62,4 @@ test {// Reduce logging noise during the testsenvironment "aws.accessKeyId", "foo"environment "aws.secretAccessKey", "bar"- environment "aws.sharedCredentialsFile", "conf/credentials"- environment "aws.configFile", "conf/config"}diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.javaindex 834beb016c1..70a63444798 100644--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java@@ -39,10 +39,9 @@ public class AbstractS3ClientTest extends SolrTestCaseJ4 {@Beforepublic void setUpClient() {- System.setProperty("aws.sharedCredentialsFile", "conf/temp");- System.setProperty("aws.configFile", "conf/temp");System.setProperty("aws.accessKeyId", "foo");System.setProperty("aws.secretAccessKey", "bar");+client =new S3StorageClient(BUCKET_NAME, "us-east-1", "", false, "http://localhost:" + S3_MOCK_RULE.getHttpPort());diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.javaindex 0272bd3be08..2a5828c1c6a 100644--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java@@ -299,8 +299,6 @@ public class S3BackupRepositoryTest extends AbstractBackupRepositoryTest {protected S3BackupRepository getRepository() {System.setProperty("aws.accessKeyId", "foo");System.setProperty("aws.secretAccessKey", "bar");- System.setProperty("aws.sharedCredentialsFile", "conf/temp");- System.setProperty("aws.configFile", "conf/temp");NamedList<Object> args = getBaseBackupRepositoryConfiguration();diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.javaindex 37a4d4da3cf..60ee8a84325 100644--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java@@ -87,8 +87,7 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest {public static void setupClass() throws Exception {System.setProperty("aws.accessKeyId", "foo");System.setProperty("aws.secretAccessKey", "bar");- System.setProperty("aws.sharedCredentialsFile", "conf/temp");- System.setProperty("aws.configFile", "conf/temp");+configureCluster(NUM_SHARDS) // nodes.addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath()).withSolrXml(diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.javaindex cefae1a1eaf..0068035aca7 100644--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java@@ -67,38 +67,38 @@ public class S3IndexInputTest extends SolrTestCaseJ4 {File file = new File(tmp, "content");FileUtils.write(file, content, StandardCharsets.UTF_8);- SliceInputStream slicedStream = new SliceInputStream(new FileInputStream(file), slice);- S3IndexInput input = new S3IndexInput(slicedStream, "path", file.length());-- // Now read the file- ByteBuffer buffer;- if (directBuffer) {- buffer = ByteBuffer.allocateDirect((int) file.length());- } else {- buffer = ByteBuffer.allocate((int) file.length());+ try (SliceInputStream slicedStream = new SliceInputStream(new FileInputStream(file), slice);+ S3IndexInput input = new S3IndexInput(slicedStream, "path", file.length())) {++ // Now read the file+ ByteBuffer buffer;+ if (directBuffer) {+ buffer = ByteBuffer.allocateDirect((int) file.length());+ } else {+ buffer = ByteBuffer.allocate((int) file.length());+ }+ input.readInternal(buffer);++ // Check the buffer content, in a way that works for both heap and direct buffers+ buffer.position(0);+ byte[] bytes = new byte[buffer.remaining()];+ buffer.get(bytes);+ assertEquals(content, new String(bytes, Charset.defaultCharset()));++ // Ensure we actually made many calls+ int expectedReadCount;+ if (directBuffer) {+ // For direct buffer, we may be caped by internal buffer if it's smaller than the size+ // defined by the test+ expectedReadCount = content.length() / Math.min(slice, S3IndexInput.LOCAL_BUFFER_SIZE) + 1;+ } else {+ expectedReadCount = content.length() / slice + 1;+ }+ assertEquals(+ "S3IndexInput did an unexpected number of reads",+ expectedReadCount,+ slicedStream.readCount);}- input.readInternal(buffer);- input.close();-- // Check the buffer content, in a way that works for both heap and direct buffers- buffer.position(0);- byte[] bytes = new byte[buffer.remaining()];- buffer.get(bytes);- assertEquals(content, new String(bytes, Charset.defaultCharset()));-- // Ensure we actually made many calls- int expectedReadCount;- if (directBuffer) {- // For direct buffer, we may be caped by internal buffer if it's smaller than the size defined- // by the test- expectedReadCount = content.length() / Math.min(slice, S3IndexInput.LOCAL_BUFFER_SIZE) + 1;- } else {- expectedReadCount = content.length() / slice + 1;- }- assertEquals(- "S3IndexInput did an unexpected number of reads",- expectedReadCount,- slicedStream.readCount);}/** Input stream that reads, but not too much in a single call. */@@ -126,5 +126,11 @@ public class S3IndexInputTest extends SolrTestCaseJ4 {int slicedLength = Math.min(slice, length);return super.read(b, off, slicedLength);}++ @Override+ public void close() throws IOException {+ input.close();+ super.close();+ }}}diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.javaindex 36323cc8318..3fcd5a3d781 100644--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java@@ -41,8 +41,6 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {@Beforepublic void setUpClient() {- System.setProperty("aws.sharedCredentialsFile", "conf/temp");- System.setProperty("aws.configFile", "conf/temp");s3 = S3_MOCK_RULE.createS3ClientV2();}@@ -57,58 +55,58 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {*/@Testpublic void testWriteByteByByte() throws IOException {-- S3OutputStream output = new S3OutputStream(s3, "byte-by-byte", BUCKET);- output.write('h');- output.write('e');- output.write('l');- output.write('l');- output.write('o');- output.close();+ try (S3OutputStream output = new S3OutputStream(s3, "byte-by-byte", BUCKET)) {+ output.write('h');+ output.write('e');+ output.write('l');+ output.write('l');+ output.write('o');+ }// Check we can re-read same content- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"));- String read = IOUtils.toString(input, Charset.defaultCharset());- assertEquals("Contents saved to S3 file did not match expected", "hello", read);+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"))) {+ String read = IOUtils.toString(input, Charset.defaultCharset());+ assertEquals("Contents saved to S3 file did not match expected", "hello", read);+ }}/** Write a small byte array, which is smaller than S3 part size. */@Testpublic void testWriteSmallBuffer() throws IOException {-// must be smaller than S3 part sizebyte[] buffer = "hello".getBytes(Charset.defaultCharset());// pre-check -- ensure that our test string isn't too bigassertTrue(buffer.length < S3OutputStream.PART_SIZE);- S3OutputStream output = new S3OutputStream(s3, "small-buffer", BUCKET);- output.write(buffer);- output.close();+ try (S3OutputStream output = new S3OutputStream(s3, "small-buffer", BUCKET)) {+ output.write(buffer);+ }// Check we can re-read same content- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"));- String read = IOUtils.toString(input, Charset.defaultCharset());- assertEquals("hello", read);+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"))) {+ String read = IOUtils.toString(input, Charset.defaultCharset());+ assertEquals("hello", read);+ }}/** Write a byte array larger than S3 part size. Simulate a real multi-part upload. */@Testpublic void testWriteLargeBuffer() throws IOException {-// must be larger than S3 part sizeString content = RandomStringUtils.randomAlphanumeric(S3OutputStream.PART_SIZE + 1024);byte[] buffer = content.getBytes(Charset.defaultCharset());// pre-check -- ensure that our test string isn't too smallassertTrue(buffer.length > S3OutputStream.PART_SIZE);- S3OutputStream output = new S3OutputStream(s3, "large-buffer", BUCKET);- output.write(buffer);- output.close();+ try (S3OutputStream output = new S3OutputStream(s3, "large-buffer", BUCKET)) {+ output.write(buffer);+ }// Check we can re-read same content- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"));- String read = IOUtils.toString(input, Charset.defaultCharset());- assertEquals(new String(buffer, Charset.defaultCharset()), read);+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"))) {+ String read = IOUtils.toString(input, Charset.defaultCharset());+ assertEquals(new String(buffer, Charset.defaultCharset()), read);+ }}/** Check flush is a no-op if data size is lower than required size of S3 part. */@@ -118,21 +116,22 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {byte[] buffer = "hello".getBytes(Charset.defaultCharset());assertTrue(buffer.length < S3OutputStream.PART_SIZE);- S3OutputStream output = new S3OutputStream(s3, "flush-small", BUCKET);- output.write(buffer);- output.flush();+ try (S3OutputStream output = new S3OutputStream(s3, "flush-small", BUCKET)) {+ output.write(buffer);+ output.flush();- buffer = ", world!".getBytes(Charset.defaultCharset());- output.write(buffer);- output.close();+ buffer = ", world!".getBytes(Charset.defaultCharset());+ output.write(buffer);+ }// Check we can re-read same content- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"));- String read = IOUtils.toString(input, Charset.defaultCharset());- assertEquals(- "Flushing a small frame of an S3OutputStream should not impact data written",- "hello, world!",- read);+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"))) {+ String read = IOUtils.toString(input, Charset.defaultCharset());+ assertEquals(+ "Flushing a small frame of an S3OutputStream should not impact data written",+ "hello, world!",+ read);+ }}/** Check flush is happening when data in buffer is larger than S3 minimal part size. */@@ -143,20 +142,21 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {byte[] buffer = content.getBytes(Charset.defaultCharset());assertTrue(buffer.length > S3OutputStream.MIN_PART_SIZE);- S3OutputStream output = new S3OutputStream(s3, "flush-large", BUCKET);- output.write(buffer);- output.flush();+ try (S3OutputStream output = new S3OutputStream(s3, "flush-large", BUCKET)) {+ output.write(buffer);+ output.flush();- buffer = "some more".getBytes(Charset.defaultCharset());- output.write(buffer);- output.close();+ buffer = "some more".getBytes(Charset.defaultCharset());+ output.write(buffer);+ }// Check we can re-read same content- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"));- String read = IOUtils.toString(input, Charset.defaultCharset());- assertEquals(- "Flushing a large frame of an S3OutputStream should not impact data written",- content + "some more",- read);+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"))) {+ String read = IOUtils.toString(input, Charset.defaultCharset());+ assertEquals(+ "Flushing a large frame of an S3OutputStream should not impact data written",+ content + "some more",+ read);+ }}}diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policyindex 8142ee5e635..cd7ec9d16ef 100644--- a/solr/server/etc/security.policy+++ b/solr/server/etc/security.policy@@ -106,7 +106,7 @@ grant {permission java.lang.RuntimePermission "writeFileDescriptor";// needed by hadoop httppermission java.lang.RuntimePermission "getProtectionDomain";- // needed by aws s3 sdk+ // needed by aws s3 sdk (Apache HTTP Client)permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";// These two *have* to be spelled out a separate@@ -148,7 +148,7 @@ grant {permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";permission java.security.SecurityPermission "insertProvider";- // Needed by JWT integration tests+ // Needed by JWT integration tests & S3 clientspermission java.lang.RuntimePermission "setFactory";permission javax.xml.bind.JAXBPermission "setDatatypeConverter";@@ -206,6 +206,11 @@ grant {permission java.io.FilePermission "${log4j.configurationFile}", "read,write,delete,readlink";+ // Credentials for S3 Repository+ permission java.io.FilePermission "${aws.sharedCredentialsFile}", "read,readlink";+ permission java.io.FilePermission "${aws.configFile}", "read,readlink";+ permission java.io.FilePermission "${user.home}${/}.aws${/}-", "read,readlink";+// expanded to a wildcard if set, allows all networking everywherepermission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve";};diff --git a/solr/solr-ref-guide/src/backup-restore.adoc b/solr/solr-ref-guide/src/backup-restore.adocindex 22a43b83b59..27b6d84d259 100644--- a/solr/solr-ref-guide/src/backup-restore.adoc+++ b/solr/solr-ref-guide/src/backup-restore.adoc@@ -752,4 +752,4 @@ An example configuration to enable S3 backups and restore can be seen below:<str name="s3.region">us-west-2</str></repository></backup>-----\ No newline at end of file+----