Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15599: Fix Security Policies for S3 Repository

Houston Putman93be49b65bd5
diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy
index b9137a81adb..6aabbb66e7f 100644
--- a/gradle/testing/randomization/policies/solr-tests.policy
+++ b/gradle/testing/randomization/policies/solr-tests.policy
@@ -140,7 +140,7 @@ grant {
permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
permission java.security.SecurityPermission "insertProvider";
- // Needed by JWT integration tests
+ // Needed by JWT integration tests & S3 tests
permission java.lang.RuntimePermission "setFactory";
permission javax.xml.bind.JAXBPermission "setDatatypeConverter";
@@ -202,6 +202,11 @@ grant {
permission java.io.FilePermission "${log4j.configurationFile}", "read,write,delete,readlink";
+ // Credentials for S3 Repository
+ permission java.io.FilePermission "${aws.sharedCredentialsFile}", "read,readlink";
+ permission java.io.FilePermission "${aws.configFile}", "read,readlink";
+ permission java.io.FilePermission "${user.home}${/}.aws${/}-", "read,readlink";
+
// expanded to a wildcard if set, allows all networking everywhere
permission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve";
};
diff --git a/solr/contrib/s3-repository/build.gradle b/solr/contrib/s3-repository/build.gradle
index 91e3dc73da6..8e1ee3a14a4 100644
--- a/solr/contrib/s3-repository/build.gradle
+++ b/solr/contrib/s3-repository/build.gradle
@@ -62,6 +62,4 @@ test {
// Reduce logging noise during the tests
environment "aws.accessKeyId", "foo"
environment "aws.secretAccessKey", "bar"
- environment "aws.sharedCredentialsFile", "conf/credentials"
- environment "aws.configFile", "conf/config"
}
diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java
index 834beb016c1..70a63444798 100644
--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java
+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/AbstractS3ClientTest.java
@@ -39,10 +39,9 @@ public class AbstractS3ClientTest extends SolrTestCaseJ4 {
@Before
public void setUpClient() {
- System.setProperty("aws.sharedCredentialsFile", "conf/temp");
- System.setProperty("aws.configFile", "conf/temp");
System.setProperty("aws.accessKeyId", "foo");
System.setProperty("aws.secretAccessKey", "bar");
+
client =
new S3StorageClient(
BUCKET_NAME, "us-east-1", "", false, "http://localhost:" + S3_MOCK_RULE.getHttpPort());
diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java
index 0272bd3be08..2a5828c1c6a 100644
--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java
+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3BackupRepositoryTest.java
@@ -299,8 +299,6 @@ public class S3BackupRepositoryTest extends AbstractBackupRepositoryTest {
protected S3BackupRepository getRepository() {
System.setProperty("aws.accessKeyId", "foo");
System.setProperty("aws.secretAccessKey", "bar");
- System.setProperty("aws.sharedCredentialsFile", "conf/temp");
- System.setProperty("aws.configFile", "conf/temp");
NamedList<Object> args = getBaseBackupRepositoryConfiguration();
diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java
index 37a4d4da3cf..60ee8a84325 100644
--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java
+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IncrementalBackupTest.java
@@ -87,8 +87,7 @@ public class S3IncrementalBackupTest extends AbstractIncrementalBackupTest {
public static void setupClass() throws Exception {
System.setProperty("aws.accessKeyId", "foo");
System.setProperty("aws.secretAccessKey", "bar");
- System.setProperty("aws.sharedCredentialsFile", "conf/temp");
- System.setProperty("aws.configFile", "conf/temp");
+
configureCluster(NUM_SHARDS) // nodes
.addConfig("conf1", getFile("conf/solrconfig.xml").getParentFile().toPath())
.withSolrXml(
diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java
index cefae1a1eaf..0068035aca7 100644
--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java
+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3IndexInputTest.java
@@ -67,38 +67,38 @@ public class S3IndexInputTest extends SolrTestCaseJ4 {
File file = new File(tmp, "content");
FileUtils.write(file, content, StandardCharsets.UTF_8);
- SliceInputStream slicedStream = new SliceInputStream(new FileInputStream(file), slice);
- S3IndexInput input = new S3IndexInput(slicedStream, "path", file.length());
-
- // Now read the file
- ByteBuffer buffer;
- if (directBuffer) {
- buffer = ByteBuffer.allocateDirect((int) file.length());
- } else {
- buffer = ByteBuffer.allocate((int) file.length());
+ try (SliceInputStream slicedStream = new SliceInputStream(new FileInputStream(file), slice);
+ S3IndexInput input = new S3IndexInput(slicedStream, "path", file.length())) {
+
+ // Now read the file
+ ByteBuffer buffer;
+ if (directBuffer) {
+ buffer = ByteBuffer.allocateDirect((int) file.length());
+ } else {
+ buffer = ByteBuffer.allocate((int) file.length());
+ }
+ input.readInternal(buffer);
+
+ // Check the buffer content, in a way that works for both heap and direct buffers
+ buffer.position(0);
+ byte[] bytes = new byte[buffer.remaining()];
+ buffer.get(bytes);
+ assertEquals(content, new String(bytes, Charset.defaultCharset()));
+
+ // Ensure we actually made many calls
+ int expectedReadCount;
+ if (directBuffer) {
+ // For direct buffer, we may be caped by internal buffer if it's smaller than the size
+ // defined by the test
+ expectedReadCount = content.length() / Math.min(slice, S3IndexInput.LOCAL_BUFFER_SIZE) + 1;
+ } else {
+ expectedReadCount = content.length() / slice + 1;
+ }
+ assertEquals(
+ "S3IndexInput did an unexpected number of reads",
+ expectedReadCount,
+ slicedStream.readCount);
}
- input.readInternal(buffer);
- input.close();
-
- // Check the buffer content, in a way that works for both heap and direct buffers
- buffer.position(0);
- byte[] bytes = new byte[buffer.remaining()];
- buffer.get(bytes);
- assertEquals(content, new String(bytes, Charset.defaultCharset()));
-
- // Ensure we actually made many calls
- int expectedReadCount;
- if (directBuffer) {
- // For direct buffer, we may be caped by internal buffer if it's smaller than the size defined
- // by the test
- expectedReadCount = content.length() / Math.min(slice, S3IndexInput.LOCAL_BUFFER_SIZE) + 1;
- } else {
- expectedReadCount = content.length() / slice + 1;
- }
- assertEquals(
- "S3IndexInput did an unexpected number of reads",
- expectedReadCount,
- slicedStream.readCount);
}
/** Input stream that reads, but not too much in a single call. */
@@ -126,5 +126,11 @@ public class S3IndexInputTest extends SolrTestCaseJ4 {
int slicedLength = Math.min(slice, length);
return super.read(b, off, slicedLength);
}
+
+ @Override
+ public void close() throws IOException {
+ input.close();
+ super.close();
+ }
}
}
diff --git a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java
index 36323cc8318..3fcd5a3d781 100644
--- a/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java
+++ b/solr/contrib/s3-repository/src/test/org/apache/solr/s3/S3OutputStreamTest.java
@@ -41,8 +41,6 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {
@Before
public void setUpClient() {
- System.setProperty("aws.sharedCredentialsFile", "conf/temp");
- System.setProperty("aws.configFile", "conf/temp");
s3 = S3_MOCK_RULE.createS3ClientV2();
}
@@ -57,58 +55,58 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {
*/
@Test
public void testWriteByteByByte() throws IOException {
-
- S3OutputStream output = new S3OutputStream(s3, "byte-by-byte", BUCKET);
- output.write('h');
- output.write('e');
- output.write('l');
- output.write('l');
- output.write('o');
- output.close();
+ try (S3OutputStream output = new S3OutputStream(s3, "byte-by-byte", BUCKET)) {
+ output.write('h');
+ output.write('e');
+ output.write('l');
+ output.write('l');
+ output.write('o');
+ }
// Check we can re-read same content
- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"));
- String read = IOUtils.toString(input, Charset.defaultCharset());
- assertEquals("Contents saved to S3 file did not match expected", "hello", read);
+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("byte-by-byte"))) {
+ String read = IOUtils.toString(input, Charset.defaultCharset());
+ assertEquals("Contents saved to S3 file did not match expected", "hello", read);
+ }
}
/** Write a small byte array, which is smaller than S3 part size. */
@Test
public void testWriteSmallBuffer() throws IOException {
-
// must be smaller than S3 part size
byte[] buffer = "hello".getBytes(Charset.defaultCharset());
// pre-check -- ensure that our test string isn't too big
assertTrue(buffer.length < S3OutputStream.PART_SIZE);
- S3OutputStream output = new S3OutputStream(s3, "small-buffer", BUCKET);
- output.write(buffer);
- output.close();
+ try (S3OutputStream output = new S3OutputStream(s3, "small-buffer", BUCKET)) {
+ output.write(buffer);
+ }
// Check we can re-read same content
- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"));
- String read = IOUtils.toString(input, Charset.defaultCharset());
- assertEquals("hello", read);
+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("small-buffer"))) {
+ String read = IOUtils.toString(input, Charset.defaultCharset());
+ assertEquals("hello", read);
+ }
}
/** Write a byte array larger than S3 part size. Simulate a real multi-part upload. */
@Test
public void testWriteLargeBuffer() throws IOException {
-
// must be larger than S3 part size
String content = RandomStringUtils.randomAlphanumeric(S3OutputStream.PART_SIZE + 1024);
byte[] buffer = content.getBytes(Charset.defaultCharset());
// pre-check -- ensure that our test string isn't too small
assertTrue(buffer.length > S3OutputStream.PART_SIZE);
- S3OutputStream output = new S3OutputStream(s3, "large-buffer", BUCKET);
- output.write(buffer);
- output.close();
+ try (S3OutputStream output = new S3OutputStream(s3, "large-buffer", BUCKET)) {
+ output.write(buffer);
+ }
// Check we can re-read same content
- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"));
- String read = IOUtils.toString(input, Charset.defaultCharset());
- assertEquals(new String(buffer, Charset.defaultCharset()), read);
+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("large-buffer"))) {
+ String read = IOUtils.toString(input, Charset.defaultCharset());
+ assertEquals(new String(buffer, Charset.defaultCharset()), read);
+ }
}
/** Check flush is a no-op if data size is lower than required size of S3 part. */
@@ -118,21 +116,22 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {
byte[] buffer = "hello".getBytes(Charset.defaultCharset());
assertTrue(buffer.length < S3OutputStream.PART_SIZE);
- S3OutputStream output = new S3OutputStream(s3, "flush-small", BUCKET);
- output.write(buffer);
- output.flush();
+ try (S3OutputStream output = new S3OutputStream(s3, "flush-small", BUCKET)) {
+ output.write(buffer);
+ output.flush();
- buffer = ", world!".getBytes(Charset.defaultCharset());
- output.write(buffer);
- output.close();
+ buffer = ", world!".getBytes(Charset.defaultCharset());
+ output.write(buffer);
+ }
// Check we can re-read same content
- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"));
- String read = IOUtils.toString(input, Charset.defaultCharset());
- assertEquals(
- "Flushing a small frame of an S3OutputStream should not impact data written",
- "hello, world!",
- read);
+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-small"))) {
+ String read = IOUtils.toString(input, Charset.defaultCharset());
+ assertEquals(
+ "Flushing a small frame of an S3OutputStream should not impact data written",
+ "hello, world!",
+ read);
+ }
}
/** Check flush is happening when data in buffer is larger than S3 minimal part size. */
@@ -143,20 +142,21 @@ public class S3OutputStreamTest extends SolrTestCaseJ4 {
byte[] buffer = content.getBytes(Charset.defaultCharset());
assertTrue(buffer.length > S3OutputStream.MIN_PART_SIZE);
- S3OutputStream output = new S3OutputStream(s3, "flush-large", BUCKET);
- output.write(buffer);
- output.flush();
+ try (S3OutputStream output = new S3OutputStream(s3, "flush-large", BUCKET)) {
+ output.write(buffer);
+ output.flush();
- buffer = "some more".getBytes(Charset.defaultCharset());
- output.write(buffer);
- output.close();
+ buffer = "some more".getBytes(Charset.defaultCharset());
+ output.write(buffer);
+ }
// Check we can re-read same content
- InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"));
- String read = IOUtils.toString(input, Charset.defaultCharset());
- assertEquals(
- "Flushing a large frame of an S3OutputStream should not impact data written",
- content + "some more",
- read);
+ try (InputStream input = s3.getObject(b -> b.bucket(BUCKET).key("flush-large"))) {
+ String read = IOUtils.toString(input, Charset.defaultCharset());
+ assertEquals(
+ "Flushing a large frame of an S3OutputStream should not impact data written",
+ content + "some more",
+ read);
+ }
}
}
diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy
index 8142ee5e635..cd7ec9d16ef 100644
--- a/solr/server/etc/security.policy
+++ b/solr/server/etc/security.policy
@@ -106,7 +106,7 @@ grant {
permission java.lang.RuntimePermission "writeFileDescriptor";
// needed by hadoop http
permission java.lang.RuntimePermission "getProtectionDomain";
- // needed by aws s3 sdk
+ // needed by aws s3 sdk (Apache HTTP Client)
permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";
// These two *have* to be spelled out a separate
@@ -148,7 +148,7 @@ grant {
permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
permission java.security.SecurityPermission "insertProvider";
- // Needed by JWT integration tests
+ // Needed by JWT integration tests & S3 clients
permission java.lang.RuntimePermission "setFactory";
permission javax.xml.bind.JAXBPermission "setDatatypeConverter";
@@ -206,6 +206,11 @@ grant {
permission java.io.FilePermission "${log4j.configurationFile}", "read,write,delete,readlink";
+ // Credentials for S3 Repository
+ permission java.io.FilePermission "${aws.sharedCredentialsFile}", "read,readlink";
+ permission java.io.FilePermission "${aws.configFile}", "read,readlink";
+ permission java.io.FilePermission "${user.home}${/}.aws${/}-", "read,readlink";
+
// expanded to a wildcard if set, allows all networking everywhere
permission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve";
};
diff --git a/solr/solr-ref-guide/src/backup-restore.adoc b/solr/solr-ref-guide/src/backup-restore.adoc
index 22a43b83b59..27b6d84d259 100644
--- a/solr/solr-ref-guide/src/backup-restore.adoc
+++ b/solr/solr-ref-guide/src/backup-restore.adoc
@@ -752,4 +752,4 @@ An example configuration to enable S3 backups and restore can be seen below:
<str name="s3.region">us-west-2</str>
</repository>
</backup>
-----
\ No newline at end of file
+----