Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

Refactor: Use InputStream.transferTo and readAllBytes (#2669)

David Smileyc6651a74447e
diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java
index 9d2296d63bb..fd3f755b9cf 100644
--- a/solr/core/src/java/org/apache/solr/cli/PostTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java
@@ -69,6 +69,7 @@ import javax.xml.xpath.XPathFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DeprecatedAttributes;
import org.apache.commons.cli.Option;
+import org.apache.commons.io.output.NullOutputStream;
import org.apache.solr.client.api.util.SolrVersion;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
@@ -1064,16 +1065,12 @@ public class PostTool extends ToolBase {
* source and thrown away.
*/
private static void pipe(InputStream source, OutputStream dest) throws IOException {
- byte[] buf = new byte[1024];
- int read = 0;
- while ((read = source.read(buf)) >= 0) {
- if (null != dest) {
- dest.write(buf, 0, read);
- }
- }
- if (null != dest) {
- dest.flush();
+ if (dest == null) {
+ dest = NullOutputStream.INSTANCE;
}
+ // copy source to dest
+ source.transferTo(dest);
+ dest.flush();
}
public FileFilter getFileFilterFromFileTypes(String fileTypes) {
diff --git a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
index ff59b99f624..003a9412741 100644
--- a/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/BlobHandler.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.util.ArrayList;
@@ -34,6 +35,7 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.input.BoundedInputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
@@ -111,8 +113,8 @@ public class BlobHandler extends RequestHandlerBase
for (ContentStream stream : req.getContentStreams()) {
ByteBuffer payload;
- try (InputStream is = stream.getStream()) {
- payload = Utils.toByteArray(is, maxSize);
+ try (InputStream is = boundedInputStream(stream.getStream(), maxSize)) {
+ payload = Utils.toByteArray(is);
}
MessageDigest m = MessageDigest.getInstance("MD5");
m.update(payload.array(), payload.arrayOffset() + payload.position(), payload.limit());
@@ -261,6 +263,16 @@ public class BlobHandler extends RequestHandlerBase
}
}
+ private static InputStream boundedInputStream(final InputStream is, final long maxLength)
+ throws IOException {
+ return new BoundedInputStream(is, maxLength) {
+ @Override
+ protected void onMaxLength(long maxLength, long count) {
+ throw new BufferOverflowException();
+ }
+ };
+ }
+
private void verifyWithRealtimeGet(
String blobName, long version, SolrQueryRequest req, Map<String, Object> doc) {
for (; ; ) {
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java
index 17909b5771f..d35a76a75c6 100644
--- a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java
@@ -21,9 +21,7 @@ import static org.apache.solr.common.params.CommonParams.JSON_MIME;
import static org.apache.solr.handler.loader.CSVLoaderBase.SEPARATOR;
import java.io.BufferedReader;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.lang.invoke.MethodHandles;
@@ -65,14 +63,11 @@ public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
private static final int MAX_STREAM_SIZE = (5 * 1024 * 1024);
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- public static byte[] streamAsBytes(final InputStream in) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- byte[] buf = new byte[1024];
- int r;
- try (in) {
- while ((r = in.read(buf)) != -1) baos.write(buf, 0, r);
+ /** Reads all bytes from the stream. */
+ private static byte[] readAllBytes(ContentStream cs) throws IOException {
+ try (var is = cs.getStream()) {
+ return is.readAllBytes();
}
- return baos.toByteArray();
}
@Override
@@ -101,7 +96,7 @@ public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
fileSource = stream.getSourceInfo() != null ? stream.getSourceInfo() : "file";
}
- byte[] uploadedBytes = streamAsBytes(stream.getStream());
+ byte[] uploadedBytes = readAllBytes(stream);
// recheck the upload size in case the stream returned null for getSize
if (uploadedBytes.length > MAX_STREAM_SIZE) {
throw new SolrException(
@@ -208,8 +203,7 @@ public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
String jsonStr =
new String(
- streamAsBytes(stream.getStream()),
- charset != null ? charset : ContentStreamBase.DEFAULT_CHARSET);
+ readAllBytes(stream), charset != null ? charset : ContentStreamBase.DEFAULT_CHARSET);
String[] lines = jsonStr.split("\n");
if (lines.length > 1) {
for (String line : lines) {
@@ -239,7 +233,7 @@ public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
protected List<SolrInputDocument> loadXmlDocs(
SolrParams params, ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad)
throws IOException {
- String xmlString = readInputAsString(stream.getStream()).trim();
+ String xmlString = new String(readAllBytes(stream), StandardCharsets.UTF_8).trim();
List<SolrInputDocument> docs;
if (xmlString.contains("<add>") && xmlString.contains("<doc>")) {
XMLInputFactory inputFactory = XMLInputFactory.newInstance();
@@ -320,10 +314,6 @@ public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
return docs;
}
- protected String readInputAsString(InputStream in) throws IOException {
- return new String(streamAsBytes(in), StandardCharsets.UTF_8);
- }
-
protected char detectTSV(String csvStr) {
char sep = ',';
int endOfFirstLine = csvStr.indexOf('\n');
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
index a8a4b5091d4..f34793aa744 100644
--- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
@@ -241,9 +241,10 @@ public class SchemaDesignerAPI implements SchemaDesignerConstants {
"File '" + file + "' not found in configSet: " + configSet);
}
- byte[] data =
- DefaultSampleDocumentsLoader.streamAsBytes(
- extractSingleContentStream(req, true).getStream());
+ byte[] data;
+ try (InputStream in = extractSingleContentStream(req, true).getStream()) {
+ data = in.readAllBytes();
+ }
Exception updateFileError = null;
boolean requestIsTrusted =
ConfigSetAPIBase.isTrusted(req.getUserPrincipal(), coreContainer.getAuthenticationPlugin());
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
index 0125e7988d8..90eb3eb0c1c 100644
--- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
@@ -65,6 +65,7 @@ import org.apache.http.client.utils.URIBuilder;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.IOSupplier;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
@@ -542,12 +543,12 @@ class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {
((CloudLegacySolrClient) cloudClient()).getHttpClient().execute(httpGet);
int statusCode = entity.getStatusLine().getStatusCode();
if (statusCode == HttpStatus.SC_OK) {
- byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
+ byte[] bytes = readAllBytes(() -> entity.getEntity().getContent());
if (bytes.length > 0) {
docs = (List<SolrInputDocument>) Utils.fromJavabin(bytes);
}
} else if (statusCode != HttpStatus.SC_NOT_FOUND) {
- byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
+ byte[] bytes = readAllBytes(() -> entity.getEntity().getContent());
throw new IOException(
"Failed to lookup stored docs for "
+ configSet
@@ -562,10 +563,14 @@ class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {
void storeSampleDocs(final String configSet, List<SolrInputDocument> docs) throws IOException {
docs.forEach(d -> d.removeField(VERSION_FIELD)); // remove _version_ field before storing ...
- postDataToBlobStore(
- cloudClient(),
- configSet + "_sample",
- DefaultSampleDocumentsLoader.streamAsBytes(toJavabin(docs)));
+ postDataToBlobStore(cloudClient(), configSet + "_sample", readAllBytes(() -> toJavabin(docs)));
+ }
+
+ /** Gets the stream, reads all the bytes, closes the stream. */
+ static byte[] readAllBytes(IOSupplier<InputStream> hasStream) throws IOException {
+ try (InputStream in = hasStream.get()) {
+ return in.readAllBytes();
+ }
}
protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, byte[] bytes)
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 68142392df1..f57c2db82c3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -1759,14 +1759,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
zout.putNextEntry(new ZipEntry("test." + fileType));
try (InputStream in = new FileInputStream(fileOrDirectory)) {
- byte[] buffer = new byte[1024];
- while (true) {
- int readCount = in.read(buffer);
- if (readCount < 0) {
- break;
- }
- zout.write(buffer, 0, readCount);
- }
+ in.transferTo(zout);
}
zout.closeEntry();
@@ -1788,8 +1781,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
Deque<File> queue = new ArrayDeque<>();
queue.push(directory);
OutputStream out = new FileOutputStream(zipfile);
- ZipOutputStream zout = new ZipOutputStream(out);
- try {
+ try (ZipOutputStream zout = new ZipOutputStream(out)) {
while (!queue.isEmpty()) {
directory = queue.pop();
for (File kid : directory.listFiles()) {
@@ -1801,26 +1793,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
} else {
zout.putNextEntry(new ZipEntry(name));
- InputStream in = new FileInputStream(kid);
- try {
- byte[] buffer = new byte[1024];
- while (true) {
- int readCount = in.read(buffer);
- if (readCount < 0) {
- break;
- }
- zout.write(buffer, 0, readCount);
- }
- } finally {
- in.close();
+ try (InputStream in = new FileInputStream(kid)) {
+ in.transferTo(zout);
}
zout.closeEntry();
}
}
}
- } finally {
- zout.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
index 979ecd9ef0b..af6412da947 100644
--- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
+++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
@@ -256,7 +256,7 @@ public class TestSchemaDesignerConfigSetHelper extends SolrCloudTestCase
helper.postDataToBlobStore(
cluster.getSolrClient(),
configSet + "_sample",
- DefaultSampleDocumentsLoader.streamAsBytes(toJavabin(Collections.singletonList(doc))));
+ SchemaDesignerConfigSetHelper.readAllBytes(() -> toJavabin(List.of(doc))));
List<SolrInputDocument> docs = helper.getStoredSampleDocs(configSet);
assertTrue(docs != null && docs.size() == 1);
diff --git a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java
index be423b1a21e..cd4285b19fb 100644
--- a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java
+++ b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java
@@ -139,22 +139,13 @@ public class TestCborDataFormat extends SolrCloudTestCase {
request.setResponseParser(new InputStreamResponseParser(wt));
}
result = client.request(request, testCollection);
- byte[] b = copyStream((InputStream) result.get("stream"));
+ InputStream inputStream = (InputStream) result.get("stream");
+ byte[] b = inputStream.readAllBytes();
System.out.println(wt + "_time : " + timer.getTime());
System.out.println(wt + "_size : " + b.length);
return b;
}
- private static byte[] copyStream(InputStream inputStream) throws IOException {
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- byte[] buffer = new byte[4096];
- int bytesRead;
- while ((bytesRead = inputStream.read(buffer)) != -1) {
- outputStream.write(buffer, 0, bytesRead);
- }
- return outputStream.toByteArray();
- }
-
private void modifySchema(String testCollection, CloudSolrClient client)
throws SolrServerException, IOException {
GenericSolrRequest req =
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/BytesOutputStream.java b/solr/solrj/src/java/org/apache/solr/common/util/BytesOutputStream.java
deleted file mode 100644
index b1b2bd45644..00000000000
--- a/solr/solrj/src/java/org/apache/solr/common/util/BytesOutputStream.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.common.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
-
-public class BytesOutputStream extends OutputStream {
- private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
-
- protected byte[] buf;
-
- protected int sz;
-
- public BytesOutputStream() {
- this(64);
- }
-
- public BytesOutputStream(int size) {
- if (size < 0) {
- throw new IllegalArgumentException("Size must be > 0: " + size);
- }
- buf = new byte[size];
- }
-
- public byte[] toBytes() {
- return Arrays.copyOf(buf, sz);
- }
-
- public Bytes bytes() {
- return new Bytes(buf, 0, sz);
- }
-
- public InputStream inputStream() {
- return new ByteArrayInputStream(buf);
- }
-
- private void ensureCapacity(int minCapacity) {
- if (minCapacity - buf.length > 0) expandBuf(minCapacity);
- }
-
- /** * Write a byte to the stream. */
- @Override
- public void write(int b) {
-
- try {
- buf[sz] = (byte) b;
- sz += 1;
- } catch (IndexOutOfBoundsException e) {
- ensureCapacity(sz + 1);
- buf[sz] = (byte) b;
- sz += 1;
- }
- }
-
- @Override
- public void write(byte[] b, int off, int len) {
- try {
- System.arraycopy(b, off, buf, sz, len);
- sz += len;
- } catch (IndexOutOfBoundsException e) {
- ensureCapacity(sz + len);
- System.arraycopy(b, off, buf, sz, len);
- sz += len;
- }
- }
-
- public void writeBytes(byte[] b) {
- write(b, 0, b.length);
- }
-
- public void reset() {
- sz = 0;
- }
-
- public int size() {
- return sz;
- }
-
- public String toString(String charset) {
- try {
- return new String(buf, 0, sz, charset);
- } catch (UnsupportedEncodingException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- private void expandBuf(int minCapacity) {
- int oldCapacity = buf.length;
- int newCapacity = oldCapacity << 1;
- if (newCapacity - minCapacity < 0) newCapacity = minCapacity;
- if (newCapacity - MAX_ARRAY_SIZE > 0) {
- if (minCapacity < 0)
- // overflow
- throw new OutOfMemoryError();
- newCapacity = (minCapacity > MAX_ARRAY_SIZE) ? Integer.MAX_VALUE : MAX_ARRAY_SIZE;
- }
- buf = Arrays.copyOf(buf, newCapacity);
- }
-
- @Override
- public void close() {
- // noop
- }
-
- public static class Bytes {
-
- public final byte[] bytes;
- public final int offset;
- public final int length;
-
- public Bytes(byte[] bytes, int offset, int length) {
- this.bytes = bytes;
- this.offset = offset;
- this.length = length;
- }
- }
-}
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
index 8a9aec1b80f..7102ab75598 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java
@@ -1175,29 +1175,10 @@ public class Utils {
}
}
+ /** Reads an input stream into a byte array. Does not close the input. */
public static ByteBuffer toByteArray(InputStream is) throws IOException {
- return toByteArray(is, Integer.MAX_VALUE);
- }
-
- /**
- * Reads an input stream into a byte array
- *
- * @param is the input stream
- * @return the byte array
- * @throws IOException If there is a low-level I/O error.
- */
- public static ByteBuffer toByteArray(InputStream is, long maxSize) throws IOException {
try (BAOS bos = new BAOS()) {
- long sz = 0;
- int next = is.read();
- while (next > -1) {
- if (++sz > maxSize) {
- throw new BufferOverflowException();
- }
- bos.write(next);
- next = is.read();
- }
- bos.flush();
+ is.transferTo(bos);
return bos.getByteBuffer();
}
}