Skip to content

The Unified and Autonomous Development Platform

fix: Incorrect character encoding detected for long text (#1871)

Robin Shend2dc6633bd00
diff --git a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
index f9d64b9300..54abfddb27 100644
--- a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
+++ b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
@@ -1,107 +1,104 @@
-package io.onedev.server.util;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-import javax.annotation.Nullable;
-
-import org.apache.tika.Tika;
-import org.apache.tika.mime.MediaType;
-
-public class ContentDetector {
-
- private static final Tika tika = new Tika();
-
- /**
- * Read leading information of specified stream until the charset is detected.
- *
- * @param contentStream
- * stream to be read for charset detection
- * @return
- * detected charset, or <tt>null</tt> if charset can not be detected
- */
- public static @Nullable Charset detectCharset(InputStream contentStream) {
- try {
- return UniversalEncodingDetector.detect(contentStream);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Read leading information of specified content bytes to detect content charset.
- *
- * @param contentBytes
- * content to be detected
- * @return
- * charset of the content, or <tt>null</tt> if charset can not be detected
- */
- public static @Nullable Charset detectCharset(byte[] contentBytes) {
- if (contentBytes.length != 0) {
- try {
- return UniversalEncodingDetector.detect(contentBytes);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- } else {
- return null;
- }
- }
-
- public static boolean isBinary(byte[] contentBytes, @Nullable String fileName) {
- if (contentBytes.length == 0)
- return false;
-
- MediaType mediaType = detectMediaType(contentBytes, fileName);
-
- return !mediaType.getType().equalsIgnoreCase("text")
- && !mediaType.equals(MediaType.application("rls-services+xml"))
- && !mediaType.equals(MediaType.application("xhtml+xml"))
- && !mediaType.equals(MediaType.APPLICATION_XML)
- && !mediaType.equals(MediaType.application("x-bat"))
- && !mediaType.equals(MediaType.application("x-tex"))
- && !mediaType.equals(MediaType.application("json"))
- && !mediaType.equals(MediaType.application("x-sh"))
- && !mediaType.equals(MediaType.application("javascript"))
- && !mediaType.equals(MediaType.application("x-httpd-jsp"))
- && !mediaType.equals(MediaType.application("x-httpd-php"));
- }
-
- /**
- * Get text from specified content bytes, optionally with help of file name.
- *
- * @param contentBytes
- * content bytes to construct text from
- * @param fileName
- * file name to help deciding if supplied content bytes represents text
- * @return
- * text representation of content bytes, or <tt>null</tt> if content
- * can not be converted to text
- */
- @Nullable
- public static String convertToText(byte[] contentBytes, @Nullable String fileName) {
- if (!isBinary(contentBytes, fileName)) {
- Charset charset = detectCharset(contentBytes);
- if (charset != null)
- return new String(contentBytes, charset);
- else
- return new String(contentBytes);
- } else {
- return null;
- }
- }
-
- public static MediaType detectMediaType(byte[] contentBytes, @Nullable String fileName) {
- return MediaType.parse(tika.detect(contentBytes, fileName));
- }
-
- public static MediaType detectMediaType(InputStream contentStream, @Nullable String fileName) {
- try {
- return MediaType.parse(tika.detect(contentStream, fileName));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
-}
+package io.onedev.server.util;
+
+import org.apache.tika.Tika;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.mime.MediaType;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+
+public class ContentDetector {
+
+ private static final Tika tika = new Tika();
+
+ /**
+ * Read leading information of specified content bytes to detect content charset.
+ *
+ * @param bytes
+ * content to be detected
+ * @return
+ * charset of the content, or <tt>null</tt> if charset can not be detected
+ */
+ @Nullable
+ public static Charset detectCharset(byte[] bytes) {
+ if (bytes.length != 0) {
+ var listener = new UniversalEncodingListener(new Metadata());
+ var pos = 0;
+ var lookAhead = 1024;
+ while (true) {
+ var left = bytes.length - pos;
+ if (left < lookAhead) {
+ listener.handleData(bytes, pos, left);
+ break;
+ } else {
+ listener.handleData(bytes, pos, lookAhead);
+ if (listener.isDone())
+ break;
+ else
+ pos += lookAhead;
+ }
+ }
+ return listener.dataEnd();
+ } else {
+ return null;
+ }
+ }
+
+ public static boolean isBinary(byte[] contentBytes, @Nullable String fileName) {
+ if (contentBytes.length == 0)
+ return false;
+
+ MediaType mediaType = detectMediaType(contentBytes, fileName);
+
+ return !mediaType.getType().equalsIgnoreCase("text")
+ && !mediaType.equals(MediaType.application("rls-services+xml"))
+ && !mediaType.equals(MediaType.application("xhtml+xml"))
+ && !mediaType.equals(MediaType.APPLICATION_XML)
+ && !mediaType.equals(MediaType.application("x-bat"))
+ && !mediaType.equals(MediaType.application("x-tex"))
+ && !mediaType.equals(MediaType.application("json"))
+ && !mediaType.equals(MediaType.application("x-sh"))
+ && !mediaType.equals(MediaType.application("javascript"))
+ && !mediaType.equals(MediaType.application("x-httpd-jsp"))
+ && !mediaType.equals(MediaType.application("x-httpd-php"));
+ }
+
+ /**
+ * Get text from specified content bytes, optionally with help of file name.
+ *
+ * @param contentBytes
+ * content bytes to construct text from
+ * @param fileName
+ * file name to help deciding if supplied content bytes represents text
+ * @return
+ * text representation of content bytes, or <tt>null</tt> if content
+ * can not be converted to text
+ */
+ @Nullable
+ public static String convertToText(byte[] contentBytes, @Nullable String fileName) {
+ if (!isBinary(contentBytes, fileName)) {
+ Charset charset = detectCharset(contentBytes);
+ if (charset != null)
+ return new String(contentBytes, charset);
+ else
+ return new String(contentBytes);
+ } else {
+ return null;
+ }
+ }
+
+ public static MediaType detectMediaType(byte[] contentBytes, @Nullable String fileName) {
+ return MediaType.parse(tika.detect(contentBytes, fileName));
+ }
+
+ public static MediaType detectMediaType(InputStream contentStream, @Nullable String fileName) {
+ try {
+ return MediaType.parse(tika.detect(contentStream, fileName));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
diff --git a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.java b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.java
deleted file mode 100644
index baf0e3e3c5..0000000000
--- a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.java
+++ /dev/null
@@ -1,78 +0,0 @@
-package io.onedev.server.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-
-import org.apache.tika.metadata.Metadata;
-import org.mozilla.universalchardet.UniversalDetector;
-
-/**
- * Copied from tika UniversalEncodingDetector
- *
- */
-public class UniversalEncodingDetector {
- private static final int BUFSIZE = 1024;
-
- private static final int LOOKAHEAD = 16 * BUFSIZE;
-
- public static Charset detect(InputStream input) throws IOException {
- input.mark(LOOKAHEAD);
- try {
- UniversalEncodingListener listener =
- new UniversalEncodingListener(new Metadata());
-
- byte[] b = new byte[BUFSIZE];
- int n = 0;
- int m = input.read(b);
- while (m != -1 && n < LOOKAHEAD && !listener.isDone()) {
- n += m;
- listener.handleData(b, 0, m);
- m = input.read(b, 0, Math.min(b.length, LOOKAHEAD - n));
- }
-
- return listener.dataEnd();
- } catch (IOException e) {
- throw e;
- } finally {
- input.reset();
- }
- }
-
- public static Charset detect(byte[] buffer) throws IOException {
- return detect(new ByteArrayInputStream(buffer));
- }
-
- public static boolean isBinary(byte[] bytes) throws IOException {
- return isBinary(new ByteArrayInputStream(bytes));
- }
-
- public static boolean isBinary(InputStream in) throws IOException {
- byte[] buf = new byte[4];
- in.mark(5);
- int len = in.read(buf);
- in.reset();
-
- UniversalDetector detector = new UniversalDetector(null);
- detector.handleData(buf, 0, len);
- if (detector.isDone()) {
- return false;
- }
-
- //Not UTF check ASCII text
- in.mark(LOOKAHEAD);
- len = 0;
- int b;
- while ((b = in.read()) != -1 && len < (LOOKAHEAD - 192)) {
- len++;
- if (b == 0) {
- in.reset();
- return true;
- }
- }
- in.reset();
- return false;
- }
-
-}