The Unified and Autonomous Development Platform
fix: Incorrect character encoding detected for long text (#1871)
Robin Shend2dc6633bd00
diff --git a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java b/server-core/src/main/java/io/onedev/server/util/ContentDetector.javaindex f9d64b9300..54abfddb27 100644--- a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java+++ b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java@@ -1,107 +1,104 @@-package io.onedev.server.util;--import java.io.IOException;-import java.io.InputStream;-import java.nio.charset.Charset;--import javax.annotation.Nullable;--import org.apache.tika.Tika;-import org.apache.tika.mime.MediaType;--public class ContentDetector {-- private static final Tika tika = new Tika();-- /**- * Read leading information of specified stream until the charset is detected.- *- * @param contentStream- * stream to be read for charset detection- * @return- * detected charset, or <tt>null</tt> if charset can not be detected- */- public static @Nullable Charset detectCharset(InputStream contentStream) {- try {- return UniversalEncodingDetector.detect(contentStream);- } catch (IOException e) {- throw new RuntimeException(e);- }- }-- /**- * Read leading information of specified content bytes to detect content charset.- *- * @param contentBytes- * content to be detected- * @return- * charset of the content, or <tt>null</tt> if charset can not be detected- */- public static @Nullable Charset detectCharset(byte[] contentBytes) {- if (contentBytes.length != 0) {- try {- return UniversalEncodingDetector.detect(contentBytes);- } catch (IOException e) {- throw new RuntimeException(e);- }- } else {- return null;- }- }-- public static boolean isBinary(byte[] contentBytes, @Nullable String fileName) {- if (contentBytes.length == 0)- return false;-- MediaType mediaType = detectMediaType(contentBytes, fileName);-- return !mediaType.getType().equalsIgnoreCase("text")- && !mediaType.equals(MediaType.application("rls-services+xml"))- && !mediaType.equals(MediaType.application("xhtml+xml"))- && !mediaType.equals(MediaType.APPLICATION_XML)- && !mediaType.equals(MediaType.application("x-bat"))- && !mediaType.equals(MediaType.application("x-tex"))- && !mediaType.equals(MediaType.application("json"))- && !mediaType.equals(MediaType.application("x-sh"))- && !mediaType.equals(MediaType.application("javascript"))- && !mediaType.equals(MediaType.application("x-httpd-jsp"))- && !mediaType.equals(MediaType.application("x-httpd-php"));- }-- /**- * Get text from specified content bytes, optionally with help of file name.- *- * @param contentBytes- * content bytes to construct text from- * @param fileName- * file name to help deciding if supplied content bytes represents text- * @return- * text representation of content bytes, or <tt>null</tt> if content- * can not be converted to text- */- @Nullable- public static String convertToText(byte[] contentBytes, @Nullable String fileName) {- if (!isBinary(contentBytes, fileName)) {- Charset charset = detectCharset(contentBytes);- if (charset != null)- return new String(contentBytes, charset);- else- return new String(contentBytes);- } else {- return null;- }- }-- public static MediaType detectMediaType(byte[] contentBytes, @Nullable String fileName) {- return MediaType.parse(tika.detect(contentBytes, fileName));- }-- public static MediaType detectMediaType(InputStream contentStream, @Nullable String fileName) {- try {- return MediaType.parse(tika.detect(contentStream, fileName));- } catch (IOException e) {- throw new RuntimeException(e);- }- }--}+package io.onedev.server.util;++import org.apache.tika.Tika;+import org.apache.tika.metadata.Metadata;+import org.apache.tika.mime.MediaType;++import javax.annotation.Nullable;+import java.io.IOException;+import java.io.InputStream;+import java.nio.charset.Charset;++public class ContentDetector {++ private static final Tika tika = new Tika();++ /**+ * Read leading information of specified content bytes to detect content charset.+ *+ * @param bytes+ * content to be detected+ * @return+ * charset of the content, or <tt>null</tt> if charset can not be detected+ */+ @Nullable+ public static Charset detectCharset(byte[] bytes) {+ if (bytes.length != 0) {+ var listener = new UniversalEncodingListener(new Metadata());+ var pos = 0;+ var lookAhead = 1024;+ while (true) {+ var left = bytes.length - pos;+ if (left < lookAhead) {+ listener.handleData(bytes, pos, left);+ break;+ } else {+ listener.handleData(bytes, pos, lookAhead);+ if (listener.isDone())+ break;+ else+ pos += lookAhead;+ }+ }+ return listener.dataEnd();+ } else {+ return null;+ }+ }++ public static boolean isBinary(byte[] contentBytes, @Nullable String fileName) {+ if (contentBytes.length == 0)+ return false;++ MediaType mediaType = detectMediaType(contentBytes, fileName);++ return !mediaType.getType().equalsIgnoreCase("text")+ && !mediaType.equals(MediaType.application("rls-services+xml"))+ && !mediaType.equals(MediaType.application("xhtml+xml"))+ && !mediaType.equals(MediaType.APPLICATION_XML)+ && !mediaType.equals(MediaType.application("x-bat"))+ && !mediaType.equals(MediaType.application("x-tex"))+ && !mediaType.equals(MediaType.application("json"))+ && !mediaType.equals(MediaType.application("x-sh"))+ && !mediaType.equals(MediaType.application("javascript"))+ && !mediaType.equals(MediaType.application("x-httpd-jsp"))+ && !mediaType.equals(MediaType.application("x-httpd-php"));+ }++ /**+ * Get text from specified content bytes, optionally with help of file name.+ *+ * @param contentBytes+ * content bytes to construct text from+ * @param fileName+ * file name to help deciding if supplied content bytes represents text+ * @return+ * text representation of content bytes, or <tt>null</tt> if content+ * can not be converted to text+ */+ @Nullable+ public static String convertToText(byte[] contentBytes, @Nullable String fileName) {+ if (!isBinary(contentBytes, fileName)) {+ Charset charset = detectCharset(contentBytes);+ if (charset != null)+ return new String(contentBytes, charset);+ else+ return new String(contentBytes);+ } else {+ return null;+ }+ }++ public static MediaType detectMediaType(byte[] contentBytes, @Nullable String fileName) {+ return MediaType.parse(tika.detect(contentBytes, fileName));+ }++ public static MediaType detectMediaType(InputStream contentStream, @Nullable String fileName) {+ try {+ return MediaType.parse(tika.detect(contentStream, fileName));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }++}diff --git a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.java b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.javadeleted file mode 100644index baf0e3e3c5..0000000000--- a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingDetector.java+++ /dev/null@@ -1,78 +0,0 @@-package io.onedev.server.util;--import java.io.ByteArrayInputStream;-import java.io.IOException;-import java.io.InputStream;-import java.nio.charset.Charset;--import org.apache.tika.metadata.Metadata;-import org.mozilla.universalchardet.UniversalDetector;--/**- * Copied from tika UniversalEncodingDetector- *- */-public class UniversalEncodingDetector {- private static final int BUFSIZE = 1024;-- private static final int LOOKAHEAD = 16 * BUFSIZE;-- public static Charset detect(InputStream input) throws IOException {- input.mark(LOOKAHEAD);- try {- UniversalEncodingListener listener =- new UniversalEncodingListener(new Metadata());-- byte[] b = new byte[BUFSIZE];- int n = 0;- int m = input.read(b);- while (m != -1 && n < LOOKAHEAD && !listener.isDone()) {- n += m;- listener.handleData(b, 0, m);- m = input.read(b, 0, Math.min(b.length, LOOKAHEAD - n));- }-- return listener.dataEnd();- } catch (IOException e) {- throw e;- } finally {- input.reset();- }- }-- public static Charset detect(byte[] buffer) throws IOException {- return detect(new ByteArrayInputStream(buffer));- }-- public static boolean isBinary(byte[] bytes) throws IOException {- return isBinary(new ByteArrayInputStream(bytes));- }-- public static boolean isBinary(InputStream in) throws IOException {- byte[] buf = new byte[4];- in.mark(5);- int len = in.read(buf);- in.reset();-- UniversalDetector detector = new UniversalDetector(null);- detector.handleData(buf, 0, len);- if (detector.isDone()) {- return false;- }-- //Not UTF check ASCII text- in.mark(LOOKAHEAD);- len = 0;- int b;- while ((b = in.read()) != -1 && len < (LOOKAHEAD - 192)) {- len++;- if (b == 0) {- in.reset();- return true;- }- }- in.reset();- return false;- }--}