The Unified and Autonomous Development Platform
fix: Code indexing stuck occasionally (OD-2072)
Robin Shen2e5aa6fbdf74
diff --git a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java b/server-core/src/main/java/io/onedev/server/util/ContentDetector.javaindex 54abfddb27..bf3995bb74 100644--- a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java+++ b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java@@ -5,12 +5,15 @@ import org.apache.tika.metadata.Metadata;import org.apache.tika.mime.MediaType;import javax.annotation.Nullable;+import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.InputStream;import java.nio.charset.Charset;public class ContentDetector {+ private static final int BUFSIZE = 1024;+private static final Tika tika = new Tika();/**@@ -24,23 +27,25 @@ public class ContentDetector {@Nullablepublic static Charset detectCharset(byte[] bytes) {if (bytes.length != 0) {- var listener = new UniversalEncodingListener(new Metadata());- var pos = 0;- var lookAhead = 1024;- while (true) {- var left = bytes.length - pos;- if (left < lookAhead) {- listener.handleData(bytes, pos, left);- break;- } else {- listener.handleData(bytes, pos, lookAhead);- if (listener.isDone())- break;- else- pos += lookAhead;+ var input = new ByteArrayInputStream(bytes);+ input.mark(bytes.length);+ try {+ UniversalEncodingListener listener = new UniversalEncodingListener(new Metadata());++ byte[] b = new byte[BUFSIZE];+ int n = 0;+ int m = input.read(b);+ while (m != -1 && n < bytes.length && !listener.isDone()) {+ n += m;+ listener.handleData(b, 0, m);+ m = input.read(b, 0, Math.min(b.length, bytes.length - n));}- }- return listener.dataEnd();+ return listener.dataEnd();+ } catch (IOException e) {+ throw new RuntimeException(e);+ } finally {+ input.reset();+ }} else {return null;}diff --git a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.javaindex 18178a3721..fb950646f9 100644--- a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java+++ b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java@@ -42,7 +42,6 @@ class UniversalEncodingListener implements CharsetListener {}}- @Overridepublic void report(String name) {if (Constants.CHARSET_WINDOWS_1252.equals(name)) {if (hint != null) {@@ -64,7 +63,7 @@ class UniversalEncodingListener implements CharsetListener {}try {this.charset = CharsetUtils.forName(name);- } catch (Exception e) {+ } catch (IllegalArgumentException e) {// ignore}}diff --git a/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java b/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.javaindex 20756bdeab..8fe6e992fe 100644--- a/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java+++ b/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java@@ -1,19 +1,35 @@package io.onedev.server.util;+import com.google.common.io.Resources;import org.junit.Test;+import java.io.IOException;+import java.nio.charset.StandardCharsets;++import static org.junit.Assert.assertEquals;+public class ContentDetectorTest {@Testpublic void detectCharset() {- /*- try (var is = Resources.getResource(ContentDetectorTest.class, "charset-detect1").openStream()) {+ try (var is = Resources.getResource(ContentDetectorTest.class, "iso-8859-1.txt").openStream()) {+ var bytes = IOUtils.toByteArray(is);+ assertEquals(StandardCharsets.ISO_8859_1, ContentDetector.detectCharset(bytes));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ try (var is = Resources.getResource(ContentDetectorTest.class, "utf-8.txt").openStream()) {+ var bytes = IOUtils.toByteArray(is);+ assertEquals(StandardCharsets.UTF_8, ContentDetector.detectCharset(bytes));+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ try (var is = Resources.getResource(ContentDetectorTest.class, "gb18030.txt").openStream()) {var bytes = IOUtils.toByteArray(is);- ContentDetector.detectCharset(bytes);+ assertEquals("GB18030", ContentDetector.detectCharset(bytes).name());} catch (IOException e) {throw new RuntimeException(e);}- */}}\ No newline at end of filediff --git a/server-core/src/test/java/io/onedev/server/util/gb18030.txt b/server-core/src/test/java/io/onedev/server/util/gb18030.txtnew file mode 100644index 0000000000..0bbab65e0c--- /dev/null+++ b/server-core/src/test/java/io/onedev/server/util/gb18030.txt@@ -0,0 +1,4 @@+��ǰ���¹�+���ǵ���˪+��ͷ������+��ͷ˼����diff --git a/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txt b/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txtnew file mode 100644index 0000000000..f8e4860627--- /dev/null+++ b/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txt@@ -0,0 +1,7 @@+Metadata-Version: 2.1+Name: poetry-core+Version: 1.5.0+Summary: Poetry PEP 517 Build Backend+Home-page: https://github.com/python-poetry/poetry-core+License: MIT+Keywords: packaging,dependency,poetrydiff --git a/server-core/src/test/java/io/onedev/server/util/charset-detect1 b/server-core/src/test/java/io/onedev/server/util/utf-8.txtsimilarity index 100%rename from server-core/src/test/java/io/onedev/server/util/charset-detect1rename to server-core/src/test/java/io/onedev/server/util/utf-8.txt