Skip to content

The Unified and Autonomous Development Platform

fix: Code indexing stuck occasionally (OD-2072)

Robin Shen2e5aa6fbdf74
diff --git a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
index 54abfddb27..bf3995bb74 100644
--- a/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
+++ b/server-core/src/main/java/io/onedev/server/util/ContentDetector.java
@@ -5,12 +5,15 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.mime.MediaType;
import javax.annotation.Nullable;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
public class ContentDetector {
+ private static final int BUFSIZE = 1024;
+
private static final Tika tika = new Tika();
/**
@@ -24,23 +27,25 @@ public class ContentDetector {
@Nullable
public static Charset detectCharset(byte[] bytes) {
if (bytes.length != 0) {
- var listener = new UniversalEncodingListener(new Metadata());
- var pos = 0;
- var lookAhead = 1024;
- while (true) {
- var left = bytes.length - pos;
- if (left < lookAhead) {
- listener.handleData(bytes, pos, left);
- break;
- } else {
- listener.handleData(bytes, pos, lookAhead);
- if (listener.isDone())
- break;
- else
- pos += lookAhead;
+ var input = new ByteArrayInputStream(bytes);
+ input.mark(bytes.length);
+ try {
+ UniversalEncodingListener listener = new UniversalEncodingListener(new Metadata());
+
+ byte[] b = new byte[BUFSIZE];
+ int n = 0;
+ int m = input.read(b);
+ while (m != -1 && n < bytes.length && !listener.isDone()) {
+ n += m;
+ listener.handleData(b, 0, m);
+ m = input.read(b, 0, Math.min(b.length, bytes.length - n));
}
- }
- return listener.dataEnd();
+ return listener.dataEnd();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ input.reset();
+ }
} else {
return null;
}
diff --git a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java
index 18178a3721..fb950646f9 100644
--- a/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java
+++ b/server-core/src/main/java/io/onedev/server/util/UniversalEncodingListener.java
@@ -42,7 +42,6 @@ class UniversalEncodingListener implements CharsetListener {
}
}
- @Override
public void report(String name) {
if (Constants.CHARSET_WINDOWS_1252.equals(name)) {
if (hint != null) {
@@ -64,7 +63,7 @@ class UniversalEncodingListener implements CharsetListener {
}
try {
this.charset = CharsetUtils.forName(name);
- } catch (Exception e) {
+ } catch (IllegalArgumentException e) {
// ignore
}
}
diff --git a/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java b/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java
index 20756bdeab..8fe6e992fe 100644
--- a/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java
+++ b/server-core/src/test/java/io/onedev/server/util/ContentDetectorTest.java
@@ -1,19 +1,35 @@
package io.onedev.server.util;
+import com.google.common.io.Resources;
import org.junit.Test;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.Assert.assertEquals;
+
public class ContentDetectorTest {
@Test
public void detectCharset() {
- /*
- try (var is = Resources.getResource(ContentDetectorTest.class, "charset-detect1").openStream()) {
+ try (var is = Resources.getResource(ContentDetectorTest.class, "iso-8859-1.txt").openStream()) {
+ var bytes = IOUtils.toByteArray(is);
+ assertEquals(StandardCharsets.ISO_8859_1, ContentDetector.detectCharset(bytes));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ try (var is = Resources.getResource(ContentDetectorTest.class, "utf-8.txt").openStream()) {
+ var bytes = IOUtils.toByteArray(is);
+ assertEquals(StandardCharsets.UTF_8, ContentDetector.detectCharset(bytes));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ try (var is = Resources.getResource(ContentDetectorTest.class, "gb18030.txt").openStream()) {
var bytes = IOUtils.toByteArray(is);
- ContentDetector.detectCharset(bytes);
+ assertEquals("GB18030", ContentDetector.detectCharset(bytes).name());
} catch (IOException e) {
throw new RuntimeException(e);
}
- */
}
}
\ No newline at end of file
diff --git a/server-core/src/test/java/io/onedev/server/util/gb18030.txt b/server-core/src/test/java/io/onedev/server/util/gb18030.txt
new file mode 100644
index 0000000000..0bbab65e0c
--- /dev/null
+++ b/server-core/src/test/java/io/onedev/server/util/gb18030.txt
@@ -0,0 +1,4 @@
+��ǰ���¹�
+���ǵ���˪
+��ͷ������
+��ͷ˼����
diff --git a/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txt b/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txt
new file mode 100644
index 0000000000..f8e4860627
--- /dev/null
+++ b/server-core/src/test/java/io/onedev/server/util/iso-8859-1.txt
@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: poetry-core
+Version: 1.5.0
+Summary: Poetry PEP 517 Build Backend
+Home-page: https://github.com/python-poetry/poetry-core
+License: MIT
+Keywords: packaging,dependency,poetry
diff --git a/server-core/src/test/java/io/onedev/server/util/charset-detect1 b/server-core/src/test/java/io/onedev/server/util/utf-8.txt
similarity index 100%
rename from server-core/src/test/java/io/onedev/server/util/charset-detect1
rename to server-core/src/test/java/io/onedev/server/util/utf-8.txt