Skip to content

web search done right

use more sophisticated encoding detection when utf8 decoding fails. (#172)

Mikkel Denkerf8c58b3c03df
diff --git a/Cargo.lock b/Cargo.lock
index 70a54258..8ad805ab 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -704,6 +704,17 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+[[package]]
+name = "chardetng"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
[[package]]
name = "chitchat"
version = "0.5.0"
@@ -4459,6 +4470,7 @@ dependencies = [
"candle-core",
"candle-nn",
"candle-transformers",
+ "chardetng",
"chitchat",
"chrono",
"clap",
diff --git a/Cargo.toml b/Cargo.toml
index d5ca6e25..b2a78c4a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,9 +1,9 @@
[workspace]
members = [
- "crates/core",
- "crates/optics",
- "crates/kuchiki",
- "crates/zimba",
+ "crates/core",
+ "crates/optics",
+ "crates/kuchiki",
+ "crates/zimba",
]
resolver = "2"
@@ -24,16 +24,20 @@ axum-extra = {version = "0.9.0"}
axum-macros = "0.4.0"
base64 = "0.21.4"
bincode = "1.3.3"
+bindgen = "0.69.2"
bitvec = "1.0.1"
bytemuck = {version = "1.13.1", features = ["derive"]}
byteorder = "1.4.3"
bzip2 = "0.4.4"
-candle-transformers = "0.3.3"
-candle-nn = "0.3.3"
candle-core = "0.3.3"
+candle-nn = "0.3.3"
+candle-transformers = "0.3.3"
+cc = {version = "1", features = ["parallel"]}
+chardetng = "0.1.17"
chitchat = "0.5.0"
chrono = {version = "0.4.23", features = ["serde"]}
clap = {version = "4.4.6", features = ["derive"]}
+cmake = "0.1"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
csv = "1.1.6"
@@ -46,7 +50,7 @@ fnv = "1.0.3"
fst = {version = "0.4.7", features = ["levenshtein"]}
futures = "0.3.21"
half = {version = "2.2.1", features = ["serde"]}
-hashbrown = {version = "0.14.0", features = ["serde" ]}
+hashbrown = {version = "0.14.0", features = ["serde"]}
http = "1.0.0"
image = "0.24.3"
indicatif = {version = "0.17.7", features = ["rayon"]}
@@ -110,9 +114,6 @@ utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}
uuid = "1.1.2"
whatlang = {version = "0.16.0", features = ["serde"]}
zstd = "0.13"
-bindgen = "0.69.2"
-cmake = "0.1"
-cc = { version = "1", features = ["parallel"] }
[profile.test.package]
flate2.opt-level = 3
diff --git a/assets/licenses.html b/assets/licenses.html
index e6a53b5f..ffb81a34 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,7 +44,7 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (388)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (389)</li>
<li><a href="#MIT">MIT License</a> (175)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (8)</li>
@@ -1101,6 +1101,7 @@
<h3 id="Apache-2.0">Apache License 2.0</h3>
<h4>Used by:</h4>
<ul class="license-used-by">
+ <li><a href=" https://github.com/hsivonen/chardetng ">chardetng 0.1.17</a></li>
<li><a href=" https://github.com/KyleMayes/clang-sys ">clang-sys 1.7.0</a></li>
<li><a href=" https://github.com/tormol/encode_unicode ">encode_unicode 0.3.6</a></li>
<li><a href=" https://github.com/hsivonen/encoding_rs ">encoding_rs 0.8.33</a></li>
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 229f6456..64097cfe 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -4,8 +4,8 @@ default-run = "stract"
edition = "2021"
license = "AGPL-3.0"
name = "stract"
-version = "0.1.0"
publish = false
+version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -33,9 +33,10 @@ bitvec = {workspace = true}
bytemuck = {workspace = true}
byteorder = {workspace = true}
bzip2 = {workspace = true}
-candle-transformers = {workspace = true}
-candle-nn = {workspace = true}
candle-core = {workspace = true}
+candle-nn = {workspace = true}
+candle-transformers = {workspace = true}
+chardetng = {workspace = true}
chitchat = {workspace = true}
chrono = {workspace = true}
clap = {workspace = true}
diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs
index c8664dc4..cffeff1c 100644
--- a/crates/core/src/warc.rs
+++ b/crates/core/src/warc.rs
@@ -42,19 +42,17 @@ fn rtrim(s: &mut String) {
s.truncate(s.trim_end().len());
}
-fn decode(raw: &[u8]) -> String {
+fn decode_string(raw: &[u8]) -> String {
if let Ok(res) = String::from_utf8(raw.to_owned()) {
res
} else {
- let encodings = [
- encoding_rs::WINDOWS_1251,
- encoding_rs::GBK,
- encoding_rs::SHIFT_JIS,
- encoding_rs::EUC_JP,
- encoding_rs::EUC_KR,
- ];
-
- for enc in encodings {
+ let mut detector = chardetng::EncodingDetector::new();
+ detector.feed(raw, true);
+ let (enc, conf) = detector.guess_assess(None, true);
+
+ if conf {
+ dbg!(enc.name());
+
let (cow, _, had_errors) = enc.decode(raw);
if !had_errors {
return cow.to_string();
@@ -284,7 +282,7 @@ pub struct Response {
impl Response {
fn from_raw(record: RawWarcRecord) -> Result<Self> {
- let content = decode(&record.content[..]);
+ let content = decode_string(&record.content[..]);
let (_header, content) = content
.split_once("\r\n\r\n")
@@ -852,6 +850,23 @@ mod tests {
assert_eq!(records[0].metadata.fetch_time_ms, 0);
}
+ #[test]
+ fn character_encodings() {
+ for (encoding, s) in [
+ (
+ encoding_rs::WINDOWS_1252,
+ "Groupe CROISEUR LEGER après 10 courses",
+ ),
+ (encoding_rs::EUC_JP, "あいうえお"),
+ (encoding_rs::EUC_KR, "안녕하세요"),
+ ] {
+ let encoded = encoding.encode(s).0;
+ let string = decode_string(&encoded);
+
+ assert_eq!(s, string, "Failed for encoding {:?}", encoding.name());
+ }
+ }
+
proptest! {
#[test]
fn write_read_invariant_prop(records: Vec<WarcRecord>) {