web search done right
use more sophisticated encoding detection when utf8 decoding fails. (#172)
Mikkel Denkerf8c58b3c03df
diff --git a/Cargo.lock b/Cargo.lockindex 70a54258..8ad805ab 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -704,6 +704,17 @@ version = "1.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"+[[package]]+name = "chardetng"+version = "0.1.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea"+dependencies = [+ "cfg-if",+ "encoding_rs",+ "memchr",+]+[[package]]name = "chitchat"version = "0.5.0"@@ -4459,6 +4470,7 @@ dependencies = ["candle-core","candle-nn","candle-transformers",+ "chardetng","chitchat","chrono","clap",diff --git a/Cargo.toml b/Cargo.tomlindex d5ca6e25..b2a78c4a 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,9 +1,9 @@[workspace]members = [- "crates/core",- "crates/optics",- "crates/kuchiki",- "crates/zimba",+ "crates/core",+ "crates/optics",+ "crates/kuchiki",+ "crates/zimba",]resolver = "2"@@ -24,16 +24,20 @@ axum-extra = {version = "0.9.0"}axum-macros = "0.4.0"base64 = "0.21.4"bincode = "1.3.3"+bindgen = "0.69.2"bitvec = "1.0.1"bytemuck = {version = "1.13.1", features = ["derive"]}byteorder = "1.4.3"bzip2 = "0.4.4"-candle-transformers = "0.3.3"-candle-nn = "0.3.3"candle-core = "0.3.3"+candle-nn = "0.3.3"+candle-transformers = "0.3.3"+cc = {version = "1", features = ["parallel"]}+chardetng = "0.1.17"chitchat = "0.5.0"chrono = {version = "0.4.23", features = ["serde"]}clap = {version = "4.4.6", features = ["derive"]}+cmake = "0.1"criterion = "0.5.1"crossbeam-channel = "0.5.6"csv = "1.1.6"@@ -46,7 +50,7 @@ fnv = "1.0.3"fst = {version = "0.4.7", features = ["levenshtein"]}futures = "0.3.21"half = {version = "2.2.1", features = ["serde"]}-hashbrown = {version = "0.14.0", features = ["serde" ]}+hashbrown = {version = "0.14.0", features = ["serde"]}http = "1.0.0"image = "0.24.3"indicatif = {version = "0.17.7", features = ["rayon"]}@@ -110,9 +114,6 @@ utoipa-swagger-ui = {version = "5.0.0", features = ["axum"]}uuid = "1.1.2"whatlang = {version = "0.16.0", features = ["serde"]}zstd = "0.13"-bindgen = "0.69.2"-cmake = "0.1"-cc = { version = "1", features = ["parallel"] }[profile.test.package]flate2.opt-level = 3diff --git a/assets/licenses.html b/assets/licenses.htmlindex e6a53b5f..ffb81a34 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (388)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (389)</li><li><a href="#MIT">MIT License</a> (175)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (8)</li>@@ -1101,6 +1101,7 @@<h3 id="Apache-2.0">Apache License 2.0</h3><h4>Used by:</h4><ul class="license-used-by">+ <li><a href=" https://github.com/hsivonen/chardetng ">chardetng 0.1.17</a></li><li><a href=" https://github.com/KyleMayes/clang-sys ">clang-sys 1.7.0</a></li><li><a href=" https://github.com/tormol/encode_unicode ">encode_unicode 0.3.6</a></li><li><a href=" https://github.com/hsivonen/encoding_rs ">encoding_rs 0.8.33</a></li>diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 229f6456..64097cfe 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -4,8 +4,8 @@ default-run = "stract"edition = "2021"license = "AGPL-3.0"name = "stract"-version = "0.1.0"publish = false+version = "0.1.0"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html@@ -33,9 +33,10 @@ bitvec = {workspace = true}bytemuck = {workspace = true}byteorder = {workspace = true}bzip2 = {workspace = true}-candle-transformers = {workspace = true}-candle-nn = {workspace = true}candle-core = {workspace = true}+candle-nn = {workspace = true}+candle-transformers = {workspace = true}+chardetng = {workspace = true}chitchat = {workspace = true}chrono = {workspace = true}clap = {workspace = true}diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rsindex c8664dc4..cffeff1c 100644--- a/crates/core/src/warc.rs+++ b/crates/core/src/warc.rs@@ -42,19 +42,17 @@ fn rtrim(s: &mut String) {s.truncate(s.trim_end().len());}-fn decode(raw: &[u8]) -> String {+fn decode_string(raw: &[u8]) -> String {if let Ok(res) = String::from_utf8(raw.to_owned()) {res} else {- let encodings = [- encoding_rs::WINDOWS_1251,- encoding_rs::GBK,- encoding_rs::SHIFT_JIS,- encoding_rs::EUC_JP,- encoding_rs::EUC_KR,- ];-- for enc in encodings {+ let mut detector = chardetng::EncodingDetector::new();+ detector.feed(raw, true);+ let (enc, conf) = detector.guess_assess(None, true);++ if conf {+ dbg!(enc.name());+let (cow, _, had_errors) = enc.decode(raw);if !had_errors {return cow.to_string();@@ -284,7 +282,7 @@ pub struct Response {impl Response {fn from_raw(record: RawWarcRecord) -> Result<Self> {- let content = decode(&record.content[..]);+ let content = decode_string(&record.content[..]);let (_header, content) = content.split_once("\r\n\r\n")@@ -852,6 +850,23 @@ mod tests {assert_eq!(records[0].metadata.fetch_time_ms, 0);}+ #[test]+ fn character_encodings() {+ for (encoding, s) in [+ (+ encoding_rs::WINDOWS_1252,+ "Groupe CROISEUR LEGER après 10 courses",+ ),+ (encoding_rs::EUC_JP, "あいうえお"),+ (encoding_rs::EUC_KR, "안녕하세요"),+ ] {+ let encoded = encoding.encode(s).0;+ let string = decode_string(&encoded);++ assert_eq!(s, string, "Failed for encoding {:?}", encoding.name());+ }+ }+proptest! {#[test]fn write_read_invariant_prop(records: Vec<WarcRecord>) {