web search done right
change configure command to use new object store
Mikkel Denkerb91d655b2f57
diff --git a/Cargo.lock b/Cargo.lockindex e66615e9..10066b31 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -350,15 +350,6 @@ dependencies = ["wyz",]-[[package]]-name = "block-buffer"-version = "0.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"-dependencies = [- "generic-array",-]-[[package]]name = "block-buffer"version = "0.10.3"@@ -440,7 +431,7 @@ dependencies = ["reqwest","serde","serde_json",- "sha2 0.10.6",+ "sha2","tar","tempfile","thiserror",@@ -738,16 +729,6 @@ dependencies = ["typenum",]-[[package]]-name = "crypto-mac"-version = "0.11.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"-dependencies = [- "generic-array",- "subtle",-]-[[package]]name = "cssparser"version = "0.27.2"@@ -846,8 +827,6 @@ dependencies = ["regex","reqwest","rocksdb",- "rusoto_core",- "rusoto_s3","rust-stemmers","serde","serde_json",@@ -989,22 +968,13 @@ version = "0.1.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"-[[package]]-name = "digest"-version = "0.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"-dependencies = [- "generic-array",-]-[[package]]name = "digest"version = "0.10.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"dependencies = [- "block-buffer 0.10.3",+ "block-buffer","crypto-common",]@@ -1498,22 +1468,6 @@ dependencies = ["libc",]-[[package]]-name = "hex"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"--[[package]]-name = "hmac"-version = "0.11.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"-dependencies = [- "crypto-mac",- "digest 0.9.0",-]-[[package]]name = "html-escape"version = "0.2.13"@@ -1613,21 +1567,6 @@ dependencies = ["want",]-[[package]]-name = "hyper-rustls"-version = "0.23.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"-dependencies = [- "http",- "hyper",- "log",- "rustls",- "rustls-native-certs",- "tokio",- "tokio-rustls",-]-[[package]]name = "hyper-tls"version = "0.5.0"@@ -2081,17 +2020,6 @@ dependencies = ["rawpointer",]-[[package]]-name = "md-5"-version = "0.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"-dependencies = [- "block-buffer 0.9.0",- "digest 0.9.0",- "opaque-debug",-]-[[package]]name = "md5"version = "0.7.0"@@ -2399,12 +2327,6 @@ version = "11.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"-[[package]]-name = "opaque-debug"-version = "0.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"-[[package]]name = "openssl"version = "0.10.45"@@ -3011,88 +2933,6 @@ dependencies = ["librocksdb-sys",]-[[package]]-name = "rusoto_core"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"-dependencies = [- "async-trait",- "base64 0.13.1",- "bytes",- "crc32fast",- "futures",- "http",- "hyper",- "hyper-rustls",- "lazy_static",- "log",- "rusoto_credential",- "rusoto_signature",- "rustc_version",- "serde",- "serde_json",- "tokio",- "xml-rs",-]--[[package]]-name = "rusoto_credential"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"-dependencies = [- "async-trait",- "chrono",- "dirs-next",- "futures",- "hyper",- "serde",- "serde_json",- "shlex",- "tokio",- "zeroize",-]--[[package]]-name = "rusoto_s3"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"-dependencies = [- "async-trait",- "bytes",- "futures",- "rusoto_core",- "xml-rs",-]--[[package]]-name = "rusoto_signature"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"-dependencies = [- "base64 0.13.1",- "bytes",- "chrono",- "digest 0.9.0",- "futures",- "hex",- "hmac",- "http",- "hyper",- "log",- "md-5",- "percent-encoding",- "pin-project-lite",- "rusoto_credential",- "rustc_version",- "serde",- "sha2 0.9.9",- "tokio",-]-[[package]]name = "rust-stemmers"version = "1.2.0"@@ -3130,27 +2970,6 @@ dependencies = ["webpki",]-[[package]]-name = "rustls-native-certs"-version = "0.6.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"-dependencies = [- "openssl-probe",- "rustls-pemfile",- "schannel",- "security-framework",-]--[[package]]-name = "rustls-pemfile"-version = "1.0.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"-dependencies = [- "base64 0.21.0",-]-[[package]]name = "rustversion"version = "1.0.11"@@ -3327,19 +3146,6 @@ dependencies = ["stable_deref_trait",]-[[package]]-name = "sha2"-version = "0.9.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"-dependencies = [- "block-buffer 0.9.0",- "cfg-if",- "cpufeatures",- "digest 0.9.0",- "opaque-debug",-]-[[package]]name = "sha2"version = "0.10.6"@@ -3348,7 +3154,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"dependencies = ["cfg-if","cpufeatures",- "digest 0.10.6",+ "digest",][[package]]@@ -3489,12 +3295,6 @@ version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"-[[package]]-name = "subtle"-version = "2.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"-[[package]]name = "syn"version = "1.0.107"@@ -3878,17 +3678,6 @@ dependencies = ["tokio",]-[[package]]-name = "tokio-rustls"-version = "0.23.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"-dependencies = [- "rustls",- "tokio",- "webpki",-]-[[package]]name = "tokio-stream"version = "0.1.11"@@ -4551,18 +4340,6 @@ dependencies = ["libc",]-[[package]]-name = "xml-rs"-version = "0.8.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"--[[package]]-name = "zeroize"-version = "1.5.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"-[[package]]name = "zip"version = "0.5.13"diff --git a/Cargo.toml b/Cargo.tomlindex c9828b87..376624d8 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -57,8 +57,6 @@ base64 = "0.13.0"async-trait = "0.1.57"axum-macros = "0.2.3"urlencoding = "2.1.2"-rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false }-rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] }bitvec = "1.0.1"criterion = "0.3.6"maplit = "1.0.2"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex c2d4979c..00af4f19 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -9,7 +9,7 @@ default-run = "cuely"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[features]-dev = ["rusoto_s3", "rusoto_core"]+dev = [][[bin]]name = "cuely"@@ -61,8 +61,6 @@ base64 = { workspace = true }async-trait = { workspace = true }axum-macros = { workspace = true }urlencoding = { workspace = true }-rusoto_s3 = { workspace = true, optional = true }-rusoto_core = { workspace = true, optional = true }bitvec = { workspace = true }tower-http = { workspace = true }once_cell = { workspace = true }diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 56d675d3..cde0b0f7 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -14,11 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use rusoto_core::credential::StaticProvider;-use rusoto_core::HttpClient;-use rusoto_s3::{GetObjectRequest, S3Client, S3};use tokio::fs::File;use tokio::io;+use tokio_stream::StreamExt;use tracing::{debug, info};use crate::entrypoint::{dmoz_parser, indexer};@@ -37,13 +35,6 @@ fn download_files() {.build().unwrap().block_on(async {- let creds = StaticProvider::new(String::new(), String::new(), None, None);- let client = S3Client::new_with(- HttpClient::new().unwrap(),- creds,- rusoto_core::Region::EuCentral1,- );-for name in ["enwiki_subset.xml.bz2","queries_us.csv",@@ -52,20 +43,18 @@ fn download_files() {"content.rdf.u8.gz",] {info!("Downloading {}", name);- let mut object = client- .get_object(GetObjectRequest {- bucket: BUCKET_NAME.into(),- key: name.into(),- ..Default::default()- })+ let body = reqwest::get(format!("http://s3.cuely.io/{BUCKET_NAME}/{name}")).await.unwrap();- let body = object.body.take().expect("The object has no body");-- let mut body = body.into_async_read();let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();- io::copy(&mut body, &mut file).await.unwrap();+ let mut bytes = body.bytes_stream();++ while let Some(item) = bytes.next().await {+ io::copy(&mut item.unwrap().as_ref(), &mut file)+ .await+ .unwrap();+ }}});}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 5dd55ec3..252a6e8f 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -41,7 +41,7 @@ impl ToString for Term {fn to_string(&self) -> String {match self {Term::Simple(term) => term.clone(),- Term::Phrase(phrase) => "\"".to_string() + phrase + "\"",+ Term::Phrase(phrase) => "\"".to_string() + phrase.as_str() + "\"",Term::Not(term) => "-".to_string() + term.to_string().as_str(),Term::Site(site) => "site:".to_string() + site.as_str(),Term::Title(title) => "intitle:".to_string() + title.as_str(),diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 9dd3d95f..6b2523b4 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -138,7 +138,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {for website in websites.iter_mut() {let website = website.as_mut_ranking();- let text = website.title.clone() + ". " + &website.clean_body;+ let text = website.title.clone() + ". " + website.clean_body.as_str();bodies.push(text);}diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex 83f5aabd..f6142799 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -58,7 +58,8 @@ impl From<spell::Correction> for HighlightedSpellCorrection {for term in correction.terms {match term {CorrectionTerm::Corrected(correction) => {- highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));+ highlighted+ .push_str(&("<b><i>".to_string() + correction.as_str() + "</i></b>"));raw.push_str(&correction);}CorrectionTerm::NotCorrected(orig) => {@@ -96,7 +97,7 @@ fn prettify_url(url: &Url) -> String {pretty_url = Url::from(pretty_url.clone()).strip_protocol().replace('/', " › ");- pretty_url = protocol + &pretty_url;+ pretty_url = protocol + pretty_url.as_str();pretty_url}diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex d5396c7b..ce38af79 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -158,25 +158,25 @@ impl Url {if prot.is_empty() {url.full() + self.0.strip_prefix('/').unwrap()} else {- prot + "://" + url.site() + &self.0+ prot + "://" + url.site() + self.0.as_str()}}- (true, false) => url.full() + &self.0,+ (true, false) => url.full() + self.0.as_str(),(false, true) => {let prot = url.protocol().to_string();if prot.is_empty() {- url.full() + &self.0+ url.full() + self.0.as_str()} else {- prot + "://" + url.site() + &self.0+ prot + "://" + url.site() + self.0.as_str()}}- (false, false) => url.full() + "/" + &self.0,+ (false, false) => url.full() + "/" + self.0.as_str(),};}pub fn full(&self) -> String {if self.find_protocol_end() == 0 {- "https://".to_string() + &self.0+ "https://".to_string() + self.0.as_str()} else {self.0.clone()}