Skip to content

web search done right

change configure command to use new object store

Mikkel Denkerb91d655b2f57
diff --git a/Cargo.lock b/Cargo.lock
index e66615e9..10066b31 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -350,15 +350,6 @@ dependencies = [
"wyz",
]
-[[package]]
-name = "block-buffer"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
-dependencies = [
- "generic-array",
-]
-
[[package]]
name = "block-buffer"
version = "0.10.3"
@@ -440,7 +431,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
- "sha2 0.10.6",
+ "sha2",
"tar",
"tempfile",
"thiserror",
@@ -738,16 +729,6 @@ dependencies = [
"typenum",
]
-[[package]]
-name = "crypto-mac"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
-dependencies = [
- "generic-array",
- "subtle",
-]
-
[[package]]
name = "cssparser"
version = "0.27.2"
@@ -846,8 +827,6 @@ dependencies = [
"regex",
"reqwest",
"rocksdb",
- "rusoto_core",
- "rusoto_s3",
"rust-stemmers",
"serde",
"serde_json",
@@ -989,22 +968,13 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
-[[package]]
-name = "digest"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
-dependencies = [
- "generic-array",
-]
-
[[package]]
name = "digest"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f"
dependencies = [
- "block-buffer 0.10.3",
+ "block-buffer",
"crypto-common",
]
@@ -1498,22 +1468,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "hmac"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
-dependencies = [
- "crypto-mac",
- "digest 0.9.0",
-]
-
[[package]]
name = "html-escape"
version = "0.2.13"
@@ -1613,21 +1567,6 @@ dependencies = [
"want",
]
-[[package]]
-name = "hyper-rustls"
-version = "0.23.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
-dependencies = [
- "http",
- "hyper",
- "log",
- "rustls",
- "rustls-native-certs",
- "tokio",
- "tokio-rustls",
-]
-
[[package]]
name = "hyper-tls"
version = "0.5.0"
@@ -2081,17 +2020,6 @@ dependencies = [
"rawpointer",
]
-[[package]]
-name = "md-5"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
-dependencies = [
- "block-buffer 0.9.0",
- "digest 0.9.0",
- "opaque-debug",
-]
-
[[package]]
name = "md5"
version = "0.7.0"
@@ -2399,12 +2327,6 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
-[[package]]
-name = "opaque-debug"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
-
[[package]]
name = "openssl"
version = "0.10.45"
@@ -3011,88 +2933,6 @@ dependencies = [
"librocksdb-sys",
]
-[[package]]
-name = "rusoto_core"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
-dependencies = [
- "async-trait",
- "base64 0.13.1",
- "bytes",
- "crc32fast",
- "futures",
- "http",
- "hyper",
- "hyper-rustls",
- "lazy_static",
- "log",
- "rusoto_credential",
- "rusoto_signature",
- "rustc_version",
- "serde",
- "serde_json",
- "tokio",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_credential"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"
-dependencies = [
- "async-trait",
- "chrono",
- "dirs-next",
- "futures",
- "hyper",
- "serde",
- "serde_json",
- "shlex",
- "tokio",
- "zeroize",
-]
-
-[[package]]
-name = "rusoto_s3"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"
-dependencies = [
- "async-trait",
- "bytes",
- "futures",
- "rusoto_core",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_signature"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
-dependencies = [
- "base64 0.13.1",
- "bytes",
- "chrono",
- "digest 0.9.0",
- "futures",
- "hex",
- "hmac",
- "http",
- "hyper",
- "log",
- "md-5",
- "percent-encoding",
- "pin-project-lite",
- "rusoto_credential",
- "rustc_version",
- "serde",
- "sha2 0.9.9",
- "tokio",
-]
-
[[package]]
name = "rust-stemmers"
version = "1.2.0"
@@ -3130,27 +2970,6 @@ dependencies = [
"webpki",
]
-[[package]]
-name = "rustls-native-certs"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50"
-dependencies = [
- "openssl-probe",
- "rustls-pemfile",
- "schannel",
- "security-framework",
-]
-
-[[package]]
-name = "rustls-pemfile"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
-dependencies = [
- "base64 0.21.0",
-]
-
[[package]]
name = "rustversion"
version = "1.0.11"
@@ -3327,19 +3146,6 @@ dependencies = [
"stable_deref_trait",
]
-[[package]]
-name = "sha2"
-version = "0.9.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
-dependencies = [
- "block-buffer 0.9.0",
- "cfg-if",
- "cpufeatures",
- "digest 0.9.0",
- "opaque-debug",
-]
-
[[package]]
name = "sha2"
version = "0.10.6"
@@ -3348,7 +3154,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0"
dependencies = [
"cfg-if",
"cpufeatures",
- "digest 0.10.6",
+ "digest",
]
[[package]]
@@ -3489,12 +3295,6 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-[[package]]
-name = "subtle"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
-
[[package]]
name = "syn"
version = "1.0.107"
@@ -3878,17 +3678,6 @@ dependencies = [
"tokio",
]
-[[package]]
-name = "tokio-rustls"
-version = "0.23.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
-dependencies = [
- "rustls",
- "tokio",
- "webpki",
-]
-
[[package]]
name = "tokio-stream"
version = "0.1.11"
@@ -4551,18 +4340,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "xml-rs"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
-
-[[package]]
-name = "zeroize"
-version = "1.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
-
[[package]]
name = "zip"
version = "0.5.13"
diff --git a/Cargo.toml b/Cargo.toml
index c9828b87..376624d8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -57,8 +57,6 @@ base64 = "0.13.0"
async-trait = "0.1.57"
axum-macros = "0.2.3"
urlencoding = "2.1.2"
-rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false }
-rusoto_core = { version = "0.48.0", default-features = false, features = ["rustls"] }
bitvec = "1.0.1"
criterion = "0.3.6"
maplit = "1.0.2"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index c2d4979c..00af4f19 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -9,7 +9,7 @@ default-run = "cuely"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
-dev = ["rusoto_s3", "rusoto_core"]
+dev = []
[[bin]]
name = "cuely"
@@ -61,8 +61,6 @@ base64 = { workspace = true }
async-trait = { workspace = true }
axum-macros = { workspace = true }
urlencoding = { workspace = true }
-rusoto_s3 = { workspace = true, optional = true }
-rusoto_core = { workspace = true, optional = true }
bitvec = { workspace = true }
tower-http = { workspace = true }
once_cell = { workspace = true }
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 56d675d3..cde0b0f7 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -14,11 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use rusoto_core::credential::StaticProvider;
-use rusoto_core::HttpClient;
-use rusoto_s3::{GetObjectRequest, S3Client, S3};
use tokio::fs::File;
use tokio::io;
+use tokio_stream::StreamExt;
use tracing::{debug, info};
use crate::entrypoint::{dmoz_parser, indexer};
@@ -37,13 +35,6 @@ fn download_files() {
.build()
.unwrap()
.block_on(async {
- let creds = StaticProvider::new(String::new(), String::new(), None, None);
- let client = S3Client::new_with(
- HttpClient::new().unwrap(),
- creds,
- rusoto_core::Region::EuCentral1,
- );
-
for name in [
"enwiki_subset.xml.bz2",
"queries_us.csv",
@@ -52,20 +43,18 @@ fn download_files() {
"content.rdf.u8.gz",
] {
info!("Downloading {}", name);
- let mut object = client
- .get_object(GetObjectRequest {
- bucket: BUCKET_NAME.into(),
- key: name.into(),
- ..Default::default()
- })
+ let body = reqwest::get(format!("http://s3.cuely.io/{BUCKET_NAME}/{name}"))
.await
.unwrap();
- let body = object.body.take().expect("The object has no body");
-
- let mut body = body.into_async_read();
let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();
- io::copy(&mut body, &mut file).await.unwrap();
+ let mut bytes = body.bytes_stream();
+
+ while let Some(item) = bytes.next().await {
+ io::copy(&mut item.unwrap().as_ref(), &mut file)
+ .await
+ .unwrap();
+ }
}
});
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 5dd55ec3..252a6e8f 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -41,7 +41,7 @@ impl ToString for Term {
fn to_string(&self) -> String {
match self {
Term::Simple(term) => term.clone(),
- Term::Phrase(phrase) => "\"".to_string() + phrase + "\"",
+ Term::Phrase(phrase) => "\"".to_string() + phrase.as_str() + "\"",
Term::Not(term) => "-".to_string() + term.to_string().as_str(),
Term::Site(site) => "site:".to_string() + site.as_str(),
Term::Title(title) => "intitle:".to_string() + title.as_str(),
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 9dd3d95f..6b2523b4 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -138,7 +138,7 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
for website in websites.iter_mut() {
let website = website.as_mut_ranking();
- let text = website.title.clone() + ". " + &website.clean_body;
+ let text = website.title.clone() + ". " + website.clean_body.as_str();
bodies.push(text);
}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index 83f5aabd..f6142799 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -58,7 +58,8 @@ impl From<spell::Correction> for HighlightedSpellCorrection {
for term in correction.terms {
match term {
CorrectionTerm::Corrected(correction) => {
- highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));
+ highlighted
+ .push_str(&("<b><i>".to_string() + correction.as_str() + "</i></b>"));
raw.push_str(&correction);
}
CorrectionTerm::NotCorrected(orig) => {
@@ -96,7 +97,7 @@ fn prettify_url(url: &Url) -> String {
pretty_url = Url::from(pretty_url.clone())
.strip_protocol()
.replace('/', " › ");
- pretty_url = protocol + &pretty_url;
+ pretty_url = protocol + pretty_url.as_str();
pretty_url
}
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index d5396c7b..ce38af79 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -158,25 +158,25 @@ impl Url {
if prot.is_empty() {
url.full() + self.0.strip_prefix('/').unwrap()
} else {
- prot + "://" + url.site() + &self.0
+ prot + "://" + url.site() + self.0.as_str()
}
}
- (true, false) => url.full() + &self.0,
+ (true, false) => url.full() + self.0.as_str(),
(false, true) => {
let prot = url.protocol().to_string();
if prot.is_empty() {
- url.full() + &self.0
+ url.full() + self.0.as_str()
} else {
- prot + "://" + url.site() + &self.0
+ prot + "://" + url.site() + self.0.as_str()
}
}
- (false, false) => url.full() + "/" + &self.0,
+ (false, false) => url.full() + "/" + self.0.as_str(),
};
}
pub fn full(&self) -> String {
if self.find_protocol_end() == 0 {
- "https://".to_string() + &self.0
+ "https://".to_string() + self.0.as_str()
} else {
self.0.clone()
}