Skip to content

web search done right

sled graph store seems to work

Mikkel Denkere638d062b295
diff --git a/Cargo.toml b/Cargo.toml
index 6fadab0d..c02f8296 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ edition = "2021"
name = "cuely"
[dependencies]
+bincode = "1.3.3"
encoding_rs = "0.8.31"
flate2 = "1.0.23"
itertools = "0.10.3"
@@ -16,6 +17,8 @@ rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
scraper = "0.13.0"
serde = "1.0.137"
+sled = { version = "0.34.7", features = ["compression"] }
+tantivy = "0.17.0"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
toml = "0.5.9"
diff --git a/src/indexer.rs b/src/indexer.rs
index 0dfe6704..e294c15c 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -39,12 +39,10 @@ impl Indexer {
println!("Downloaded {} bytes", raw_object.len());
let warc = WarcFile::new(&raw_object[..]);
- for record in warc {
- if let Ok(record) = record {
- let webpage = Webpage::parse(&record.response.body);
- println!("TEST: {:?}", webpage.text());
- println!();
- }
+ for record in warc.flatten() {
+ let webpage = Webpage::parse(&record.response.body);
+ println!("TEST: {:?}", webpage.text());
+ println!();
}
panic!();
diff --git a/src/webpage.rs b/src/webpage.rs
index e75097e0..83679be1 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -36,7 +36,7 @@ impl Webpage {
self.dom
.select(selector)
.filter(|el| selector.matches(el))
- .map(|el| {
+ .filter_map(|el| {
if let Some(node) = (*el).first_child() {
if let Node::Text(text) = node.value() {
Some(text)
@@ -47,8 +47,6 @@ impl Webpage {
None
}
})
- .filter(|text| text.is_some())
- .map(|text| text.unwrap())
.map(|t| String::from(t.trim()))
.filter(|t| !t.is_empty())
.collect::<Vec<String>>()