web search done right
sled graph store seems to work
Mikkel Denkere638d062b295
diff --git a/Cargo.toml b/Cargo.tomlindex 6fadab0d..c02f8296 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -9,6 +9,7 @@ edition = "2021"name = "cuely"[dependencies]+bincode = "1.3.3"encoding_rs = "0.8.31"flate2 = "1.0.23"itertools = "0.10.3"@@ -16,6 +17,8 @@ rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"scraper = "0.13.0"serde = "1.0.137"+sled = { version = "0.34.7", features = ["compression"] }+tantivy = "0.17.0"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }toml = "0.5.9"diff --git a/src/indexer.rs b/src/indexer.rsindex 0dfe6704..e294c15c 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -39,12 +39,10 @@ impl Indexer {println!("Downloaded {} bytes", raw_object.len());let warc = WarcFile::new(&raw_object[..]);- for record in warc {- if let Ok(record) = record {- let webpage = Webpage::parse(&record.response.body);- println!("TEST: {:?}", webpage.text());- println!();- }+ for record in warc.flatten() {+ let webpage = Webpage::parse(&record.response.body);+ println!("TEST: {:?}", webpage.text());+ println!();}panic!();diff --git a/src/webpage.rs b/src/webpage.rsindex e75097e0..83679be1 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -36,7 +36,7 @@ impl Webpage {self.dom.select(selector).filter(|el| selector.matches(el))- .map(|el| {+ .filter_map(|el| {if let Some(node) = (*el).first_child() {if let Node::Text(text) = node.value() {Some(text)@@ -47,8 +47,6 @@ impl Webpage {None}})- .filter(|text| text.is_some())- .map(|text| text.unwrap()).map(|t| String::from(t.trim())).filter(|t| !t.is_empty()).collect::<Vec<String>>()