web search done right
simple memory graph store
Mikkel Denker910b64c44762
diff --git a/src/indexer.rs b/src/indexer.rsindex 3fb63af4..0dfe6704 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -28,6 +28,7 @@ impl Indexer {pub async fn run(self) -> Result<()> {for warc_s3_path in self.warc_paths {println!("{}", warc_s3_path);+let raw_object = Indexer::download_from_s3(warc_s3_path,self.config.s3.name.clone(),@@ -42,9 +43,11 @@ impl Indexer {if let Ok(record) = record {let webpage = Webpage::parse(&record.response.body);println!("TEST: {:?}", webpage.text());- panic!();+ println!();}}++ panic!();}Ok(())diff --git a/src/lib.rs b/src/lib.rsindex aec43650..9a938697 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -5,6 +5,7 @@ use thiserror::Error;mod indexer;mod warc;+mod webgraph;mod webpage;pub use indexer::Indexer;diff --git a/src/webpage.rs b/src/webpage.rsindex 3416ee77..e75097e0 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -55,8 +55,23 @@ impl Webpage {}pub(crate) fn text(&self) -> String {- let selector =- Selector::parse("body :not(script):not(style)").expect("Failed to parse selector");+ let selector = Selector::parse(+ "body a,+ body div,+ body span,+ body p,+ body h1,+ body h2,+ body h3,+ body h4,+ body li,+ body ul,+ body ol,+ body nav,+ body pre+ ",+ )+ .expect("Failed to parse selector");Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string()).collect::<String>().trim()