Skip to content

web search done right

simple memory graph store

Mikkel Denker910b64c44762
diff --git a/src/indexer.rs b/src/indexer.rs
index 3fb63af4..0dfe6704 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -28,6 +28,7 @@ impl Indexer {
pub async fn run(self) -> Result<()> {
for warc_s3_path in self.warc_paths {
println!("{}", warc_s3_path);
+
let raw_object = Indexer::download_from_s3(
warc_s3_path,
self.config.s3.name.clone(),
@@ -42,9 +43,11 @@ impl Indexer {
if let Ok(record) = record {
let webpage = Webpage::parse(&record.response.body);
println!("TEST: {:?}", webpage.text());
- panic!();
+ println!();
}
}
+
+ panic!();
}
Ok(())
diff --git a/src/lib.rs b/src/lib.rs
index aec43650..9a938697 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,6 +5,7 @@ use thiserror::Error;
mod indexer;
mod warc;
+mod webgraph;
mod webpage;
pub use indexer::Indexer;
diff --git a/src/webpage.rs b/src/webpage.rs
index 3416ee77..e75097e0 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -55,8 +55,23 @@ impl Webpage {
}
pub(crate) fn text(&self) -> String {
- let selector =
- Selector::parse("body :not(script):not(style)").expect("Failed to parse selector");
+ let selector = Selector::parse(
+ "body a,
+ body div,
+ body span,
+ body p,
+ body h1,
+ body h2,
+ body h3,
+ body h4,
+ body li,
+ body ul,
+ body ol,
+ body nav,
+ body pre
+ ",
+ )
+ .expect("Failed to parse selector");
Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string())
.collect::<String>()
.trim()