Skip to content

web search done right

hopefully more robust way to detect disambiguation pages. this is difficult to test as there is no disambiguation pages in the dev .zim file. let's just try on prod

Mikkel Denker6bb731183805
diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rs
index 41c795c3..413311d6 100644
--- a/crates/core/src/entrypoint/entity.rs
+++ b/crates/core/src/entrypoint/entity.rs
@@ -99,7 +99,20 @@ fn article_to_entity(article: Article) -> Entity {
.map(|n| node_into_span(n.as_node()))
.unwrap_or_default();
- let is_disambiguation = root.select_first("#disambigbox").is_some();
+ let mut is_disambiguation = false;
+
+ if let Some(catlinks) = root.select_first("#catlinks") {
+ for link in catlinks.as_node().select("a").unwrap() {
+ if link
+ .text_contents()
+ .to_ascii_lowercase()
+ .contains("disambiguation")
+ {
+ is_disambiguation = true;
+ break;
+ }
+ }
+ }
Entity {
is_disambiguation,
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index e09c2ce5..8ed8f36d 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -442,7 +442,13 @@ impl InvertedIndex {
}) {
if query.simple_terms().is_empty() {
let snippet = if let Some(description) = page.description.as_deref() {
- description.split_whitespace().take(50).join(" ")
+ let snip = description.split_whitespace().take(50).join(" ");
+
+ if snip.split_ascii_whitespace().count() < 10 {
+ page.body.split_whitespace().take(50).join(" ")
+ } else {
+ snip
+ }
} else {
page.body.split_whitespace().take(50).join(" ")
};