Skip to content

web search done right

add a 'linksfrom' query operator to match against backlinks

Mikkel Denker58da0b356774
diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rs
index a17bafa6..c1455e68 100644
--- a/crates/core/src/entrypoint/indexer/job.rs
+++ b/crates/core/src/entrypoint/indexer/job.rs
@@ -96,7 +96,7 @@ impl Job {
has_page_centrality = true;
}
- if !webpage.backlink_labels.is_empty() {
+ if !webpage.backlinks.is_empty() {
has_backlinks = true;
}
trace!("inserting webpage: {:?}", webpage.html.url());
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 60a018bd..e01cece8 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -34,9 +34,11 @@ use crate::Result;
use crate::index::Index;
use crate::rake::RakeModel;
use crate::ranking::SignalComputer;
-use crate::webgraph::{self, EdgeLimit, Node, NodeID};
+use crate::webgraph::{self, EdgeLimit, FullEdge, Node, NodeID};
use crate::webpage::{safety_classifier, Html, Webpage};
+const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
+
pub struct Config {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
@@ -84,17 +86,16 @@ impl Webgraph {
&self,
ids: Vec<NodeID>,
limit: EdgeLimit,
- ) -> Vec<Vec<String>> {
+ ) -> Vec<Vec<FullEdge>> {
let edges = match self {
Self::Remote(webgraph) => {
- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))
- .unwrap_or_default()
+ crate::block_on(webgraph.batch_ingoing_edges_by_id(&ids, limit)).unwrap_or_default()
}
Self::Local(webgraph) => {
let mut res = Vec::new();
for id in ids {
- res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));
+ res.push(webgraph.ingoing_edges_by_id(&id, limit));
}
res
@@ -106,10 +107,9 @@ impl Webgraph {
.map(|edges| {
edges
.into_iter()
- .map(|edge| edge.label)
- .filter(|label| !label.is_empty())
- .filter(|label| {
- let label = label.to_lowercase();
+ .filter(|e| !e.label.is_empty())
+ .filter(|e| {
+ let label = e.label.to_lowercase();
let stopwords = [
"click",
"click here",
@@ -393,10 +393,10 @@ impl IndexingWorker {
.map(|w| Node::from(w.html.url()).id())
.collect::<Vec<_>>();
- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(512));
+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);
- for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {
- page.backlink_labels = backlink;
+ for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {
+ page.backlinks = backlinks;
}
}
}
@@ -455,7 +455,7 @@ impl IndexingWorker {
// make sure we remember to set everything
let mut webpage = Webpage {
html: prepared.html,
- backlink_labels: prepared.backlink_labels,
+ backlinks: prepared.backlinks,
page_centrality: prepared.page_centrality,
page_centrality_rank: prepared.page_centrality_rank,
host_centrality: prepared.host_centrality,
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 42ef0156..5d26c26e 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -53,6 +53,7 @@ sonic_service!(
[
GetNode,
IngoingEdges,
+ IngoingEdgesById,
OutgoingEdges,
RawIngoingEdges,
RawOutgoingEdges,
@@ -89,6 +90,20 @@ impl Message<WebGraphService> for IngoingEdges {
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct IngoingEdgesById {
+ pub node: NodeID,
+ pub limit: EdgeLimit,
+}
+
+impl Message<WebGraphService> for IngoingEdgesById {
+ type Response = Vec<FullEdge>;
+
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ server.graph.ingoing_edges_by_id(&self.node, self.limit)
+ }
+}
+
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct OutgoingEdges {
pub node: Node,
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 8317f103..e0df2cec 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -232,6 +232,7 @@ mod tests {
ranking::{Ranker, SignalComputer},
search_ctx::Ctx,
searcher::SearchQuery,
+ webgraph::{FullEdge, Node},
webpage::{schema_org, Html, Webpage},
OneOrMany,
};
@@ -502,7 +503,11 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlink_labels: vec!["B site is great".to_string()],
+ backlinks: vec![FullEdge {
+ from: Node::from("https://www.a.com"),
+ to: Node::from("https://www.b.com"),
+ label: "B site is great".to_string(),
+ }],
host_centrality: 1.0,
fetch_time_ms: 500,
..Default::default()
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index ebd592ba..a7a30c0c 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -206,7 +206,13 @@ impl tantivy::query::Query for Query {
#[cfg(test)]
mod tests {
- use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};
+ use crate::{
+ index::Index,
+ rand_words,
+ searcher::LocalSearcher,
+ webgraph::{FullEdge, Node},
+ webpage::Webpage,
+ };
use proptest::prelude::*;
use super::*;
@@ -635,6 +641,86 @@ mod tests {
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
+ #[test]
+ fn link_from_query() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut webpage = Webpage::test_parse(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website
+ </body>
+ </html>
+ "#,
+ "https://www.first.com",
+ )
+ .unwrap();
+
+ webpage.backlinks = vec![FullEdge {
+ from: Node::from("https://www.second.com/example/abc"),
+ to: Node::from("https://www.first.com"),
+ label: String::new(),
+ }];
+
+ index.insert(&webpage).expect("failed to insert webpage");
+ let mut webpage = Webpage::test_parse(
+ r#"
+ <html>
+ <head>
+ <title>Test test</title>
+ </head>
+ <body>
+ This test page does not contain the forbidden word
+ <a href="https://www.first.com">First</a>
+ </body>
+ </html>
+ "#,
+ "https://www.second.com/example/abc",
+ )
+ .unwrap();
+ webpage.backlinks = vec![];
+
+ index.insert(&webpage).expect("failed to insert webpage");
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "test linkfrom:second.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+
+ let query = SearchQuery {
+ query: "test linkfrom:www.second.com".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+
+ let query = SearchQuery {
+ query: "test linkfrom:second.com/example".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+
+ let query = SearchQuery {
+ query: "test linksfrom:second.com/example/abc".to_string(),
+ ..Default::default()
+ };
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ }
+
#[test]
fn title_query() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index 5a10d5b1..5590fefe 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -121,18 +121,24 @@ fn site_field(input: &str) -> nom::IResult<&str, Term> {
fn links_to_field(input: &str) -> nom::IResult<&str, Term> {
// parse 'linksto:' and then a simple term
- let (input, _) = nom::bytes::complete::tag("linksto:")(input)?;
+ let (input, _) = nom::branch::alt((
+ nom::bytes::complete::tag("linksto:"),
+ nom::bytes::complete::tag("linkto:"),
+ ))(input)?;
let (input, output) = simple_str(input)?;
Ok((input, Term::LinkTo(output.to_string())))
}
-fn link_to_field(input: &str) -> nom::IResult<&str, Term> {
- // parse 'linkto:' and then a simple term
- let (input, _) = nom::bytes::complete::tag("linkto:")(input)?;
+fn links_from_field(input: &str) -> nom::IResult<&str, Term> {
+ // parse 'linksfrom:' and then a simple term
+ let (input, _) = nom::branch::alt((
+ nom::bytes::complete::tag("linksfrom:"),
+ nom::bytes::complete::tag("linkfrom:"),
+ ))(input)?;
let (input, output) = simple_str(input)?;
- Ok((input, Term::LinkTo(output.to_string())))
+ Ok((input, Term::LinkFrom(output.to_string())))
}
fn title_field(input: &str) -> nom::IResult<&str, Term> {
@@ -163,7 +169,7 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {
nom::branch::alt((
site_field,
links_to_field,
- link_to_field,
+ links_from_field,
title_field,
body_field,
url_field,
diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rs
index 7eb26d31..01631829 100644
--- a/crates/core/src/query/parser/term.rs
+++ b/crates/core/src/query/parser/term.rs
@@ -99,6 +99,7 @@ pub enum Term {
SimpleOrPhrase(SimpleOrPhrase),
Site(String),
LinkTo(String),
+ LinkFrom(String),
Title(SimpleOrPhrase),
Body(SimpleOrPhrase),
Url(SimpleOrPhrase),
@@ -113,6 +114,7 @@ impl std::fmt::Display for Term {
Term::Not(term) => write!(f, "-{}", term),
Term::Site(site) => write!(f, "site:{}", site),
Term::LinkTo(site) => write!(f, "linkto:{}", site),
+ Term::LinkFrom(site) => write!(f, "linkfrom:{}", site),
Term::Title(title) => write!(f, "intitle:{}", title),
Term::Body(body) => write!(f, "inbody:{}", body),
Term::Url(url) => write!(f, "inurl:{}", url),
@@ -134,6 +136,7 @@ impl Term {
Term::SimpleOrPhrase(s) => Term::SimpleOrPhrase(s.truncate()),
Term::Site(s) => Term::Site(s),
Term::LinkTo(s) => Term::LinkTo(s),
+ Term::LinkFrom(s) => Term::LinkFrom(s),
Term::Title(s) => Term::Title(s.truncate()),
Term::Body(s) => Term::Body(s.truncate()),
Term::Url(s) => Term::Url(s.truncate()),
diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rs
index 2840e221..214945fb 100644
--- a/crates/core/src/query/plan/node.rs
+++ b/crates/core/src/query/plan/node.rs
@@ -135,6 +135,10 @@ impl Node {
text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),
field: text_field::Links.into(),
}),
+ ParserTerm::LinkFrom(s) => Node::Term(Term {
+ text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),
+ field: text_field::Backlinks.into(),
+ }),
ParserTerm::Title(t) => Node::Term(Term {
text: t,
field: text_field::Title.into(),
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index a590511f..b14f6840 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -130,6 +130,7 @@ mod tests {
index::Index,
models::dual_encoder::DualEncoder,
searcher::{LocalSearcher, SearchQuery},
+ webgraph::{FullEdge, Node},
webpage::{Html, Webpage},
};
@@ -249,7 +250,7 @@ mod tests {
"https://www.b.com",
)
.unwrap(),
- backlink_labels: vec![],
+ backlinks: vec![],
fetch_time_ms: 500,
page_centrality: 5.0,
..Default::default()
@@ -430,7 +431,11 @@ mod tests {
"https://www.first.com",
)
.unwrap(),
- backlink_labels: vec!["test this is the best test site".to_string()],
+ backlinks: vec![FullEdge {
+ from: Node::from("https://www.backlink.com"),
+ to: Node::from("https://www.first.com"),
+ label: "test this is the best test site".to_string(),
+ }],
fetch_time_ms: 500,
..Default::default()
})
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index d8e4baaf..2775c39a 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -162,7 +162,7 @@ mod tests {
"https://www.third.com",
)
.unwrap(),
- backlink_labels: vec![],
+ backlinks: vec![],
host_centrality: 1.0,
node_id: Some(Node::from("https://www.third.com").into_host().id()),
..Default::default()
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index 88c68787..7177050b 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -199,6 +199,7 @@ pub enum TextFieldEnum {
Keywords,
KeyPhrases,
Links,
+ Backlinks,
}
enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
@@ -237,6 +238,7 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
Keywords,
KeyPhrases,
Links,
+ Backlinks,
]);
impl TextFieldEnum {
@@ -1043,7 +1045,7 @@ impl TextField for BacklinkText {
doc.add_text(
self.tantivy_field(schema)
.unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
- webpage.backlink_labels.join("\n"),
+ webpage.backlinks.iter().map(|e| e.label.clone()).join("\n"),
);
Ok(())
@@ -1560,6 +1562,51 @@ impl TextField for Links {
}
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Backlinks;
+impl TextField for Backlinks {
+ fn name(&self) -> &str {
+ "backlinks"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
+ FieldTokenizer::Url(UrlTokenizer)
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema)
+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
+ webpage
+ .backlinks
+ .iter()
+ .map(|l| l.from.as_str().to_string())
+ .join("\n"),
+ );
+
+ Ok(())
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _: &Html,
+ _: &mut FnCache,
+ _: &mut TantivyDocument,
+ _: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+}
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Keywords;
impl TextField for Keywords {
diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rs
index 86070059..cdcf1df6 100644
--- a/crates/core/src/tokenizer/fields/url.rs
+++ b/crates/core/src/tokenizer/fields/url.rs
@@ -18,7 +18,10 @@ use std::collections::VecDeque;
use tantivy::tokenizer::BoxTokenStream;
-use crate::tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};
+use crate::{
+ tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve},
+ webpage::url_ext::UrlExt,
+};
#[derive(Clone, Default)]
struct ParsedUrl {
@@ -40,7 +43,7 @@ impl UrlTokenizer {
.or_else(|_| url::Url::parse(&format!("http://{}", text)))
.map(|url| {
let domain = Some(
- url.host_str()
+ url.normalized_host()
.unwrap_or("")
.split_preserve(|c| matches!(c, '.'))
.filter(|s| !(*s).is_empty())
@@ -217,12 +220,12 @@ mod tests {
fn url() {
assert_eq!(
tokenize_url("https://www.example.com"),
- vec!["www", ".", "example", ".", "com ", "/"]
+ vec!["example", ".", "com ", "/"]
);
assert_eq!(
tokenize_url("https://www.example.com/test"),
- vec!["www", ".", "example", ".", "com ", "/", "test",]
+ vec!["example", ".", "com ", "/", "test",]
);
assert_eq!(
@@ -242,26 +245,17 @@ mod tests {
fn multiple_urls() {
assert_eq!(
tokenize_url("https://www.example.com\nhttps://www.example.com"),
- vec![
- "www", ".", "example", ".", "com ", "/", "\n", "www", ".", "example", ".", "com ",
- "/"
- ]
+ vec!["example", ".", "com ", "/", "\n", "example", ".", "com ", "/"]
);
assert_eq!(
tokenize_url("https://www.example.com/test\nhttps://www.abcd.com"),
- vec![
- "www", ".", "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".",
- "com ", "/"
- ]
+ vec!["example", ".", "com ", "/", "test", "\n", "abcd", ".", "com ", "/"]
);
assert_eq!(
tokenize_url("https://example.com/test\nhttps://www.abcd.com/test"),
- vec![
- "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".", "com ", "/",
- "test",
- ]
+ vec!["example", ".", "com ", "/", "test", "\n", "abcd", ".", "com ", "/", "test",]
);
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 3da57b59..8898f519 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -207,13 +207,17 @@ impl Webgraph {
}
pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
+ self.ingoing_edges_by_id(&node.id(), limit)
+ }
+
+ pub fn ingoing_edges_by_id(&self, node_id: &NodeID, limit: EdgeLimit) -> Vec<FullEdge> {
let dedup = |edges: &mut Vec<SegmentEdge<String>>| {
edges.sort_by_key(|e| e.from.node());
edges.dedup_by_key(|e| e.from.node());
};
let mut edges = self.inner_edges(
- |segment| segment.ingoing_edges_with_label(&node.id(), &limit),
+ |segment| segment.ingoing_edges_with_label(node_id, &limit),
dedup,
);
edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index ecc7b2eb..2ecee109 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -33,7 +33,7 @@ use crate::{
},
},
entrypoint::webgraph_server::{
- GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,
+ GetNode, IngoingEdges, IngoingEdgesById, OutgoingEdges, PagesByHosts, RawIngoingEdges,
RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,
},
Result,
@@ -300,6 +300,37 @@ impl RemoteWebgraph {
Ok(edges)
}
+ pub async fn batch_ingoing_edges_by_id(
+ &self,
+ ids: &[NodeID],
+ limit: EdgeLimit,
+ ) -> Result<Vec<Vec<FullEdge>>> {
+ let reqs: Vec<_> = ids
+ .iter()
+ .map(|id| IngoingEdgesById { node: *id, limit })
+ .collect();
+
+ let res = self
+ .conn()
+ .await
+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
+ .await?;
+
+ let mut edges = vec![vec![]; ids.len()];
+
+ for (_, res) in res {
+ debug_assert!(res.len() <= 1);
+
+ for (_, res) in res {
+ for (i, rep) in res.into_iter().enumerate() {
+ edges[i].extend(rep);
+ }
+ }
+ }
+
+ Ok(edges)
+ }
+
pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
let res = self
.conn()
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index c05c6b40..b6c58712 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -16,7 +16,7 @@
use crate::{
schema::{numerical_field::NumericalField, text_field::TextField, Field},
- webgraph::NodeID,
+ webgraph::{FullEdge, NodeID},
Result,
};
use candle_core::Tensor;
@@ -41,7 +41,7 @@ pub use region::Region;
#[derive(Debug)]
pub struct Webpage {
pub html: Html,
- pub backlink_labels: Vec<String>,
+ pub backlinks: Vec<FullEdge>,
pub host_centrality: f64,
pub host_centrality_rank: u64,
pub page_centrality: f64,
@@ -62,7 +62,7 @@ impl Default for Webpage {
fn default() -> Self {
Self {
html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),
- backlink_labels: Default::default(),
+ backlinks: Default::default(),
host_centrality: Default::default(),
host_centrality_rank: u64::MAX,
page_centrality: Default::default(),
@@ -84,7 +84,7 @@ impl From<Html> for Webpage {
fn from(html: Html) -> Self {
Self {
html,
- backlink_labels: Default::default(),
+ backlinks: Default::default(),
host_centrality: Default::default(),
host_centrality_rank: u64::MAX,
page_centrality: Default::default(),