web search done right
add a 'linksfrom' query operator to match against backlinks
Mikkel Denker58da0b356774
diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex a17bafa6..c1455e68 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -96,7 +96,7 @@ impl Job {has_page_centrality = true;}- if !webpage.backlink_labels.is_empty() {+ if !webpage.backlinks.is_empty() {has_backlinks = true;}trace!("inserting webpage: {:?}", webpage.html.url());diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 60a018bd..e01cece8 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -34,9 +34,11 @@ use crate::Result;use crate::index::Index;use crate::rake::RakeModel;use crate::ranking::SignalComputer;-use crate::webgraph::{self, EdgeLimit, Node, NodeID};+use crate::webgraph::{self, EdgeLimit, FullEdge, Node, NodeID};use crate::webpage::{safety_classifier, Html, Webpage};+const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);+pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,@@ -84,17 +86,16 @@ impl Webgraph {&self,ids: Vec<NodeID>,limit: EdgeLimit,- ) -> Vec<Vec<String>> {+ ) -> Vec<Vec<FullEdge>> {let edges = match self {Self::Remote(webgraph) => {- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))- .unwrap_or_default()+ crate::block_on(webgraph.batch_ingoing_edges_by_id(&ids, limit)).unwrap_or_default()}Self::Local(webgraph) => {let mut res = Vec::new();for id in ids {- res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));+ res.push(webgraph.ingoing_edges_by_id(&id, limit));}res@@ -106,10 +107,9 @@ impl Webgraph {.map(|edges| {edges.into_iter()- .map(|edge| edge.label)- .filter(|label| !label.is_empty())- .filter(|label| {- let label = label.to_lowercase();+ .filter(|e| !e.label.is_empty())+ .filter(|e| {+ let label = e.label.to_lowercase();let stopwords = ["click","click here",@@ -393,10 +393,10 @@ impl IndexingWorker {.map(|w| Node::from(w.html.url()).id()).collect::<Vec<_>>();- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(512));+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);- for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {- page.backlink_labels = backlink;+ for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {+ page.backlinks = backlinks;}}}@@ -455,7 +455,7 @@ impl IndexingWorker {// make sure we remember to set everythinglet mut webpage = Webpage {html: prepared.html,- backlink_labels: prepared.backlink_labels,+ backlinks: prepared.backlinks,page_centrality: prepared.page_centrality,page_centrality_rank: prepared.page_centrality_rank,host_centrality: prepared.host_centrality,diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 42ef0156..5d26c26e 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -53,6 +53,7 @@ sonic_service!([GetNode,IngoingEdges,+ IngoingEdgesById,OutgoingEdges,RawIngoingEdges,RawOutgoingEdges,@@ -89,6 +90,20 @@ impl Message<WebGraphService> for IngoingEdges {}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct IngoingEdgesById {+ pub node: NodeID,+ pub limit: EdgeLimit,+}++impl Message<WebGraphService> for IngoingEdgesById {+ type Response = Vec<FullEdge>;++ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.ingoing_edges_by_id(&self.node, self.limit)+ }+}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct OutgoingEdges {pub node: Node,diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 8317f103..e0df2cec 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -232,6 +232,7 @@ mod tests {ranking::{Ranker, SignalComputer},search_ctx::Ctx,searcher::SearchQuery,+ webgraph::{FullEdge, Node},webpage::{schema_org, Html, Webpage},OneOrMany,};@@ -502,7 +503,11 @@ mod tests {"https://www.b.com",).unwrap(),- backlink_labels: vec!["B site is great".to_string()],+ backlinks: vec![FullEdge {+ from: Node::from("https://www.a.com"),+ to: Node::from("https://www.b.com"),+ label: "B site is great".to_string(),+ }],host_centrality: 1.0,fetch_time_ms: 500,..Default::default()diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex ebd592ba..a7a30c0c 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -206,7 +206,13 @@ impl tantivy::query::Query for Query {#[cfg(test)]mod tests {- use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};+ use crate::{+ index::Index,+ rand_words,+ searcher::LocalSearcher,+ webgraph::{FullEdge, Node},+ webpage::Webpage,+ };use proptest::prelude::*;use super::*;@@ -635,6 +641,86 @@ mod tests {assert_eq!(result.webpages[0].url, "https://www.first.com/");}+ #[test]+ fn link_from_query() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut webpage = Webpage::test_parse(+ r#"+ <html>+ <head>+ <title>Test website</title>+ </head>+ <body>+ This is a test website+ </body>+ </html>+ "#,+ "https://www.first.com",+ )+ .unwrap();++ webpage.backlinks = vec![FullEdge {+ from: Node::from("https://www.second.com/example/abc"),+ to: Node::from("https://www.first.com"),+ label: String::new(),+ }];++ index.insert(&webpage).expect("failed to insert webpage");+ let mut webpage = Webpage::test_parse(+ r#"+ <html>+ <head>+ <title>Test test</title>+ </head>+ <body>+ This test page does not contain the forbidden word+ <a href="https://www.first.com">First</a>+ </body>+ </html>+ "#,+ "https://www.second.com/example/abc",+ )+ .unwrap();+ webpage.backlinks = vec![];++ index.insert(&webpage).expect("failed to insert webpage");+ index.commit().expect("failed to commit index");+ let searcher = LocalSearcher::from(index);++ let query = SearchQuery {+ query: "test linkfrom:second.com".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com/");++ let query = SearchQuery {+ query: "test linkfrom:www.second.com".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com/");++ let query = SearchQuery {+ query: "test linkfrom:second.com/example".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com/");++ let query = SearchQuery {+ query: "test linksfrom:second.com/example/abc".to_string(),+ ..Default::default()+ };+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com/");+ }+#[test]fn title_query() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsindex 5a10d5b1..5590fefe 100644--- a/crates/core/src/query/parser/mod.rs+++ b/crates/core/src/query/parser/mod.rs@@ -121,18 +121,24 @@ fn site_field(input: &str) -> nom::IResult<&str, Term> {fn links_to_field(input: &str) -> nom::IResult<&str, Term> {// parse 'linksto:' and then a simple term- let (input, _) = nom::bytes::complete::tag("linksto:")(input)?;+ let (input, _) = nom::branch::alt((+ nom::bytes::complete::tag("linksto:"),+ nom::bytes::complete::tag("linkto:"),+ ))(input)?;let (input, output) = simple_str(input)?;Ok((input, Term::LinkTo(output.to_string())))}-fn link_to_field(input: &str) -> nom::IResult<&str, Term> {- // parse 'linkto:' and then a simple term- let (input, _) = nom::bytes::complete::tag("linkto:")(input)?;+fn links_from_field(input: &str) -> nom::IResult<&str, Term> {+ // parse 'linksfrom:' and then a simple term+ let (input, _) = nom::branch::alt((+ nom::bytes::complete::tag("linksfrom:"),+ nom::bytes::complete::tag("linkfrom:"),+ ))(input)?;let (input, output) = simple_str(input)?;- Ok((input, Term::LinkTo(output.to_string())))+ Ok((input, Term::LinkFrom(output.to_string())))}fn title_field(input: &str) -> nom::IResult<&str, Term> {@@ -163,7 +169,7 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {nom::branch::alt((site_field,links_to_field,- link_to_field,+ links_from_field,title_field,body_field,url_field,diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rsindex 7eb26d31..01631829 100644--- a/crates/core/src/query/parser/term.rs+++ b/crates/core/src/query/parser/term.rs@@ -99,6 +99,7 @@ pub enum Term {SimpleOrPhrase(SimpleOrPhrase),Site(String),LinkTo(String),+ LinkFrom(String),Title(SimpleOrPhrase),Body(SimpleOrPhrase),Url(SimpleOrPhrase),@@ -113,6 +114,7 @@ impl std::fmt::Display for Term {Term::Not(term) => write!(f, "-{}", term),Term::Site(site) => write!(f, "site:{}", site),Term::LinkTo(site) => write!(f, "linkto:{}", site),+ Term::LinkFrom(site) => write!(f, "linkfrom:{}", site),Term::Title(title) => write!(f, "intitle:{}", title),Term::Body(body) => write!(f, "inbody:{}", body),Term::Url(url) => write!(f, "inurl:{}", url),@@ -134,6 +136,7 @@ impl Term {Term::SimpleOrPhrase(s) => Term::SimpleOrPhrase(s.truncate()),Term::Site(s) => Term::Site(s),Term::LinkTo(s) => Term::LinkTo(s),+ Term::LinkFrom(s) => Term::LinkFrom(s),Term::Title(s) => Term::Title(s.truncate()),Term::Body(s) => Term::Body(s.truncate()),Term::Url(s) => Term::Url(s.truncate()),diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsindex 2840e221..214945fb 100644--- a/crates/core/src/query/plan/node.rs+++ b/crates/core/src/query/plan/node.rs@@ -135,6 +135,10 @@ impl Node {text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),field: text_field::Links.into(),}),+ ParserTerm::LinkFrom(s) => Node::Term(Term {+ text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),+ field: text_field::Backlinks.into(),+ }),ParserTerm::Title(t) => Node::Term(Term {text: t,field: text_field::Title.into(),diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex a590511f..b14f6840 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -130,6 +130,7 @@ mod tests {index::Index,models::dual_encoder::DualEncoder,searcher::{LocalSearcher, SearchQuery},+ webgraph::{FullEdge, Node},webpage::{Html, Webpage},};@@ -249,7 +250,7 @@ mod tests {"https://www.b.com",).unwrap(),- backlink_labels: vec![],+ backlinks: vec![],fetch_time_ms: 500,page_centrality: 5.0,..Default::default()@@ -430,7 +431,11 @@ mod tests {"https://www.first.com",).unwrap(),- backlink_labels: vec!["test this is the best test site".to_string()],+ backlinks: vec![FullEdge {+ from: Node::from("https://www.backlink.com"),+ to: Node::from("https://www.first.com"),+ label: "test this is the best test site".to_string(),+ }],fetch_time_ms: 500,..Default::default()})diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex d8e4baaf..2775c39a 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -162,7 +162,7 @@ mod tests {"https://www.third.com",).unwrap(),- backlink_labels: vec![],+ backlinks: vec![],host_centrality: 1.0,node_id: Some(Node::from("https://www.third.com").into_host().id()),..Default::default()diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex 88c68787..7177050b 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -199,6 +199,7 @@ pub enum TextFieldEnum {Keywords,KeyPhrases,Links,+ Backlinks,}enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,@@ -237,6 +238,7 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,Keywords,KeyPhrases,Links,+ Backlinks,]);impl TextFieldEnum {@@ -1043,7 +1045,7 @@ impl TextField for BacklinkText {doc.add_text(self.tantivy_field(schema).unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),- webpage.backlink_labels.join("\n"),+ webpage.backlinks.iter().map(|e| e.label.clone()).join("\n"),);Ok(())@@ -1560,6 +1562,51 @@ impl TextField for Links {}}+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub struct Backlinks;+impl TextField for Backlinks {+ fn name(&self) -> &str {+ "backlinks"+ }++ fn has_pos(&self) -> bool {+ true+ }++ fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {+ FieldTokenizer::Url(UrlTokenizer)+ }++ fn add_webpage_tantivy(+ &self,+ webpage: &crate::webpage::Webpage,+ doc: &mut TantivyDocument,+ schema: &tantivy::schema::Schema,+ ) -> Result<()> {+ doc.add_text(+ self.tantivy_field(schema)+ .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),+ webpage+ .backlinks+ .iter()+ .map(|l| l.from.as_str().to_string())+ .join("\n"),+ );++ Ok(())+ }++ fn add_html_tantivy(+ &self,+ _: &Html,+ _: &mut FnCache,+ _: &mut TantivyDocument,+ _: &tantivy::schema::Schema,+ ) -> Result<()> {+ Ok(())+ }+}+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct Keywords;impl TextField for Keywords {diff --git a/crates/core/src/tokenizer/fields/url.rs b/crates/core/src/tokenizer/fields/url.rsindex 86070059..cdcf1df6 100644--- a/crates/core/src/tokenizer/fields/url.rs+++ b/crates/core/src/tokenizer/fields/url.rs@@ -18,7 +18,10 @@ use std::collections::VecDeque;use tantivy::tokenizer::BoxTokenStream;-use crate::tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve};+use crate::{+ tokenizer::{add_space_last::AddSpaceLast, split_preserve::StrSplitPreserve},+ webpage::url_ext::UrlExt,+};#[derive(Clone, Default)]struct ParsedUrl {@@ -40,7 +43,7 @@ impl UrlTokenizer {.or_else(|_| url::Url::parse(&format!("http://{}", text))).map(|url| {let domain = Some(- url.host_str()+ url.normalized_host().unwrap_or("").split_preserve(|c| matches!(c, '.')).filter(|s| !(*s).is_empty())@@ -217,12 +220,12 @@ mod tests {fn url() {assert_eq!(tokenize_url("https://www.example.com"),- vec!["www", ".", "example", ".", "com ", "/"]+ vec!["example", ".", "com ", "/"]);assert_eq!(tokenize_url("https://www.example.com/test"),- vec!["www", ".", "example", ".", "com ", "/", "test",]+ vec!["example", ".", "com ", "/", "test",]);assert_eq!(@@ -242,26 +245,17 @@ mod tests {fn multiple_urls() {assert_eq!(tokenize_url("https://www.example.com\nhttps://www.example.com"),- vec![- "www", ".", "example", ".", "com ", "/", "\n", "www", ".", "example", ".", "com ",- "/"- ]+ vec!["example", ".", "com ", "/", "\n", "example", ".", "com ", "/"]);assert_eq!(tokenize_url("https://www.example.com/test\nhttps://www.abcd.com"),- vec![- "www", ".", "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".",- "com ", "/"- ]+ vec!["example", ".", "com ", "/", "test", "\n", "abcd", ".", "com ", "/"]);assert_eq!(tokenize_url("https://example.com/test\nhttps://www.abcd.com/test"),- vec![- "example", ".", "com ", "/", "test", "\n", "www", ".", "abcd", ".", "com ", "/",- "test",- ]+ vec!["example", ".", "com ", "/", "test", "\n", "abcd", ".", "com ", "/", "test",]);}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 3da57b59..8898f519 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -207,13 +207,17 @@ impl Webgraph {}pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {+ self.ingoing_edges_by_id(&node.id(), limit)+ }++ pub fn ingoing_edges_by_id(&self, node_id: &NodeID, limit: EdgeLimit) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<SegmentEdge<String>>| {edges.sort_by_key(|e| e.from.node());edges.dedup_by_key(|e| e.from.node());};let mut edges = self.inner_edges(- |segment| segment.ingoing_edges_with_label(&node.id(), &limit),+ |segment| segment.ingoing_edges_with_label(node_id, &limit),dedup,);edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex ecc7b2eb..2ecee109 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -33,7 +33,7 @@ use crate::{},},entrypoint::webgraph_server::{- GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,+ GetNode, IngoingEdges, IngoingEdgesById, OutgoingEdges, PagesByHosts, RawIngoingEdges,RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,},Result,@@ -300,6 +300,37 @@ impl RemoteWebgraph {Ok(edges)}+ pub async fn batch_ingoing_edges_by_id(+ &self,+ ids: &[NodeID],+ limit: EdgeLimit,+ ) -> Result<Vec<Vec<FullEdge>>> {+ let reqs: Vec<_> = ids+ .iter()+ .map(|id| IngoingEdgesById { node: *id, limit })+ .collect();++ let res = self+ .conn()+ .await+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut edges = vec![vec![]; ids.len()];++ for (_, res) in res {+ debug_assert!(res.len() <= 1);++ for (_, res) in res {+ for (i, rep) in res.into_iter().enumerate() {+ edges[i].extend(rep);+ }+ }+ }++ Ok(edges)+ }+pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {let res = self.conn()diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex c05c6b40..b6c58712 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -16,7 +16,7 @@use crate::{schema::{numerical_field::NumericalField, text_field::TextField, Field},- webgraph::NodeID,+ webgraph::{FullEdge, NodeID},Result,};use candle_core::Tensor;@@ -41,7 +41,7 @@ pub use region::Region;#[derive(Debug)]pub struct Webpage {pub html: Html,- pub backlink_labels: Vec<String>,+ pub backlinks: Vec<FullEdge>,pub host_centrality: f64,pub host_centrality_rank: u64,pub page_centrality: f64,@@ -62,7 +62,7 @@ impl Default for Webpage {fn default() -> Self {Self {html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),- backlink_labels: Default::default(),+ backlinks: Default::default(),host_centrality: Default::default(),host_centrality_rank: u64::MAX,page_centrality: Default::default(),@@ -84,7 +84,7 @@ impl From<Html> for Webpage {fn from(html: Html) -> Self {Self {html,- backlink_labels: Default::default(),+ backlinks: Default::default(),host_centrality: Default::default(),host_centrality_rank: u64::MAX,page_centrality: Default::default(),