Skip to content

web search done right

Rake keyword extraction (#171)

Mikkel Denkera0e371b2964c
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index 38ed2a38..3a13552f 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -41,6 +41,7 @@ use crate::config::SnippetConfig;
use crate::fastfield_reader::FastFieldReader;
use crate::query::shortcircuit::ShortCircuitQuery;
use crate::query::Query;
+use crate::rake::RakeModel;
use crate::ranking::initial::Score;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::SignalAggregator;
@@ -164,6 +165,7 @@ pub struct InvertedIndex {
schema: Arc<Schema>,
snippet_config: SnippetConfig,
fastfield_reader: FastFieldReader,
+ rake: RakeModel,
}
impl InvertedIndex {
@@ -234,6 +236,7 @@ impl InvertedIndex {
tantivy_index,
snippet_config: SnippetConfig::default(),
fastfield_reader,
+ rake: RakeModel::default(),
})
}
@@ -288,7 +291,7 @@ impl InvertedIndex {
self.writer
.as_ref()
.expect("writer has not been prepared")
- .add_document(webpage.into_tantivy(&self.schema)?)?;
+ .add_document(webpage.into_tantivy(&self.schema, &self.rake)?)?;
Ok(())
}
@@ -689,6 +692,7 @@ pub struct RetrievedWebpage {
pub likely_has_ads: bool,
pub likely_has_paywall: bool,
pub recipe_first_ingredient_tag_id: Option<String>,
+ pub keywords: Vec<String>,
}
impl RetrievedWebpage {
pub fn description(&self) -> Option<&String> {
@@ -800,6 +804,16 @@ impl From<TantivyDocument> for RetrievedWebpage {
webpage.recipe_first_ingredient_tag_id = Some(tag_id);
}
}
+ Some(Field::Text(TextField::Keywords)) => {
+ let keywords = value
+ .value()
+ .as_value()
+ .as_str()
+ .expect("Keywords field should be stored as text")
+ .to_string();
+
+ webpage.keywords = keywords.split('\n').map(|s| s.to_string()).collect();
+ }
_ => {}
}
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index fb47a263..e3fe7a44 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -61,6 +61,7 @@ mod models;
pub mod naive_bayes;
pub mod prehashed;
mod query;
+mod rake;
pub mod ranking;
mod schema;
mod search_ctx;
@@ -69,6 +70,7 @@ pub mod searcher;
mod simhash;
pub mod similar_hosts;
mod snippet;
+mod stopwords;
pub mod summarizer;
mod tokenizer;
#[allow(unused)]
diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rs
new file mode 100644
index 00000000..616fbb94
--- /dev/null
+++ b/crates/core/src/rake.rs
@@ -0,0 +1,273 @@
+//! Implementation of the RAKE keyword extraction algorithm.
+//!
+//! https://doi.org/10.1002/9780470689646.ch1
+//!
+//! We have modified the algorithm a bit so that it generates
+//! the keywords based on a summary of the text as extracted
+//! by the sentences with most frequent words.
+
+use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
+use whatlang::Lang;
+
+use crate::stopwords;
+
+fn is_sent_split(c: char) -> bool {
+ matches!(
+ c,
+ '.' | '!' | '?' | '\n' | '\r' | '\t' | '\u{2026}' | '\u{2025}' | '\u{2024}'
+ )
+}
+
+fn sentences(text: &str) -> impl Iterator<Item = Sentence<'_>> {
+ text.split_terminator(is_sent_split).map(Sentence)
+}
+
+fn phrases<'a>(
+ sentence: &'a Sentence<'a>,
+ stopwords: &'a HashSet<String>,
+ max_words: usize,
+) -> Vec<Phrase> {
+ sentence
+ .0
+ .split_whitespace()
+ .map(|word| word.replace(|c: char| matches!(c, ',' | '.'), ""))
+ .group_by(|word| !stopwords.contains(word))
+ .into_iter()
+ .filter_map(move |(is_keyword, words)| {
+ if is_keyword {
+ let mut phrase = Vec::new();
+ let mut num_words = 0;
+
+ for word in words {
+ phrase.push(word);
+ num_words += 1;
+ }
+
+ if num_words > 1 && num_words <= max_words {
+ return Some(phrase);
+ }
+ }
+
+ None
+ })
+ .map(Phrase)
+ .collect()
+}
+
+fn smmry<'a>(
+ sents: Vec<Sentence<'a>>,
+ stopwords: &HashSet<String>,
+ top_sentences: usize,
+) -> Vec<Sentence<'a>> {
+ let mut word_frequency = HashMap::new();
+
+ for sentence in &sents {
+ for word in sentence.0.split_whitespace() {
+ if !stopwords.contains(word) {
+ *word_frequency.entry(word.to_string()).or_insert(0) += 1;
+ }
+ }
+ }
+
+ let scored_sentences = sents
+ .into_iter()
+ .filter_map(|sentence| {
+ let words = sentence.0.split_whitespace().collect::<Vec<_>>();
+
+ if words.is_empty() {
+ return None;
+ }
+
+ let score = words
+ .iter()
+ .map(|word| word_frequency.get(*word).unwrap_or(&0))
+ .sum::<u64>();
+ Some(ScoredSentence { sentence, score })
+ })
+ .collect::<Vec<_>>();
+
+ scored_sentences
+ .into_iter()
+ .sorted_by(|a, b| b.score.cmp(&a.score))
+ .take(top_sentences)
+ .map(|s| s.sentence)
+ .collect::<Vec<_>>()
+}
+
+/// A sentence is a sequence of words from the input text
+/// that has been split by punctuation.
+#[derive(Debug)]
+struct Sentence<'a>(&'a str);
+
+#[derive(Debug)]
+struct ScoredSentence<'a> {
+ sentence: Sentence<'a>,
+ score: u64,
+}
+
+type Word = String;
+/// A phrase is a sub-sequence of a sentence that has been
+/// split by stopwords.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct Phrase(Vec<Word>);
+
+#[derive(Debug, Clone)]
+pub struct Keyword {
+ pub text: String,
+ pub score: f64,
+}
+
+pub struct RakeParams {
+ pub summary_sentences: usize,
+ pub max_words: usize,
+}
+
+impl Default for RakeParams {
+ fn default() -> Self {
+ Self {
+ summary_sentences: 16,
+ max_words: 5,
+ }
+ }
+}
+
+pub struct RakeModel {
+ stopwords: HashMap<Lang, HashSet<String>>,
+ params: RakeParams,
+}
+
+impl Default for RakeModel {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl RakeModel {
+ pub fn new() -> Self {
+ Self::with_params(RakeParams::default())
+ }
+
+ pub fn with_params(params: RakeParams) -> Self {
+ let stopwords = stopwords::all().clone();
+ Self { stopwords, params }
+ }
+
+ pub fn keywords(&self, text: &str, lang: Lang) -> Vec<Keyword> {
+ let text = text.to_lowercase();
+ let stopwords = self
+ .stopwords
+ .get(&lang)
+ .unwrap_or_else(|| self.stopwords.get(&Lang::Eng).expect("English stopwords"));
+
+ let sentences = sentences(&text).collect::<Vec<_>>();
+ let top_sentences = smmry(sentences, stopwords, self.params.summary_sentences);
+
+ let phrases = top_sentences
+ .into_iter()
+ .flat_map(|sentence| phrases(&sentence, stopwords, self.params.max_words))
+ .collect::<Vec<_>>();
+
+ let num_words = phrases.iter().map(|p| p.0.len()).sum::<usize>();
+
+ let mut word_frequency = HashMap::with_capacity(num_words);
+ let mut word_degree = HashMap::with_capacity(num_words);
+
+ for phrase in &phrases {
+ let words = &phrase.0;
+ let degree = words.len() as f64 - 1.0;
+ for word in words {
+ *word_frequency.entry(word.to_string()).or_insert(0.0) += 1.0;
+ *word_degree.entry(word.to_string()).or_insert(0.0) += degree;
+ }
+ }
+
+ let mut keywords = HashMap::with_capacity(num_words);
+ for phrase in phrases {
+ let words = &phrase.0;
+ let mut score = 0.0;
+ for word in words {
+ score += word_degree[word] / word_frequency[word];
+ }
+ score /= words.len() as f64;
+ keywords.insert(phrase, score);
+ }
+
+ keywords
+ .into_iter()
+ .sorted_by(|(_, score1), (_, score2)| score2.partial_cmp(score1).unwrap())
+ .map(|(phrase, score)| Keyword {
+ text: phrase.0.join(" "),
+ score,
+ })
+ .take(word_degree.len() / 3)
+ .filter(|k| k.text.len() > 1)
+ .filter(|k| k.score > 0.0)
+ .collect()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ const TURING_TEXT: &str = r##"
+ Alan Mathison Turing OBE FRS (/ˈtjʊərɪŋ/; 23 June 1912 – 7 June 1954) was an English mathematician, computer scientist, logician, cryptanalyst, philosopher and theoretical biologist.[5] Turing was highly influential in the development of theoretical computer science, providing a formalisation of the concepts of algorithm and computation with the Turing machine, which can be considered a model of a general-purpose computer.[6][7][8] He is widely considered to be the father of theoretical computer science and artificial intelligence.[9]
+
+Born in Maida Vale, London, Turing was raised in southern England. He graduated from King's College, Cambridge, with a degree in mathematics. Whilst he was a fellow at Cambridge, he published a proof demonstrating that some purely mathematical yes–no questions can never be answered by computation. He defined a Turing machine and proved that the halting problem for Turing machines is undecidable. In 1938, he earned his PhD from the Department of Mathematics at Princeton University.
+
+During the Second World War, Turing worked for the Government Code and Cypher School at Bletchley Park, Britain's codebreaking centre that produced Ultra intelligence. For a time he led Hut 8, the section that was responsible for German naval cryptanalysis. Here, he devised a number of techniques for speeding the breaking of German ciphers, including improvements to the pre-war Polish bomba method, an electromechanical machine that could find settings for the Enigma machine. Turing played a crucial role in cracking intercepted coded messages that enabled the Allies to defeat the Axis powers in many crucial engagements, including the Battle of the Atlantic.[10][11]
+
+After the war, Turing worked at the National Physical Laboratory, where he designed the Automatic Computing Engine, one of the first designs for a stored-program computer. In 1948, Turing joined Max Newman's Computing Machine Laboratory at the Victoria University of Manchester, where he helped develop the Manchester computers[12] and became interested in mathematical biology. He wrote a paper on the chemical basis of morphogenesis[13][1] and predicted oscillating chemical reactions such as the Belousov–Zhabotinsky reaction, first observed in the 1960s. Despite these accomplishments, Turing was never fully recognised in Britain during his lifetime because much of his work was covered by the Official Secrets Act.[14]
+
+Turing was prosecuted in 1952 for homosexual acts. He accepted hormone treatment with DES, a procedure commonly referred to as chemical castration, as an alternative to prison. Turing died on 7 June 1954, 16 days before his 42nd birthday, from cyanide poisoning. An inquest determined his death as a suicide, but it has been noted that the known evidence is also consistent with accidental poisoning. Following a public campaign in 2009, British prime minister Gordon Brown made an official public apology on behalf of the government for "the appalling way [Turing] was treated". Queen Elizabeth II granted a posthumous pardon in 2013. The term "Alan Turing law" is now used informally to refer to a 2017 law in the United Kingdom that retroactively pardoned men cautioned or convicted under historical legislation that outlawed homosexual acts.[15]
+
+Turing has an extensive legacy with statues of him and many things named after him, including an annual award for computer science innovations. He appears on the current Bank of England £50 note, which was released on 23 June 2021 to coincide with his birthday. A 2019 BBC series, as voted by the audience, named him the greatest person of the 20th century.
+Early life and education
+Family
+English Heritage plaque in Maida Vale, London marking Turing's birthplace in 1912
+
+Turing was born in Maida Vale, London, while his father, Julius Mathison Turing was on leave from his position with the Indian Civil Service (ICS) of the British Raj government at Chatrapur, then in the Madras Presidency and presently in Odisha state, in India.[16][17] Turing's father was the son of a clergyman, the Rev. John Robert Turing, from a Scottish family of merchants that had been based in the Netherlands and included a baronet. Turing's mother, Julius's wife, was Ethel Sara Turing (née Stoney), daughter of Edward Waller Stoney, chief engineer of the Madras Railways. The Stoneys were a Protestant Anglo-Irish gentry family from both County Tipperary and County Longford, while Ethel herself had spent much of her childhood in County Clare.[18] Julius and Ethel married on 1 October 1907 at Bartholomew's church on Clyde Road, in Dublin.[19]
+
+Julius's work with the ICS brought the family to British India, where his grandfather had been a general in the Bengal Army. However, both Julius and Ethel wanted their children to be brought up in Britain, so they moved to Maida Vale,[20] London, where Alan Turing was born on 23 June 1912, as recorded by a blue plaque on the outside of the house of his birth,[21][22] later the Colonnade Hotel.[16][23] Turing had an elder brother, John Ferrier Turing, father of Sir John Dermot Turing, 12th Baronet of the Turing baronets.[24]
+
+Turing's father's civil service commission was still active during Turing's childhood years, and his parents travelled between Hastings in the United Kingdom[25] and India, leaving their two sons to stay with a retired Army couple. At Hastings, Turing stayed at Baston Lodge, Upper Maze Hill, St Leonards-on-Sea, now marked with a blue plaque.[26] The plaque was unveiled on 23 June 2012, the centenary of Turing's birth.[27]
+
+Very early in life, Turing showed signs of the genius that he was later to display prominently.[28] His parents purchased a house in Guildford in 1927, and Turing lived there during school holidays. The location is also marked with a blue plaque.[29]
+School
+
+Turing's parents enrolled him at St Michael's, a primary school at 20 Charles Road, St Leonards-on-Sea, from the age of six to nine. The headmistress recognised his talent, noting that she has "...had clever boys and hardworking boys, but Alan is a genius".[30]
+
+Between January 1922 and 1926, Turing was educated at Hazelhurst Preparatory School, an independent school in the village of Frant in Sussex (now East Sussex).[31] In 1926, at the age of 13, he went on to Sherborne School,[32] an independent boarding school in the market town of Sherborne in Dorset, where he boarded at Westcott House. The first day of term coincided with the 1926 General Strike, in Britain, but Turing was so determined to attend that he rode his bicycle unaccompanied 60 miles (97 km) from Southampton to Sherborne, stopping overnight at an inn.[33]
+
+Turing's natural inclination towards mathematics and science did not earn him respect from some of the teachers at Sherborne, whose definition of education placed more emphasis on the classics. His headmaster wrote to his parents: "I hope he will not fall between two stools. If he is to stay at public school, he must aim at becoming educated. If he is to be solely a Scientific Specialist, he is wasting his time at a public school".[34] Despite this, Turing continued to show remarkable ability in the studies he loved, solving advanced problems in 1927 without having studied even elementary calculus. In 1928, aged 16, Turing encountered Albert Einstein's work; not only did he grasp it, but it is possible that he managed to deduce Einstein's questioning of Newton's laws of motion from a text in which this was never made explicit.[35]
+Christopher Morcom
+
+At Sherborne, Turing formed a significant friendship with fellow pupil Christopher Collan Morcom (13 July 1911 – 13 February 1930),[36] who has been described as Turing's first love.[37][38][39] Their relationship provided inspiration in Turing's future endeavours, but it was cut short by Morcom's death, in February 1930, from complications of bovine tuberculosis, contracted after drinking infected cow's milk some years previously.[40][41][42]
+
+The event caused Turing great sorrow. He coped with his grief by working that much harder on the topics of science and mathematics that he had shared with Morcom. In a letter to Morcom's mother, Frances Isobel Morcom (née Swan), Turing wrote:
+
+ I am sure I could not have found anywhere another companion so brilliant and yet so charming and unconceited. I regarded my interest in my work, and in such things as astronomy (to which he introduced me) as something to be shared with him and I think he felt a little the same about me ... I know I must put as much energy if not as much interest into my work as if he were alive, because that is what he would like me to do.[43]
+
+Turing's relationship with Morcom's mother continued long after Morcom's death, with her sending gifts to Turing, and him sending letters, typically on Morcom's birthday.[44] A day before the third anniversary of Morcom's death (13 February 1933), he wrote to Mrs. Morcom:
+
+ I expect you will be thinking of Chris when this reaches you. I shall too, and this letter is just to tell you that I shall be thinking of Chris and of you tomorrow. I am sure that he is as happy now as he was when he was here. Your affectionate Alan.[45]
+
+Some have speculated that Morcom's death was the cause of Turing's atheism and materialism.[46] Apparently, at this point in his life he still believed in such concepts as a spirit, independent of the body and surviving death. In a later letter, also written to Morcom's mother, Turing wrote:
+
+ Personally, I believe that spirit is really eternally connected with matter but certainly not by the same kind of body ... as regards the actual connection between spirit and body I consider that the body can hold on to a 'spirit', whilst the body is alive and awake the two are firmly connected. When the body is asleep I cannot guess what happens but when the body dies, the 'mechanism' of the body, holding the spirit is gone and the spirit finds a new body sooner or later, perhaps immediately.[47][48]
+
+University and work on computability
+
+After graduating from Sherborne, Turing applied for several Cambridge colleges scholarships, including Trinity and King's, eventually earning a £80 per annum scholarship (equivalent to about £4,300 as of 2023) to study at the latter.[49][50] There, Turing studied the undergraduate course in Schedule B (that is, a three-year Parts I and II, of the Mathematical Tripos, with extra courses at the end of the third year, as Part III only emerged as a separate degree in 1934) from February 1931 to November 1934 at King's College, Cambridge, where he was awarded first-class honours in mathematics. His dissertation, On the Gaussian error function, written during his senior year and delivered in November 1934 (with a deadline date of 6 December) proved a version of the central limit theorem. It was finally accepted on 16 March 1935. By spring of that same year, Turing started his master's course (Part III)—which he completed in 1937—and, at the same time, he published his first paper, a one-page article called Equivalence of left and right almost periodicity (sent on 23 April), featured in the tenth volume of the Journal of the London Mathematical Society.[51] Later that year, Turing was elected a Fellow of King's College on the strength of his dissertation.[52] However, and, unknown to Turing, this version of the theorem he proved in his paper, had already been proven, in 1922, by Jarl Waldemar Lindeberg. Despite this, the committee found Turing's methods original and so regarded the work worthy of consideration for the fellowship. Abram Besicovitch's report for the committee went so far as to say that if Turing's work had been published before Lindeberg's, it would have been "an important event in the mathematical literature of that year".[53][54][55]
+
+Between the springs of 1935 and 1936, at the same time as Church, Turing worked on the decidability of problems, starting from Godel's incompleteness theorems. In mid-April 1936, Turing sent Max Newman the first draft typescript of his investigations. That same month, Alonzo Church published his An Unsolvable Problem of Elementary Number Theory, with similar conclusions to Turing's then-yet unpublished work. Finally, on 28 May of that year, he finished and delivered his 36-page paper for publication called "On Computable Numbers, with an Application to the Entscheidungsproblem".[56] It was published in the Proceedings of the London Mathematical Society journal in two parts, the first on 30 November and the second on 23 December.[57] In this paper, Turing reformulated Kurt Gödel's 1931 results on the limits of proof and computation, replacing Gödel's universal arithmetic-based formal language with the formal and simple hypothetical devices that became known as Turing machines. The Entscheidungsproblem (decision problem) was originally posed by German mathematician David Hilbert in 1928. Turing proved that his "universal computing machine" would be capable of performing any conceivable mathematical computation if it were representable as an algorithm. He went on to prove that there was no solution to the decision problem by first showing that the halting problem for Turing machines is undecidable: it is not possible to decide algorithmically whether a Turing machine will ever halt. This paper has been called "easily the most influential math paper in history".[58] "##;
+
+ #[test]
+ fn test_keywords() {
+ let rake = RakeModel::new();
+ let keywords = rake.keywords(TURING_TEXT, Lang::Eng);
+
+ assert!(!keywords.is_empty());
+ }
+}
diff --git a/crates/core/src/ranking/models/lambdamart.rs b/crates/core/src/ranking/models/lambdamart.rs
index 24737cfd..f1962260 100644
--- a/crates/core/src/ranking/models/lambdamart.rs
+++ b/crates/core/src/ranking/models/lambdamart.rs
@@ -310,7 +310,7 @@ mod tests {
assert!(!model.trees.is_empty());
let mut features = EnumMap::new();
- features.insert(Signal::IdfSumBacklinkText, 85.7750244140625);
+ features.insert(Signal::Bm25BacklinkText, 85.7750244140625);
features.insert(Signal::Bm25CleanBody, 67.41311645507812);
features.insert(Signal::Bm25CleanBodyBigrams, 0.0);
features.insert(Signal::Bm25CleanBodyTrigrams, 0.0);
diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rs
index 1422ae0e..2904cb84 100644
--- a/crates/core/src/ranking/signal.rs
+++ b/crates/core/src/ranking/signal.rs
@@ -76,6 +76,10 @@ pub enum Signal {
Bm25StemmedCleanBody,
#[serde(rename = "bm25_all_body")]
Bm25AllBody,
+ #[serde(rename = "bm25_keywords")]
+ Bm25Keywords,
+ #[serde(rename = "bm25_backlink_text")]
+ Bm25BacklinkText,
#[serde(rename = "idf_sum_url")]
IdfSumUrl,
#[serde(rename = "idf_sum_site")]
@@ -96,8 +100,6 @@ pub enum Signal {
IdfSumDomainIfHomepageNoTokenizer,
#[serde(rename = "idf_sum_title_if_homepage")]
IdfSumTitleIfHomepage,
- #[serde(rename = "idf_sum_backlink_text")]
- IdfSumBacklinkText,
#[serde(rename = "cross_encoder_snippet")]
CrossEncoderSnippet,
#[serde(rename = "cross_encoder_title")]
@@ -140,7 +142,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 37] = [
+pub const ALL_SIGNALS: [Signal; 38] = [
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
Signal::Bm25TitleTrigrams,
@@ -150,6 +152,8 @@ pub const ALL_SIGNALS: [Signal; 37] = [
Signal::Bm25StemmedTitle,
Signal::Bm25StemmedCleanBody,
Signal::Bm25AllBody,
+ Signal::Bm25BacklinkText,
+ Signal::Bm25Keywords,
Signal::IdfSumUrl,
Signal::IdfSumSite,
Signal::IdfSumDomain,
@@ -160,7 +164,6 @@ pub const ALL_SIGNALS: [Signal; 37] = [
Signal::IdfSumDomainNameIfHomepageNoTokenizer,
Signal::IdfSumDomainIfHomepageNoTokenizer,
Signal::IdfSumTitleIfHomepage,
- Signal::IdfSumBacklinkText,
Signal::CrossEncoderSnippet,
Signal::CrossEncoderTitle,
Signal::HostCentrality,
@@ -308,7 +311,8 @@ impl Signal {
Signal::IdfSumDomainNameIfHomepageNoTokenizer => 0.0036,
Signal::IdfSumDomainIfHomepageNoTokenizer => 0.0036,
Signal::IdfSumTitleIfHomepage => 0.00022,
- Signal::IdfSumBacklinkText => 0.003,
+ Signal::Bm25BacklinkText => 0.003,
+ Signal::Bm25Keywords => 0.0005,
Signal::CrossEncoderSnippet => 0.17,
Signal::CrossEncoderTitle => 0.17,
Signal::HostCentrality => 0.5,
@@ -411,7 +415,9 @@ impl Signal {
| Signal::Bm25CleanBodyTrigrams
| Signal::Bm25StemmedTitle
| Signal::Bm25StemmedCleanBody
- | Signal::Bm25AllBody => seg_reader
+ | Signal::Bm25AllBody
+ | Signal::Bm25Keywords
+ | Signal::Bm25BacklinkText => seg_reader
.text_fields
.get_mut(self.as_textfield().unwrap())
.map(|field| bm25(field, doc)),
@@ -425,8 +431,7 @@ impl Signal {
| Signal::IdfSumDomainIfHomepage
| Signal::IdfSumDomainNameIfHomepageNoTokenizer
| Signal::IdfSumDomainIfHomepageNoTokenizer
- | Signal::IdfSumTitleIfHomepage
- | Signal::IdfSumBacklinkText => seg_reader
+ | Signal::IdfSumTitleIfHomepage => seg_reader
.text_fields
.get_mut(self.as_textfield().unwrap())
.map(|field| idf_sum(field, doc)),
@@ -523,6 +528,8 @@ impl Signal {
| Signal::Bm25StemmedTitle
| Signal::Bm25StemmedCleanBody
| Signal::Bm25AllBody
+ | Signal::Bm25BacklinkText
+ | Signal::Bm25Keywords
| Signal::IdfSumUrl
| Signal::IdfSumSite
| Signal::IdfSumDomain
@@ -533,7 +540,6 @@ impl Signal {
| Signal::IdfSumDomainNameIfHomepageNoTokenizer
| Signal::IdfSumDomainIfHomepageNoTokenizer
| Signal::IdfSumTitleIfHomepage
- | Signal::IdfSumBacklinkText
| Signal::CrossEncoderSnippet
| Signal::CrossEncoderTitle
| Signal::InboundSimilarity
@@ -582,6 +588,8 @@ impl Signal {
Signal::Bm25StemmedTitle => Some(TextField::StemmedTitle),
Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),
Signal::Bm25AllBody => Some(TextField::AllBody),
+ Signal::Bm25BacklinkText => Some(TextField::BacklinkText),
+ Signal::Bm25Keywords => Some(TextField::Keywords),
Signal::IdfSumUrl => Some(TextField::Url),
Signal::IdfSumSite => Some(TextField::SiteWithout),
Signal::IdfSumDomain => Some(TextField::Domain),
@@ -593,7 +601,6 @@ impl Signal {
Some(TextField::DomainNameIfHomepageNoTokenizer)
}
Signal::IdfSumTitleIfHomepage => Some(TextField::TitleIfHomepage),
- Signal::IdfSumBacklinkText => Some(TextField::BacklinkText),
Signal::IdfSumDomainIfHomepageNoTokenizer => {
Some(TextField::DomainIfHomepageNoTokenizer)
}
diff --git a/crates/core/src/schema.rs b/crates/core/src/schema.rs
index d2035d32..253822cd 100644
--- a/crates/core/src/schema.rs
+++ b/crates/core/src/schema.rs
@@ -61,6 +61,7 @@ pub enum TextField {
SafetyClassification,
InsertionTimestamp,
RecipeFirstIngredientTagId,
+ Keywords,
}
impl From<TextField> for usize {
@@ -103,6 +104,7 @@ impl TextField {
TextField::SafetyClassification => 1,
TextField::InsertionTimestamp => 1,
TextField::RecipeFirstIngredientTagId => 1,
+ TextField::Keywords => 1,
}
}
@@ -141,6 +143,7 @@ impl TextField {
TextField::SafetyClassification => TextField::SafetyClassification,
TextField::InsertionTimestamp => TextField::InsertionTimestamp,
TextField::RecipeFirstIngredientTagId => TextField::RecipeFirstIngredientTagId,
+ TextField::Keywords => TextField::Keywords,
}
}
@@ -187,6 +190,7 @@ impl TextField {
TextField::SafetyClassification => Tokenizer::Identity(Identity {}),
TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),
TextField::RecipeFirstIngredientTagId => Tokenizer::Identity(Identity {}),
+ TextField::Keywords => Tokenizer::default(),
}
}
@@ -231,6 +235,7 @@ impl TextField {
TextField::SafetyClassification => false,
TextField::InsertionTimestamp => false,
TextField::RecipeFirstIngredientTagId => false,
+ TextField::Keywords => false,
}
}
@@ -267,6 +272,7 @@ impl TextField {
TextField::SafetyClassification => "safety_classification",
TextField::InsertionTimestamp => "insertion_timestamp",
TextField::RecipeFirstIngredientTagId => "recipe_first_ingredient_tag_id",
+ TextField::Keywords => "keywords",
}
}
}
@@ -368,7 +374,7 @@ pub enum Field {
Text(TextField),
}
-static ALL_FIELDS: [Field; 66] = [
+static ALL_FIELDS: [Field; 67] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -399,6 +405,7 @@ static ALL_FIELDS: [Field; 66] = [
Field::Text(TextField::MicroformatTags),
Field::Text(TextField::SafetyClassification),
Field::Text(TextField::InsertionTimestamp),
+ Field::Text(TextField::Keywords),
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
@@ -555,6 +562,9 @@ impl Field {
Field::Text(TextField::InsertionTimestamp) => {
IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())
}
+ Field::Text(TextField::Keywords) => {
+ IndexingOption::Text(self.default_text_options().set_stored())
+ }
Field::Fast(FastField::IsHomepage) => {
IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
}
diff --git a/crates/core/src/stopwords.rs b/crates/core/src/stopwords.rs
new file mode 100644
index 00000000..8b13eb69
--- /dev/null
+++ b/crates/core/src/stopwords.rs
@@ -0,0 +1,80 @@
+use hashbrown::{HashMap, HashSet};
+use whatlang::Lang;
+
+macro_rules! include_stopwords {
+ ($($file:expr => $lang:expr),*) => {{
+ let mut stopwords = HashMap::new();
+
+ $(
+ stopwords.insert(
+ $lang,
+ include_str!($file)
+ .lines()
+ .map(|s| s.to_lowercase())
+ .collect(),
+ );
+ )*
+
+ stopwords
+ }};
+}
+
+static STOPWORDS: once_cell::sync::Lazy<HashMap<Lang, HashSet<String>>> =
+ once_cell::sync::Lazy::new(|| {
+ include_stopwords!(
+ "../stopwords/Afrikaans.txt" => Lang::Afr,
+ "../stopwords/Arabic.txt" => Lang::Ara,
+ "../stopwords/Armenian.txt" => Lang::Hye,
+ "../stopwords/Azerbaijani.txt" => Lang::Aze,
+ "../stopwords/Belarusian.txt" => Lang::Bel,
+ "../stopwords/Bengali.txt" => Lang::Ben,
+ "../stopwords/Bulgarian.txt" => Lang::Bul,
+ "../stopwords/Catalan.txt" => Lang::Cat,
+ "../stopwords/Croatian.txt" => Lang::Hrv,
+ "../stopwords/Czech.txt" => Lang::Ces,
+ "../stopwords/Danish.txt" => Lang::Dan,
+ "../stopwords/Dutch.txt" => Lang::Nld,
+ "../stopwords/English.txt" => Lang::Eng,
+ "../stopwords/Esperanto.txt" => Lang::Epo,
+ "../stopwords/Estonian.txt" => Lang::Est,
+ "../stopwords/Finnish.txt" => Lang::Fin,
+ "../stopwords/French.txt" => Lang::Fra,
+ "../stopwords/Georgian.txt" => Lang::Kat,
+ "../stopwords/German.txt" => Lang::Deu,
+ "../stopwords/Greek.txt" => Lang::Ell,
+ "../stopwords/Gujarati.txt" => Lang::Guj,
+ "../stopwords/Hebrew.txt" => Lang::Heb,
+ "../stopwords/Hindi.txt" => Lang::Hin,
+ "../stopwords/Hungarian.txt" => Lang::Hun,
+ "../stopwords/Indonesian.txt" => Lang::Ind,
+ "../stopwords/Italian.txt" => Lang::Ita,
+ "../stopwords/Javanese.txt" => Lang::Jav,
+ "../stopwords/Kannada.txt" => Lang::Kan,
+ "../stopwords/Korean.txt" => Lang::Kor,
+ "../stopwords/Latin.txt" => Lang::Lat,
+ "../stopwords/Latvian.txt" => Lang::Lav,
+ "../stopwords/Lithuanian.txt" => Lang::Lit,
+ "../stopwords/Macedonian.txt" => Lang::Mkd,
+ "../stopwords/Malayalam.txt" => Lang::Mal,
+ "../stopwords/Marathi.txt" => Lang::Mar,
+ "../stopwords/Nepali.txt" => Lang::Nep,
+ "../stopwords/Persian.txt" => Lang::Pes,
+ "../stopwords/Polish.txt" => Lang::Pol,
+ "../stopwords/Portuguese.txt" => Lang::Por,
+ "../stopwords/Romanian.txt" => Lang::Ron,
+ "../stopwords/Russian.txt" => Lang::Rus,
+ "../stopwords/Serbian.txt" => Lang::Srp,
+ "../stopwords/Slovak.txt" => Lang::Slk,
+ "../stopwords/Slovenian.txt" => Lang::Slv,
+ "../stopwords/Spanish.txt" => Lang::Spa,
+ "../stopwords/Japanese.txt" => Lang::Jpn
+ )
+ });
+
+pub fn get(lang: &Lang) -> Option<&'static HashSet<String>> {
+ STOPWORDS.get(lang)
+}
+
+pub fn all() -> &'static HashMap<Lang, HashSet<String>> {
+ &STOPWORDS
+}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index 69293c4f..443d5ff7 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -17,6 +17,7 @@
use crate::{
ceil_char_boundary,
prehashed::hash,
+ rake::RakeModel,
schema::{FastField, TextField},
simhash, split_u128, tokenizer,
webpage::url_ext::UrlExt,
@@ -116,7 +117,23 @@ impl Html {
PreTokenizedString { text, tokens }
}
- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+
+ pub fn keywords(&self, rake: &RakeModel) -> Vec<String> {
+ self.clean_text()
+ .map(|text| {
+ rake.keywords(text, self.lang.unwrap_or(Lang::Eng))
+ .into_iter()
+ .map(|k| k.text)
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn into_tantivy(
+ self,
+ schema: &tantivy::schema::Schema,
+ rake: &RakeModel,
+ ) -> Result<TantivyDocument> {
let mut doc = TantivyDocument::new();
let title = self.pretokenize_title()?;
@@ -398,6 +415,10 @@ impl Html {
doc.add_text(tantivy_field, "");
}
}
+ Field::Text(TextField::Keywords) => {
+ let rake_keywords = self.keywords(rake);
+ doc.add_text(tantivy_field, rake_keywords.join("\n"));
+ }
Field::Text(TextField::AllBody) => {
doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 95ec0db3..68f89979 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -493,6 +493,7 @@ mod tests {
use std::collections::HashMap;
use crate::{
+ rake::RakeModel,
schema::create_schema,
webpage::{url_ext::UrlExt, Link},
};
@@ -740,7 +741,9 @@ mod tests {
assert!(!webpage.all_text().unwrap().is_empty());
let schema = create_schema();
- webpage.into_tantivy(&schema).unwrap();
+ webpage
+ .into_tantivy(&schema, &RakeModel::default())
+ .unwrap();
}
#[test]
diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rs
index 9e7c5711..e774f72f 100644
--- a/crates/core/src/webpage/just_text.rs
+++ b/crates/core/src/webpage/just_text.rs
@@ -14,11 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::collections::{HashMap, HashSet};
-
+use hashbrown::HashSet;
use kuchiki::{iter::NodeEdge, NodeRef};
use whatlang::Lang;
+use crate::stopwords;
+
pub struct Preprocessor<const N: usize> {
removed_tags: [&'static str; N],
num_open_tags: [i64; N],
@@ -116,76 +117,6 @@ enum FinalClassification {
Bad,
}
-macro_rules! include_stopwords {
- ($($file:expr => $lang:expr),*) => {{
- let mut stopwords = HashMap::new();
-
- $(
- stopwords.insert(
- $lang,
- include_str!($file)
- .lines()
- .map(|s| s.to_string())
- .collect(),
- );
- )*
-
- stopwords
- }};
-}
-
-static STOPWORDS: once_cell::sync::Lazy<HashMap<Lang, HashSet<String>>> =
- once_cell::sync::Lazy::new(|| {
- include_stopwords!(
- "../../stopwords/Afrikaans.txt" => Lang::Afr,
- "../../stopwords/Arabic.txt" => Lang::Ara,
- "../../stopwords/Armenian.txt" => Lang::Hye,
- "../../stopwords/Azerbaijani.txt" => Lang::Aze,
- "../../stopwords/Belarusian.txt" => Lang::Bel,
- "../../stopwords/Bengali.txt" => Lang::Ben,
- "../../stopwords/Bulgarian.txt" => Lang::Bul,
- "../../stopwords/Catalan.txt" => Lang::Cat,
- "../../stopwords/Croatian.txt" => Lang::Hrv,
- "../../stopwords/Czech.txt" => Lang::Ces,
- "../../stopwords/Danish.txt" => Lang::Dan,
- "../../stopwords/Dutch.txt" => Lang::Nld,
- "../../stopwords/English.txt" => Lang::Eng,
- "../../stopwords/Esperanto.txt" => Lang::Epo,
- "../../stopwords/Estonian.txt" => Lang::Est,
- "../../stopwords/Finnish.txt" => Lang::Fin,
- "../../stopwords/French.txt" => Lang::Fra,
- "../../stopwords/Georgian.txt" => Lang::Kat,
- "../../stopwords/German.txt" => Lang::Deu,
- "../../stopwords/Greek.txt" => Lang::Ell,
- "../../stopwords/Gujarati.txt" => Lang::Guj,
- "../../stopwords/Hebrew.txt" => Lang::Heb,
- "../../stopwords/Hindi.txt" => Lang::Hin,
- "../../stopwords/Hungarian.txt" => Lang::Hun,
- "../../stopwords/Indonesian.txt" => Lang::Ind,
- "../../stopwords/Italian.txt" => Lang::Ita,
- "../../stopwords/Javanese.txt" => Lang::Jav,
- "../../stopwords/Kannada.txt" => Lang::Kan,
- "../../stopwords/Korean.txt" => Lang::Kor,
- "../../stopwords/Latin.txt" => Lang::Lat,
- "../../stopwords/Latvian.txt" => Lang::Lav,
- "../../stopwords/Lithuanian.txt" => Lang::Lit,
- "../../stopwords/Macedonian.txt" => Lang::Mkd,
- "../../stopwords/Malayalam.txt" => Lang::Mal,
- "../../stopwords/Marathi.txt" => Lang::Mar,
- "../../stopwords/Nepali.txt" => Lang::Nep,
- "../../stopwords/Persian.txt" => Lang::Pes,
- "../../stopwords/Polish.txt" => Lang::Pol,
- "../../stopwords/Portuguese.txt" => Lang::Por,
- "../../stopwords/Romanian.txt" => Lang::Ron,
- "../../stopwords/Russian.txt" => Lang::Rus,
- "../../stopwords/Serbian.txt" => Lang::Srp,
- "../../stopwords/Slovak.txt" => Lang::Slk,
- "../../stopwords/Slovenian.txt" => Lang::Slv,
- "../../stopwords/Spanish.txt" => Lang::Spa,
- "../../stopwords/Japanese.txt" => Lang::Jpn
- )
- });
-
pub struct JustText {
pub max_link_density: f64,
pub length_low: usize,
@@ -418,9 +349,7 @@ impl JustText {
) -> Vec<ClassifiedParagraph> {
let mut res = Vec::new();
- let stopwords = STOPWORDS
- .get(lang)
- .unwrap_or_else(|| STOPWORDS.get(&Lang::Eng).unwrap());
+ let stopwords = stopwords::get(lang).unwrap_or_else(|| stopwords::get(&Lang::Eng).unwrap());
for paragraph in paragraphs {
let classification = {
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index d8584bf7..4ebd2c46 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ rake::RakeModel,
schema::{FastField, TextField},
webgraph::NodeID,
Result,
@@ -101,12 +102,16 @@ impl Webpage {
})
}
- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+ pub fn into_tantivy(
+ self,
+ schema: &tantivy::schema::Schema,
+ rake: &RakeModel,
+ ) -> Result<TantivyDocument> {
let region = Region::guess_from(&self);
let dmoz_description = self.dmoz_description();
- let mut doc = self.html.into_tantivy(schema)?;
+ let mut doc = self.html.into_tantivy(schema, rake)?;
if let Ok(region) = region {
doc.add_u64(
diff --git a/crates/core/testcases/lambdamart.txt b/crates/core/testcases/lambdamart.txt
index 41415863..232241c5 100644
--- a/crates/core/testcases/lambdamart.txt
+++ b/crates/core/testcases/lambdamart.txt
@@ -5,7 +5,7 @@ num_tree_per_iteration=1
label_index=0
max_feature_idx=28
objective=lambdarank
-feature_names=idf_sum_backlink_text inbound_similarity bm25_clean_body_bigrams idf_sum_domain_if_homepage_no_tokenizer page_centrality is_homepage idf_sum_title_if_homepage url_slashes fetch_time_ms region tracker_score idf_sum_domain idf_sum_domain_no_tokenizer bm25_title bm25_stemmed_title idf_sum_domain_name_no_tokenizer host_centrality url_digits idf_sum_domain_name_if_homepage_no_tokenizer bm25_title_trigrams bm25_clean_body_trigrams bm25_title_bigrams bm25_stemmed_clean_body idf_sum_site idf_sum_site_no_tokenizer idf_sum_url idf_sum_domain_if_homepage update_timestamp bm25_clean_body
+feature_names=bm25_backlink_text inbound_similarity bm25_clean_body_bigrams idf_sum_domain_if_homepage_no_tokenizer page_centrality is_homepage idf_sum_title_if_homepage url_slashes fetch_time_ms region tracker_score idf_sum_domain idf_sum_domain_no_tokenizer bm25_title bm25_stemmed_title idf_sum_domain_name_no_tokenizer host_centrality url_digits idf_sum_domain_name_if_homepage_no_tokenizer bm25_title_trigrams bm25_clean_body_trigrams bm25_title_bigrams bm25_stemmed_clean_body idf_sum_site idf_sum_site_no_tokenizer idf_sum_url idf_sum_domain_if_homepage update_timestamp bm25_clean_body
feature_infos=[0:71.599037170410156] none [0:59.439960479736328] none [0:0.041696587] [0:1] [0:24.348766326904297] [0.14285714285714285:0.5] [0:0.16666666666666666] [0:0.70508371084865562] [0.009433962264150943:1] none none [6.0304698944091797:45.648475646972656] [0:44.676078796386719] none [7.6000000000000006e-08:0.045836137999999998] [0.066666666666666666:1] none none none [0:48.516033172607422] [0:40.354255676269531] [0:44.526699066162109] none [0:44.291984558105469] none none [4.4346046447753906:41.890125274658203]
tree_sizes=1051 753 851 859 958 856 759 960 858 651 862 971 753 861 861 973 859 761 865 869 982 763 986 763 984 759 764 765 861 762 767 765 869 869 874 872 873 879 871 883 773 774 877 768 765 777 988 878 767 879
@@ -967,7 +967,7 @@ host_centrality=35
bm25_title_bigrams=34
idf_sum_url=33
bm25_clean_body_bigrams=26
-idf_sum_backlink_text=21
+bm25_backlink_text=21
page_centrality=20
bm25_stemmed_title=20
bm25_title=18