Skip to content

web search done right

Partial support for compounded words. the query "wishlist" now also matches search results that has the terms "wish list". This is done using the bigram- and trigram fields.

Mikkel Denker7706dcdfa2a9
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index 4866676d..86b66d57 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -41,11 +41,12 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let mut searcher = LocalSearcher::new(index);
searcher.set_centrality_store(CentralityStore::open(CENTRALITY_PATH).into());
- for _ in 0..10 {
- bench!("the", searcher, c);
- bench!("dtu", searcher, c);
- bench!("the best", searcher, c);
- bench!("the circle of life", searcher, c);
+ for _ in 0..1000 {
+ // bench!("the", searcher, c);
+ // bench!("dtu", searcher, c);
+ // bench!("the best", searcher, c);
+ // bench!("the circle of life", searcher, c);
+ bench!("runescape is old the", searcher, c);
}
}
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 7df47c1f..e7b6ee50 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -78,7 +78,7 @@ impl Crawler {
}
pub fn min_crawl_delay_ms() -> u64 {
- 200
+ 4_000
}
pub fn max_crawl_delay_ms() -> u64 {
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 0a85c141..3c42dc7a 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -57,7 +57,7 @@ impl RobotsTxtManager {
let mut res = self
.client
.get(&format!("http://{}/robots.txt", site.0))
- .timeout(Duration::from_secs(30))
+ .timeout(Duration::from_secs(60))
.send()
.await;
@@ -65,7 +65,7 @@ impl RobotsTxtManager {
res = self
.client
.get(&format!("https://{}/robots.txt", site.0))
- .timeout(Duration::from_secs(30))
+ .timeout(Duration::from_secs(60))
.send()
.await;
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index b779e0b8..6ce4180d 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -73,7 +73,7 @@ impl Query {
let queries: Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> = terms
.iter()
- .flat_map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
+ .map(|term| term.as_tantivy_query(&fields, tokenizer_manager))
.collect();
let simple_terms_text: Vec<String> = terms
@@ -737,4 +737,43 @@ mod tests {
assert_eq!(result.num_hits, 0);
assert_eq!(result.webpages.len(), 0);
}
+
+ #[test]
+ fn match_compound_words() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ index
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Test website</title>
+ </head>
+ <body>
+ This is a test website {}
+ </body>
+ </html>
+ "#,
+ rand_words(1000)
+ ),
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let searcher = LocalSearcher::from(index);
+
+ let query = SearchQuery {
+ query: "testwebsite".to_string(),
+ ..Default::default()
+ };
+
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ }
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 95419433..c8206c8c 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use tantivy::{
- query::{BooleanQuery, Occur, PhraseQuery, TermQuery},
+ query::{BooleanQuery, Occur, TermQuery},
tokenizer::{TextAnalyzer, TokenizerManager},
};
@@ -46,7 +46,7 @@ impl ToString for Term {
Term::Title(title) => "intitle:".to_string() + title.as_str(),
Term::Body(body) => "inbody:".to_string() + body.as_str(),
Term::Url(url) => "inurl:".to_string() + url.as_str(),
- Term::PossibleBang(bang) => "!".to_string() + bang.as_str(),
+ Term::PossibleBang(bang) => BANG_PREFIX.to_string() + bang.as_str(),
}
}
}
@@ -55,36 +55,15 @@ fn simple_into_tantivy(
term: &str,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
-) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
- let (backlink_field, backlink_field_entry) = fields
- .iter()
- .find(|(field, _)| {
- matches!(
- ALL_FIELDS[field.field_id() as usize],
- Field::Text(TextField::BacklinkText)
- )
- })
- .unwrap();
-
- vec![
- (
- Occur::Must,
- Box::new(BooleanQuery::new(Term::into_tantivy_simple(
- term,
- fields,
- tokenizer_manager,
- ))),
- ),
- (
- Occur::Should,
- Box::new(Term::tantivy_text_query(
- backlink_field,
- backlink_field_entry,
- tokenizer_manager,
- term,
- )),
- ),
- ]
+) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
+ (
+ Occur::Must,
+ Box::new(BooleanQuery::new(Term::into_tantivy_simple(
+ term,
+ fields,
+ tokenizer_manager,
+ ))),
+ )
}
impl Term {
@@ -105,7 +84,7 @@ impl Term {
&self,
fields: &[(tantivy::schema::Field, &tantivy::schema::FieldEntry)],
tokenizer_manager: &TokenizerManager,
- ) -> Vec<(Occur, Box<dyn tantivy::query::Query + 'static>)> {
+ ) -> (Occur, Box<dyn tantivy::query::Query + 'static>) {
match self {
Term::Simple(term) => simple_into_tantivy(term, fields, tokenizer_manager),
Term::Phrase(phrase) => {
@@ -122,22 +101,22 @@ impl Term {
));
}
- vec![(Occur::Must, Box::new(BooleanQuery::new(phrases)))]
+ (Occur::Must, Box::new(BooleanQuery::new(phrases)))
}
- Term::Not(subterm) => vec![(
+ Term::Not(subterm) => (
Occur::MustNot,
- Box::new(BooleanQuery::new(
- subterm.as_tantivy_query(fields, tokenizer_manager),
- )),
- )],
- Term::Site(site) => vec![(
+ Box::new(BooleanQuery::new(vec![
+ subterm.as_tantivy_query(fields, tokenizer_manager)
+ ])),
+ ),
+ Term::Site(site) => (
Occur::Must,
Box::new(BooleanQuery::new(Term::into_tantivy_site(
site,
fields,
tokenizer_manager,
))),
- )],
+ ),
Term::Title(title) => {
let (field, entry) = fields
.iter()
@@ -149,10 +128,10 @@ impl Term {
})
.unwrap();
- vec![(
+ (
Occur::Must,
Term::tantivy_text_query(field, entry, tokenizer_manager, title),
- )]
+ )
}
Term::Body(body) => {
let (field, entry) = fields
@@ -165,10 +144,10 @@ impl Term {
})
.unwrap();
- vec![(
+ (
Occur::Must,
Term::tantivy_text_query(field, entry, tokenizer_manager, body),
- )]
+ )
}
Term::Url(url) => {
let (field, entry) = fields
@@ -181,10 +160,10 @@ impl Term {
})
.unwrap();
- vec![(
+ (
Occur::Must,
Term::tantivy_text_query(field, entry, tokenizer_manager, url),
- )]
+ )
}
Term::PossibleBang(text) => {
let mut term = String::new();
@@ -237,7 +216,7 @@ impl Term {
.collect()
}
- fn tantivy_text_query(
+ pub fn tantivy_text_query(
field: &tantivy::schema::Field,
entry: &tantivy::schema::FieldEntry,
tokenizer_manager: &TokenizerManager,
@@ -246,18 +225,28 @@ impl Term {
let analyzer = Term::get_tantivy_analyzer(entry, tokenizer_manager);
let mut processed_terms = Term::process_tantivy_term(term, analyzer, *field);
- let processed_query =
- if processed_terms.len() > 1 && ALL_FIELDS[field.field_id() as usize].has_pos() {
- Box::new(PhraseQuery::new(processed_terms)) as Box<dyn tantivy::query::Query>
- } else {
- let option = ALL_FIELDS[field.field_id() as usize]
- .as_text()
- .unwrap()
- .index_option();
-
- let term = processed_terms.remove(0);
- Box::new(TermQuery::new(term, option))
- };
+ let option = ALL_FIELDS[field.field_id() as usize]
+ .as_text()
+ .unwrap()
+ .index_option();
+
+ let processed_query = if processed_terms.len() == 1 {
+ let term = processed_terms.remove(0);
+ Box::new(TermQuery::new(term, option)) as Box<dyn tantivy::query::Query + 'static>
+ } else {
+ Box::new(BooleanQuery::new(
+ processed_terms
+ .into_iter()
+ .map(|term| {
+ (
+ Occur::Must,
+ Box::new(TermQuery::new(term, option))
+ as Box<dyn tantivy::query::Query + 'static>,
+ )
+ })
+ .collect(),
+ )) as Box<dyn tantivy::query::Query + 'static>
+ };
Box::new(processed_query)
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 50eaf3d9..3df612a9 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -558,10 +558,6 @@ impl Field {
self,
Field::Text(TextField::BacklinkText)
| Field::Text(TextField::SchemaOrgJson)
- | Field::Text(TextField::CleanBodyBigrams)
- | Field::Text(TextField::TitleBigrams)
- | Field::Text(TextField::CleanBodyTrigrams)
- | Field::Text(TextField::TitleTrigrams)
| Field::Text(TextField::MicroformatTags)
) && !self.is_fast()
}
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index c14b8863..fc81d589 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -18,7 +18,7 @@ use crate::config::SnippetConfig;
use crate::query::Query;
use crate::search_prettifier::html_escape;
use crate::spell::sentence_ranges;
-use crate::tokenizer::{Stemmed, Tokenizer};
+use crate::tokenizer::{BigramTokenizer, Stemmed, Tokenizer, TrigramTokenizer};
use crate::webpage::region::Region;
use std::collections::{HashMap, HashSet};
use std::ops::Range;
@@ -59,7 +59,7 @@ impl SnippetString {
let mut html = String::new();
let mut start_from: usize = 0;
- for item in self.highlighted.iter() {
+ for item in self.highlighted.iter().filter(|item| item.start < item.end) {
if item.start < start_from {
start_from = item.end;
continue;
@@ -77,13 +77,24 @@ impl SnippetString {
}
fn highlight(&mut self, terms: &HashSet<String>, lang: whatlang::Lang) {
- let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
- let mut stream = tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &self.fragment);
- while let Some(tok) = stream.next() {
- if terms.contains(&tok.text) {
- self.highlighted.push(tok.offset_from..tok.offset_to);
+ for tokenizer in [
+ Tokenizer::Stemmed(Stemmed::with_forced_language(lang)),
+ Tokenizer::Bigram(BigramTokenizer {}),
+ Tokenizer::Trigram(TrigramTokenizer {}),
+ ] {
+ let mut stream =
+ tantivy::tokenizer::Tokenizer::token_stream(&tokenizer, &self.fragment);
+ while let Some(tok) = stream.next() {
+ if terms.contains(&tok.text) {
+ self.highlighted.push(tok.offset_from..tok.offset_to);
+ }
}
}
+
+ // remove overlapping ranges
+ self.highlighted.sort_by(|a, b| a.start.cmp(&b.start));
+ self.highlighted
+ .dedup_by(|a, b| a.start == b.start && a.end > b.end);
}
}
@@ -428,4 +439,20 @@ Survey in 2016, 2017, and 2018."#;
""
);
}
+
+ #[test]
+ fn compounded_terms() {
+ let mut snip = snippet_string(
+ "this is a test",
+ &["thisis".to_string()],
+ whatlang::Lang::Eng,
+ SnippetConfig::default(),
+ );
+
+ let mut terms = HashSet::new();
+ terms.insert("thisis".to_string());
+
+ snip.highlight(&terms, whatlang::Lang::Eng);
+ assert_eq!(snip.to_html().as_str(), "<b>this is</b> a test");
+ }
}
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index 7dfaaf8a..71720e70 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -23,14 +23,12 @@ use std::ops::Range;
use itertools::intersperse;
use serde::{Deserialize, Serialize};
-use tantivy::tokenizer::Tokenizer as _;
use tracing::info;
use crate::index::Index;
use crate::query::parser::Term;
use crate::schema::TextField;
use crate::searcher::SearchQuery;
-use crate::tokenizer::Tokenizer;
use crate::{floor_char_boundary, query};
use self::dictionary::DictionaryBuilder;
@@ -154,75 +152,6 @@ impl Spell {
}
}
- for segment in searcher.segment_readers() {
- let inv_index = segment
- .inverted_index(
- schema
- .get_field(TextField::CleanBodyBigrams.name())
- .unwrap(),
- )
- .unwrap();
- let term_dict = inv_index.terms();
- let mut stream = term_dict.stream().unwrap();
- let mut count = 0;
-
- while let Some((term, info)) = stream.next() {
- if let Some(limit) = limit_terms {
- if count > limit {
- break;
- }
- }
-
- let term = std::str::from_utf8(term).unwrap();
- if !term.is_empty()
- && term
- .chars()
- .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())
- {
- let mut tokenizer = Tokenizer::default().token_stream(term);
- let first = tokenizer.next().unwrap().text.to_ascii_lowercase();
- let second = tokenizer.next().unwrap().text.to_ascii_lowercase();
- dict.add_bigram((first, second), info.doc_freq as u64);
- count += 1;
- }
- }
- }
-
- for segment in searcher.segment_readers() {
- let inv_index = segment
- .inverted_index(
- schema
- .get_field(TextField::CleanBodyTrigrams.name())
- .unwrap(),
- )
- .unwrap();
- let term_dict = inv_index.terms();
- let mut stream = term_dict.stream().unwrap();
- let mut count = 0;
-
- while let Some((term, info)) = stream.next() {
- if let Some(limit) = limit_terms {
- if count > limit {
- break;
- }
- }
-
- let term = std::str::from_utf8(term).unwrap();
- if !term.is_empty()
- && term
- .chars()
- .all(|c| c.is_ascii_alphabetic() || c.is_whitespace())
- {
- let mut tokenizer = Tokenizer::default().token_stream(term);
- let first = tokenizer.next().unwrap().text.to_ascii_lowercase();
- let second = tokenizer.next().unwrap().text.to_ascii_lowercase();
- let third = tokenizer.next().unwrap().text.to_ascii_lowercase();
- dict.add_trigram((first, second, third), info.doc_freq as u64);
- count += 1;
- }
- }
- }
-
dict.build()
}
diff --git a/core/src/tokenizer.rs b/core/src/tokenizer.rs
index df57008c..b8192516 100644
--- a/core/src/tokenizer.rs
+++ b/core/src/tokenizer.rs
@@ -308,7 +308,6 @@ struct NGramTokenStream<'a, const N: usize> {
inner: BoxTokenStream<'a>,
token: tantivy::tokenizer::Token,
token_window: [tantivy::tokenizer::Token; N],
- is_first: bool,
next_pos: usize,
}
@@ -318,7 +317,6 @@ impl<'a, const N: usize> NGramTokenStream<'a, N> {
inner,
token: tantivy::tokenizer::Token::default(),
token_window: array::from_fn(|_| tantivy::tokenizer::Token::default()),
- is_first: true,
next_pos: 0,
}
}
@@ -333,122 +331,47 @@ fn reuse_token_alloc(token: &mut tantivy::tokenizer::Token, new_token: &tantivy:
token.position_length = new_token.position_length;
}
-impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 1> {
+impl<'a, const N: usize> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, N> {
fn advance(&mut self) -> bool {
- self.is_first = false;
- let res = self.inner.advance();
-
- self.token_window[0].text.clear();
-
- if res {
- self.token_window[0].text += self.inner.token().text.as_str();
- self.next_pos += 1;
- }
-
- reuse_token_alloc(&mut self.token, &self.token_window[0]);
- self.token.position_length = 1;
-
- res
- }
-
- fn token(&self) -> &tantivy::tokenizer::Token {
- &self.token
- }
-
- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
- &mut self.token
- }
-}
-
-impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 2> {
- fn advance(&mut self) -> bool {
- if self.is_first {
- if !self.inner.advance() {
- return false;
- }
- reuse_token_alloc(&mut self.token_window[0], self.inner.token());
-
- if !self.inner.advance() {
- return false;
- }
- reuse_token_alloc(&mut self.token_window[1], self.inner.token());
- } else {
- if !self.inner.advance() {
- return false;
+ if !self.inner.advance() {
+ while self.token_window[0].text.is_empty()
+ && !self.token_window.iter().all(|token| token.text.is_empty())
+ {
+ self.token_window.rotate_left(1);
}
self.token_window.rotate_left(1);
- reuse_token_alloc(&mut self.token_window[1], self.inner.token());
- }
- self.is_first = false;
- self.next_pos += 1;
+ reuse_token_alloc(
+ &mut self.token_window[N - 1],
+ &tantivy::tokenizer::Token::default(),
+ );
- self.token.position = self.next_pos;
- self.token.offset_from = self.token_window[0].offset_from;
- self.token.offset_to = self.token_window[1].offset_to;
- self.token.position_length = 2;
-
- self.token.text.clear();
- for token in &self.token_window {
- self.token.text += token.text.as_str();
- self.token.text += " ";
- }
- self.token.text.pop();
-
- true
- }
-
- fn token(&self) -> &tantivy::tokenizer::Token {
- &self.token
- }
-
- fn token_mut(&mut self) -> &mut tantivy::tokenizer::Token {
- &mut self.token
- }
-}
-
-impl<'a> tantivy::tokenizer::TokenStream for NGramTokenStream<'a, 3> {
- fn advance(&mut self) -> bool {
- if self.is_first {
- if !self.inner.advance() {
+ if self.token_window.iter().all(|token| token.text.is_empty()) {
return false;
}
-
- reuse_token_alloc(&mut self.token_window[0], self.inner.token());
-
- if !self.inner.advance() {
- return false;
- }
- reuse_token_alloc(&mut self.token_window[1], self.inner.token());
-
- if !self.inner.advance() {
- return false;
- }
- reuse_token_alloc(&mut self.token_window[2], self.inner.token());
} else {
- if !self.inner.advance() {
- return false;
- }
-
self.token_window.rotate_left(1);
- reuse_token_alloc(&mut self.token_window[2], self.inner.token());
+ reuse_token_alloc(&mut self.token_window[N - 1], self.inner.token());
}
- self.is_first = false;
self.next_pos += 1;
+ let begin = self
+ .token_window
+ .iter()
+ .position(|token| !token.text.is_empty())
+ .unwrap_or(N - 1);
+
self.token.position = self.next_pos;
- self.token.offset_from = self.token_window[0].offset_from;
- self.token.offset_to = self.token_window[2].offset_to;
- self.token.position_length = 3;
+ self.token.offset_from = self.token_window[begin].offset_from;
+ self.token.offset_to = self.token_window[N - 1].offset_to;
+ self.token.position_length = N - begin;
self.token.text.clear();
for token in &self.token_window {
self.token.text += token.text.as_str();
- self.token.text += " ";
}
- self.token.text.pop();
true
}
@@ -892,38 +815,72 @@ key1.key2="this\" is @ a # test""#;
#[test]
fn bigram_tokenizer() {
assert!(tokenize_bigram("").is_empty());
- assert!(tokenize_bigram("test").is_empty());
+ assert_eq!(tokenize_bigram("test"), vec!["test".to_string()]);
- assert_eq!(tokenize_bigram("this is"), vec!["this is".to_string()]);
+ assert_eq!(
+ tokenize_bigram("this is"),
+ vec!["this".to_string(), "thisis".to_string(), "is".to_string()]
+ );
assert_eq!(
tokenize_bigram("this is a"),
- vec!["this is".to_string(), "is a".to_string()]
+ vec![
+ "this".to_string(),
+ "thisis".to_string(),
+ "isa".to_string(),
+ "a".to_string()
+ ]
);
assert_eq!(
tokenize_bigram("this is a test"),
vec![
- "this is".to_string(),
- "is a".to_string(),
- "a test".to_string()
+ "this".to_string(),
+ "thisis".to_string(),
+ "isa".to_string(),
+ "atest".to_string(),
+ "test".to_string()
]
);
assert_eq!(
tokenize_bigram("this.is"),
- vec!["this .".to_string(), ". is".to_string()]
+ vec![
+ "this".to_string(),
+ "this.".to_string(),
+ ".is".to_string(),
+ "is".to_string()
+ ]
);
}
#[test]
fn trigram_tokenizer() {
assert!(tokenize_trigram("").is_empty());
- assert!(tokenize_trigram("test").is_empty());
- assert!(tokenize_trigram("this is").is_empty());
+ assert_eq!(tokenize_trigram("test"), vec!["test".to_string()]);
+ assert_eq!(
+ tokenize_trigram("this is"),
+ vec!["this".to_string(), "thisis".to_string(), "is".to_string()]
+ );
- assert_eq!(tokenize_trigram("this is a"), vec!["this is a".to_string()]);
+ assert_eq!(
+ tokenize_trigram("this is a"),
+ vec![
+ "this".to_string(),
+ "thisis".to_string(),
+ "thisisa".to_string(),
+ "isa".to_string(),
+ "a".to_string()
+ ]
+ );
assert_eq!(
tokenize_trigram("this is a test"),
- vec!["this is a".to_string(), "is a test".to_string(),]
+ vec![
+ "this".to_string(),
+ "thisis".to_string(),
+ "thisisa".to_string(),
+ "isatest".to_string(),
+ "atest".to_string(),
+ "test".to_string()
+ ]
);
}
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index 93102b49..e8f0a2b7 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -10,7 +10,7 @@ The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open so
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.
-It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 200 ms), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 200 ms between requests and at most 60 seconds.
+It does this by waiting a certain amount of time between requests to the same domain. The waiting time is calculated by *min(max(fetchtime * politeness, 4 sec), 60 sec)* where *fetchtime* is the time it took to fetch the page. The crawler will wait at least 4 sec between requests and at most 60 seconds.
This dynamic waiting time tries to prevent us from disrupting servers that cannot handle the load, while not giving unnecessary politeness to servers that can. The politeness factor starts at 1 and is doubled every time the crawler gets a 429 response from the server (to at most 2048).
The crawler looks for the token `StractBot` in the robots.txt file to determine which pages (if any) the crawler is allowed to crawl.