Skip to content

web search done right

Schema fields as traits (#185)

Mikkel Denker2dadbf70d657
diff --git a/.gitignore b/.gitignore
index 8fc1d80a..e94fafa8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ proptest-regressions
*.pending-snap
.ipynb_checkpoints
.zed
+rustc-ice-*
diff --git a/Cargo.lock b/Cargo.lock
index b22de01f..213c5a0c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1373,6 +1373,18 @@ dependencies = [
"syn 2.0.52",
]
+[[package]]
+name = "enum_dispatch"
+version = "0.3.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e"
+dependencies = [
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
[[package]]
name = "equivalent"
version = "1.0.1"
@@ -4174,8 +4186,8 @@ dependencies = [
"smallvec",
"snap",
"socket2",
- "strum",
- "strum_macros",
+ "strum 0.23.0",
+ "strum_macros 0.23.1",
"thiserror",
"tokio",
"tracing",
@@ -4536,6 +4548,7 @@ dependencies = [
"csv",
"dashmap",
"encoding_rs",
+ "enum_dispatch",
"eventsource-stream",
"fend-core",
"flate2",
@@ -4586,6 +4599,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
+ "strum 0.26.2",
"tantivy",
"thiserror",
"tikv-jemallocator",
@@ -4650,6 +4664,15 @@ version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
+[[package]]
+name = "strum"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
+dependencies = [
+ "strum_macros 0.26.2",
+]
+
[[package]]
name = "strum_macros"
version = "0.23.1"
@@ -4663,6 +4686,19 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "strum_macros"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946"
+dependencies = [
+ "heck 0.4.1",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn 2.0.52",
+]
+
[[package]]
name = "subtle"
version = "2.5.0"
diff --git a/Cargo.toml b/Cargo.toml
index f7c975af..a0e16e54 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ crossbeam-channel = "0.5.6"
csv = "1.1.6"
dashmap = { version = "5.4.0", features = ["rayon"] }
encoding_rs = "0.8.31"
+enum_dispatch = "0.3.12"
eventsource-stream = "0.2.3"
fend-core = "1.2.2"
flate2 = "1.0.28"
@@ -95,6 +96,7 @@ scylla = { version = "0.12.0", features = ["chrono"] }
serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
+strum = { version = "0.26.2", features = ["derive"] }
tantivy = { git = "https://github.com/quickwit-oss/tantivy", rev = "182f58cea" }
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
diff --git a/assets/licenses.html b/assets/licenses.html
index f9d0aa04..cdb3b33a 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -44,8 +44,8 @@
<h2>Overview of licenses:</h2>
<ul class="licenses-overview">
- <li><a href="#Apache-2.0">Apache License 2.0</a> (394)</li>
- <li><a href="#MIT">MIT License</a> (180)</li>
+ <li><a href="#Apache-2.0">Apache License 2.0</a> (395)</li>
+ <li><a href="#MIT">MIT License</a> (182)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (8)</li>
<li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li>
@@ -9889,6 +9889,7 @@ limitations under the License.
<li><a href=" https://github.com/zrzka/anes-rs ">anes 0.1.6</a></li>
<li><a href=" https://github.com/huggingface/candle ">candle-nn 0.3.3</a></li>
<li><a href=" https://github.com/huggingface/candle ">candle-transformers 0.3.3</a></li>
+ <li><a href=" https://gitlab.com/antonok/enum_dispatch ">enum_dispatch 0.3.12</a></li>
<li><a href=" https://github.com/jpopesculian/eventsource-stream ">eventsource-stream 0.2.3</a></li>
<li><a href=" https://github.com/cbreeden/fxhash ">fxhash 0.2.1</a></li>
<li><a href=" https://github.com/starkat99/half-rs ">half 2.4.0</a></li>
@@ -13161,7 +13162,9 @@ SOFTWARE.
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://github.com/Peternator7/strum ">strum 0.23.0</a></li>
+ <li><a href=" https://github.com/Peternator7/strum ">strum 0.26.2</a></li>
<li><a href=" https://github.com/Peternator7/strum ">strum_macros 0.23.1</a></li>
+ <li><a href=" https://github.com/Peternator7/strum ">strum_macros 0.26.2</a></li>
</ul>
<pre class="license-text">MIT License
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 520c2940..4fc3035d 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -45,6 +45,7 @@ crossbeam-channel = { workspace = true }
csv = { workspace = true }
dashmap = { workspace = true }
encoding_rs = { workspace = true }
+enum_dispatch = { workspace = true }
eventsource-stream = { workspace = true }
fend-core = { workspace = true }
flate2 = { workspace = true }
@@ -91,6 +92,7 @@ scylla = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_urlencoded = { workspace = true }
+strum = { workspace = true }
tantivy = { workspace = true }
thiserror = { workspace = true }
tokenizers = { workspace = true }
diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rs
index c7aacd41..c9854fcb 100644
--- a/crates/core/examples/indexer.rs
+++ b/crates/core/examples/indexer.rs
@@ -62,6 +62,6 @@ fn main() -> anyhow::Result<()> {
println!("Indexing took {:?}", start.elapsed());
- std::fs::remove_dir(path)?;
+ std::fs::remove_dir_all(path)?;
Ok(())
}
diff --git a/crates/core/src/collector.rs b/crates/core/src/collector.rs
index bd921c0b..583bd5ad 100644
--- a/crates/core/src/collector.rs
+++ b/crates/core/src/collector.rs
@@ -30,7 +30,7 @@ use crate::{
inverted_index::{DocAddress, WebpagePointer},
prehashed::Prehashed,
ranking::initial::{InitialScoreTweaker, Score},
- schema::FastField,
+ schema::{fast_field, FastFieldEnum},
simhash,
};
@@ -139,7 +139,7 @@ pub struct TopSegmentCollector {
}
impl TopSegmentCollector {
- fn get_hash(&self, doc: DocId, field1: FastField, field2: FastField) -> Prehashed {
+ fn get_hash(&self, doc: DocId, field1: FastFieldEnum, field2: FastFieldEnum) -> Prehashed {
let field_reader = self.fastfield_segment_reader.get_field_reader(doc);
let hash = [
@@ -169,19 +169,31 @@ impl TopSegmentCollector {
let simhash: Option<u64> = self
.fastfield_segment_reader
.get_field_reader(doc)
- .get(FastField::SimHash)
+ .get(fast_field::SimHash.into())
.unwrap()
.into();
self.bucket_collector.insert(SegmentDoc {
hashes: Hashes {
- site: self.get_hash(doc, FastField::SiteHash1, FastField::SiteHash2),
- title: self.get_hash(doc, FastField::TitleHash1, FastField::TitleHash2),
- url: self.get_hash(doc, FastField::UrlHash1, FastField::UrlHash2),
+ site: self.get_hash(
+ doc,
+ fast_field::SiteHash1.into(),
+ fast_field::SiteHash2.into(),
+ ),
+ title: self.get_hash(
+ doc,
+ fast_field::TitleHash1.into(),
+ fast_field::TitleHash2.into(),
+ ),
+ url: self.get_hash(
+ doc,
+ fast_field::UrlHash1.into(),
+ fast_field::UrlHash2.into(),
+ ),
url_without_tld: self.get_hash(
doc,
- FastField::UrlWithoutTldHash1,
- FastField::UrlWithoutTldHash2,
+ fast_field::UrlWithoutTldHash1.into(),
+ fast_field::UrlWithoutTldHash2.into(),
),
simhash: simhash.unwrap(),
},
diff --git a/crates/core/src/enum_map.rs b/crates/core/src/enum_map.rs
index ed3cf07a..45eeb56f 100644
--- a/crates/core/src/enum_map.rs
+++ b/crates/core/src/enum_map.rs
@@ -16,14 +16,22 @@
use serde::{Deserialize, Serialize};
+pub trait InsertEnumMapKey: Sized {
+ fn into_usize(self) -> usize;
+}
+
+pub trait GetEnumMapKey: Sized {
+ fn from_usize(value: usize) -> Option<Self>;
+}
+
#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct EnumMap<K: Into<usize>, V> {
+pub struct EnumMap<K: InsertEnumMapKey, V> {
inner: Vec<Option<V>>,
len: usize,
_phantom: std::marker::PhantomData<K>,
}
-impl<K: Into<usize>, V> Default for EnumMap<K, V> {
+impl<K: InsertEnumMapKey, V> Default for EnumMap<K, V> {
fn default() -> Self {
Self::new()
}
@@ -31,7 +39,7 @@ impl<K: Into<usize>, V> Default for EnumMap<K, V> {
impl<K, V> EnumMap<K, V>
where
- K: Into<usize>,
+ K: InsertEnumMapKey,
{
pub fn new() -> Self {
Self {
@@ -42,7 +50,7 @@ where
}
pub fn insert(&mut self, key: K, value: V) {
- let key = key.into();
+ let key = key.into_usize();
if key >= self.inner.len() {
self.inner.resize_with(key + 1, || None);
@@ -60,7 +68,7 @@ where
}
pub fn get(&self, key: K) -> Option<&V> {
- let key = key.into();
+ let key = key.into_usize();
if key >= self.inner.len() {
None
} else {
@@ -81,7 +89,7 @@ where
}
pub fn get_mut(&mut self, key: K) -> Option<&mut V> {
- let key = key.into();
+ let key = key.into_usize();
if key >= self.inner.len() {
None
} else {
@@ -92,19 +100,19 @@ where
impl<K, V> EnumMap<K, V>
where
- K: TryFrom<usize> + Into<usize>,
+ K: GetEnumMapKey + InsertEnumMapKey,
{
pub fn keys(&self) -> impl Iterator<Item = K> + '_ {
self.inner
.iter()
.enumerate()
- .filter_map(|(key, value)| value.as_ref().and_then(|_| K::try_from(key).ok()))
+ .filter_map(|(key, value)| value.as_ref().and_then(|_| K::from_usize(key)))
}
}
impl<K, V> FromIterator<(K, V)> for EnumMap<K, V>
where
- K: Into<usize>,
+ K: InsertEnumMapKey,
{
fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {
let mut map = Self::new();
@@ -118,17 +126,17 @@ where
}
#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct EnumSet<K: Into<usize>> {
+pub struct EnumSet<K: InsertEnumMapKey> {
map: EnumMap<K, ()>,
}
-impl<K: Into<usize>> Default for EnumSet<K> {
+impl<K: InsertEnumMapKey> Default for EnumSet<K> {
fn default() -> Self {
Self::new()
}
}
-impl<K: Into<usize>> EnumSet<K> {
+impl<K: InsertEnumMapKey> EnumSet<K> {
pub fn new() -> Self {
Self {
map: EnumMap::new(),
@@ -148,10 +156,7 @@ impl<K: Into<usize>> EnumSet<K> {
}
}
-impl<K> EnumSet<K>
-where
- K: TryFrom<usize> + Into<usize>,
-{
+impl<K: InsertEnumMapKey + GetEnumMapKey> EnumSet<K> {
pub fn iter(&self) -> impl Iterator<Item = K> + '_ {
self.map.keys()
}
@@ -168,9 +173,9 @@ mod tests {
C,
}
- impl From<TestEnum> for usize {
- fn from(val: TestEnum) -> Self {
- val as usize
+ impl InsertEnumMapKey for TestEnum {
+ fn into_usize(self) -> usize {
+ self as usize
}
}
diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/fastfield_reader.rs
index 3098cd12..2380e4fe 100644
--- a/crates/core/src/fastfield_reader.rs
+++ b/crates/core/src/fastfield_reader.rs
@@ -20,7 +20,7 @@ use tantivy::{columnar::ColumnValues, DocId, SegmentId};
use crate::{
enum_map::EnumMap,
- schema::{DataType, FastField, Field},
+ schema::{fast_field::FastField, DataType, FastFieldEnum, Field},
};
#[derive(Default, Clone)]
@@ -53,7 +53,7 @@ impl FastFieldReader {
let mut u64s = EnumMap::new();
let mut bytes = EnumMap::new();
- for field in Field::all().filter_map(Field::as_fast) {
+ for field in Field::all().filter_map(|f| f.as_fast()) {
match field.data_type() {
DataType::U64 => {
if let Ok(reader) = fastfield_readers.u64(field.name()) {
@@ -83,8 +83,8 @@ impl FastFieldReader {
}
struct AllReaders {
- u64s: EnumMap<FastField, tantivy::columnar::Column<u64>>,
- bytes: EnumMap<FastField, tantivy::columnar::BytesColumn>,
+ u64s: EnumMap<FastFieldEnum, tantivy::columnar::Column<u64>>,
+ bytes: EnumMap<FastFieldEnum, tantivy::columnar::BytesColumn>,
}
pub enum Value {
@@ -147,7 +147,7 @@ pub struct FieldReader<'a> {
}
impl<'a> FieldReader<'a> {
- pub fn get(&self, field: FastField) -> Option<Value> {
+ pub fn get(&self, field: FastFieldEnum) -> Option<Value> {
match field.data_type() {
DataType::U64 => Some(
self.readers
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index e026718f..afa3ca00 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -45,7 +45,8 @@ use crate::query::Query;
use crate::ranking::initial::Score;
use crate::ranking::pipeline::RecallRankingWebpage;
use crate::ranking::SignalAggregator;
-use crate::schema::{FastField, Field, TextField};
+use crate::schema::text_field::TextField;
+use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};
use crate::search_ctx::Ctx;
use crate::snippet;
use crate::snippet::TextSnippet;
@@ -177,7 +178,9 @@ impl InvertedIndex {
} else {
let index_settings = tantivy::IndexSettings {
sort_by_field: Some(tantivy::IndexSortByField {
- field: Field::Fast(FastField::PreComputedScore).name().to_string(),
+ field: Field::Fast(FastFieldEnum::from(fast_field::PreComputedScore))
+ .name()
+ .to_string(),
order: tantivy::Order::Desc,
}),
..Default::default()
@@ -316,9 +319,7 @@ impl InvertedIndex {
pub fn delete_all_before(&self, timestamp: tantivy::DateTime) -> Result<()> {
let query = tantivy::query::RangeQuery::new_date_bounds(
- Field::Text(TextField::InsertionTimestamp)
- .name()
- .to_string(),
+ text_field::InsertionTimestamp.name().to_string(),
std::ops::Bound::Unbounded,
std::ops::Bound::Excluded(timestamp),
);
@@ -425,7 +426,7 @@ impl InvertedIndex {
let field = self
.schema()
- .get_field(Field::Fast(FastField::HostNodeID).name())
+ .get_field(Field::Fast(FastFieldEnum::from(fast_field::HostNodeID)).name())
.unwrap();
let id = doc.get_first(field).unwrap().as_u64().unwrap();
@@ -641,7 +642,7 @@ impl InvertedIndex {
let tv_searcher = self.reader.searcher();
let field = tv_searcher
.schema()
- .get_field(Field::Text(TextField::UrlNoTokenizer).name())
+ .get_field(Field::Text(TextFieldEnum::from(text_field::UrlNoTokenizer)).name())
.unwrap();
let term = tantivy::Term::from_field_text(field, url.as_str());
@@ -660,7 +661,9 @@ impl InvertedIndex {
let tv_searcher = self.reader.searcher();
let field = tv_searcher
.schema()
- .get_field(Field::Text(TextField::SiteIfHomepageNoTokenizer).name())
+ .get_field(
+ Field::Text(TextFieldEnum::from(text_field::SiteIfHomepageNoTokenizer)).name(),
+ )
.unwrap();
let host = url.normalized_host().unwrap_or_default();
@@ -707,12 +710,12 @@ impl RetrievedWebpage {
}
}
-fn str_value(field: TextField, value: &tantivy::schema::FieldValue) -> String {
+fn str_value(name: &str, value: &tantivy::schema::FieldValue) -> String {
value
.value()
.as_value()
.as_str()
- .unwrap_or_else(|| panic!("{} field should be text", field.name()))
+ .unwrap_or_else(|| panic!("{} field should be text", name))
.to_string()
}
@@ -721,21 +724,21 @@ impl From<TantivyDocument> for RetrievedWebpage {
let mut webpage = RetrievedWebpage::default();
for value in doc.field_values() {
- match Field::get(value.field.field_id() as usize).copied() {
- Some(Field::Text(TextField::Title)) => {
- webpage.title = str_value(TextField::Title, value);
+ match Field::get(value.field.field_id() as usize) {
+ Some(Field::Text(TextFieldEnum::Title(_))) => {
+ webpage.title = str_value(text_field::Title.name(), value);
}
- Some(Field::Text(TextField::StemmedCleanBody)) => {
- webpage.body = str_value(TextField::StemmedCleanBody, value);
+ Some(Field::Text(TextFieldEnum::StemmedCleanBody(_))) => {
+ webpage.body = str_value(text_field::StemmedCleanBody.name(), value);
}
- Some(Field::Text(TextField::Description)) => {
- let desc = str_value(TextField::Description, value);
+ Some(Field::Text(TextFieldEnum::Description(_))) => {
+ let desc = str_value(text_field::Description.name(), value);
webpage.description = if desc.is_empty() { None } else { Some(desc) }
}
- Some(Field::Text(TextField::Url)) => {
- webpage.url = str_value(TextField::Url, value);
+ Some(Field::Text(TextFieldEnum::Url(_))) => {
+ webpage.url = str_value(text_field::Url.name(), value);
}
- Some(Field::Fast(FastField::LastUpdated)) => {
+ Some(Field::Fast(FastFieldEnum::LastUpdated(_))) => {
webpage.updated_time = {
let timestamp = value.value().as_value().as_u64().unwrap() as i64;
if timestamp == 0 {
@@ -745,39 +748,39 @@ impl From<TantivyDocument> for RetrievedWebpage {
}
}
}
- Some(Field::Text(TextField::AllBody)) => {
- webpage.dirty_body = str_value(TextField::AllBody, value);
+ Some(Field::Text(TextFieldEnum::AllBody(_))) => {
+ webpage.dirty_body = str_value(text_field::AllBody.name(), value);
}
- Some(Field::Fast(FastField::Region)) => {
+ Some(Field::Fast(FastFieldEnum::Region(_))) => {
webpage.region = {
let id = value.value().as_value().as_u64().unwrap();
Region::from_id(id)
}
}
- Some(Field::Text(TextField::DmozDescription)) => {
- let desc = str_value(TextField::DmozDescription, value);
+ Some(Field::Text(TextFieldEnum::DmozDescription(_))) => {
+ let desc = str_value(text_field::DmozDescription.name(), value);
webpage.dmoz_description = if desc.is_empty() { None } else { Some(desc) }
}
- Some(Field::Text(TextField::SchemaOrgJson)) => {
- let json = str_value(TextField::SchemaOrgJson, value);
+ Some(Field::Text(TextFieldEnum::SchemaOrgJson(_))) => {
+ let json = str_value(text_field::SchemaOrgJson.name(), value);
webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();
}
- Some(Field::Fast(FastField::LikelyHasAds)) => {
+ Some(Field::Fast(FastFieldEnum::LikelyHasAds(_))) => {
webpage.likely_has_ads =
value.value().as_value().as_u64().unwrap_or_default() != 0;
}
- Some(Field::Fast(FastField::LikelyHasPaywall)) => {
+ Some(Field::Fast(FastFieldEnum::LikelyHasPaywall(_))) => {
webpage.likely_has_paywall =
value.value().as_value().as_u64().unwrap_or_default() != 0;
}
- Some(Field::Text(TextField::RecipeFirstIngredientTagId)) => {
- let tag_id = str_value(TextField::RecipeFirstIngredientTagId, value);
+ Some(Field::Text(TextFieldEnum::RecipeFirstIngredientTagId(_))) => {
+ let tag_id = str_value(text_field::RecipeFirstIngredientTagId.name(), value);
if !tag_id.is_empty() {
webpage.recipe_first_ingredient_tag_id = Some(tag_id);
}
}
- Some(Field::Text(TextField::Keywords)) => {
- let keywords = str_value(TextField::Keywords, value);
+ Some(Field::Text(TextFieldEnum::Keywords(_))) => {
+ let keywords = str_value(text_field::Keywords.name(), value);
webpage.keywords = keywords.split('\n').map(|s| s.to_string()).collect();
}
_ => {}
diff --git a/crates/core/src/mapreduce/dht/mod.rs b/crates/core/src/mapreduce/dht/mod.rs
index 08b8af32..6667c906 100644
--- a/crates/core/src/mapreduce/dht/mod.rs
+++ b/crates/core/src/mapreduce/dht/mod.rs
@@ -73,18 +73,18 @@ macro_rules! raft_sonic_request_response {
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub enum Response {
$(
- $req(<$req as crate::distributed::sonic::service::Message<$service>>::Response),
+ $req(<$req as $crate::distributed::sonic::service::Message<$service>>::Response),
)*
Empty,
}
$(
- impl TryFrom<Response> for <$req as crate::distributed::sonic::service::Message<$service>>::Response {
- type Error = crate::distributed::sonic::Error;
+ impl TryFrom<Response> for <$req as $crate::distributed::sonic::service::Message<$service>>::Response {
+ type Error = $crate::distributed::sonic::Error;
fn try_from(res: Response) -> Result<Self, Self::Error> {
match res {
Response::$req(res) => Ok(res),
- _ => Err(crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),
+ _ => Err($crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),
}
}
}
diff --git a/crates/core/src/mapreduce/dht/network/mod.rs b/crates/core/src/mapreduce/dht/network/mod.rs
index b6efc5a1..e3c1d01e 100644
--- a/crates/core/src/mapreduce/dht/network/mod.rs
+++ b/crates/core/src/mapreduce/dht/network/mod.rs
@@ -39,34 +39,34 @@ impl RaftNetworkFactory<TypeConfig> for Network {
}
}
-pub type AppendEntriesRequest = openraft::raft::AppendEntriesRequest<TypeConfig>;
+pub type AppendEntries = openraft::raft::AppendEntriesRequest<TypeConfig>;
pub type AppendEntriesResponse = openraft::raft::AppendEntriesResponse<NodeId>;
-pub type InstallSnapshotRequest = openraft::raft::InstallSnapshotRequest<TypeConfig>;
+pub type InstallSnapshot = openraft::raft::InstallSnapshotRequest<TypeConfig>;
pub type InstallSnapshotResponse = openraft::raft::InstallSnapshotResponse<NodeId>;
-pub type VoteRequest = openraft::raft::VoteRequest<NodeId>;
+pub type Vote = openraft::raft::VoteRequest<NodeId>;
pub type VoteResponse = openraft::raft::VoteResponse<NodeId>;
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-pub struct AddLearnerRequest {
+pub struct AddLearner {
pub id: NodeId,
pub addr: SocketAddr,
}
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-pub struct AddNodesRequest {
+pub struct AddNodes {
members: BTreeMap<NodeId, BasicNode>,
}
sonic_service!(
Server,
[
- AppendEntriesRequest,
- InstallSnapshotRequest,
- VoteRequest,
- AddLearnerRequest,
- AddNodesRequest,
+ AppendEntries,
+ InstallSnapshot,
+ Vote,
+ AddLearner,
+ AddNodes,
Get,
Set,
]
diff --git a/crates/core/src/mapreduce/dht/network/raft.rs b/crates/core/src/mapreduce/dht/network/raft.rs
index bfac4b56..0e1297f4 100644
--- a/crates/core/src/mapreduce/dht/network/raft.rs
+++ b/crates/core/src/mapreduce/dht/network/raft.rs
@@ -37,11 +37,11 @@ use crate::{
};
use super::{
- AddLearnerRequest, AddNodesRequest, AppendEntriesRequest, AppendEntriesResponse,
- InstallSnapshotRequest, InstallSnapshotResponse, Server, VoteRequest, VoteResponse,
+ AddLearner, AddNodes, AppendEntries, AppendEntriesResponse, InstallSnapshot,
+ InstallSnapshotResponse, Server, Vote, VoteResponse,
};
-impl sonic::service::Message<Server> for AppendEntriesRequest {
+impl sonic::service::Message<Server> for AppendEntries {
type Response = Result<AppendEntriesResponse, RaftError<NodeId>>;
async fn handle(self, server: &Server) -> Self::Response {
@@ -50,7 +50,7 @@ impl sonic::service::Message<Server> for AppendEntriesRequest {
}
}
-impl sonic::service::Message<Server> for InstallSnapshotRequest {
+impl sonic::service::Message<Server> for InstallSnapshot {
type Response = Result<InstallSnapshotResponse, RaftError<NodeId, InstallSnapshotError>>;
async fn handle(self, server: &Server) -> Self::Response {
@@ -59,7 +59,7 @@ impl sonic::service::Message<Server> for InstallSnapshotRequest {
}
}
-impl sonic::service::Message<Server> for VoteRequest {
+impl sonic::service::Message<Server> for Vote {
type Response = Result<VoteResponse, RaftError<NodeId>>;
async fn handle(self, server: &Server) -> Self::Response {
@@ -68,7 +68,7 @@ impl sonic::service::Message<Server> for VoteRequest {
}
}
-impl sonic::service::Message<Server> for AddLearnerRequest {
+impl sonic::service::Message<Server> for AddLearner {
type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
async fn handle(self, server: &Server) -> Self::Response {
@@ -93,7 +93,7 @@ impl sonic::service::Message<Server> for AddLearnerRequest {
}
}
-impl sonic::service::Message<Server> for AddNodesRequest {
+impl sonic::service::Message<Server> for AddNodes {
type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;
async fn handle(self, server: &Server) -> Self::Response {
@@ -160,7 +160,7 @@ impl RemoteClient {
}
async fn add_learner(&self, id: NodeId, addr: SocketAddr) -> Result<()> {
- let rpc = AddLearnerRequest { id, addr };
+ let rpc = AddLearner { id, addr };
let retry = ExponentialBackoff::from_millis(500)
.with_limit(Duration::from_secs(60))
.take(5);
@@ -220,7 +220,7 @@ impl RemoteClient {
}
async fn add_nodes(&self, members: BTreeMap<NodeId, BasicNode>) -> Result<()> {
- let rpc = AddNodesRequest { members };
+ let rpc = AddNodes { members };
let retry = ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(10));
for backoff in retry {
@@ -293,7 +293,7 @@ impl RemoteClient {
impl RaftNetwork<TypeConfig> for RemoteClient {
async fn append_entries(
&mut self,
- rpc: AppendEntriesRequest,
+ rpc: AppendEntries,
option: RPCOption,
) -> Result<AppendEntriesResponse, RPCError> {
self.send_raft_rpc(rpc, option).await?.map_err(|e| {
@@ -308,7 +308,7 @@ impl RaftNetwork<TypeConfig> for RemoteClient {
async fn install_snapshot(
&mut self,
- rpc: InstallSnapshotRequest,
+ rpc: InstallSnapshot,
option: RPCOption,
) -> Result<InstallSnapshotResponse, RPCError<InstallSnapshotError>> {
self.send_raft_rpc(rpc, option).await?.map_err(|e| {
@@ -321,11 +321,7 @@ impl RaftNetwork<TypeConfig> for RemoteClient {
})
}
- async fn vote(
- &mut self,
- rpc: VoteRequest,
- option: RPCOption,
- ) -> Result<VoteResponse, RPCError> {
+ async fn vote(&mut self, rpc: Vote, option: RPCOption) -> Result<VoteResponse, RPCError> {
self.send_raft_rpc(rpc, option).await?.map_err(|e| {
openraft::error::RemoteError {
target: self.target,
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index abd71c50..5a9bba0a 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -18,7 +18,7 @@ use crate::{
inverted_index::InvertedIndex,
query::parser::TermCompound,
ranking::SignalCoefficient,
- schema::{Field, TextField},
+ schema::{text_field, Field},
search_ctx::Ctx,
searcher::SearchQuery,
webpage::{region::Region, safety_classifier},
@@ -116,7 +116,7 @@ impl Query {
.collect();
if query.safe_search {
- let field = Field::Text(TextField::SafetyClassification);
+ let field = Field::Text(text_field::SafetyClassification.into());
let field = schema.get_field(field.name()).unwrap();
queries.push((
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 040cf05f..d52266fc 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -22,7 +22,7 @@ use tantivy::{
schema::Schema,
};
-use crate::{fastfield_reader::FastFieldReader, schema::TextField};
+use crate::{fastfield_reader::FastFieldReader, schema::text_field};
use super::{const_query::ConstQuery, pattern_query::PatternQuery, union::UnionQuery};
@@ -173,7 +173,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Site => ConstQuery::new(
PatternQuery::new(
self.pattern.clone(),
- TextField::UrlForSiteOperator,
+ text_field::UrlForSiteOperator.into(),
schema,
fastfield_reader.clone(),
)
@@ -184,7 +184,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Url => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::Url,
+ text_field::Url.into(),
schema,
fastfield_reader.clone(),
)),
@@ -193,7 +193,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Domain => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::Domain,
+ text_field::Domain.into(),
schema,
fastfield_reader.clone(),
)),
@@ -202,7 +202,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Title => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::Title,
+ text_field::Title.into(),
schema,
fastfield_reader.clone(),
)),
@@ -212,7 +212,7 @@ impl AsTantivyQuery for Matching {
Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::Description,
+ text_field::Description.into(),
schema,
fastfield_reader.clone(),
)),
@@ -221,7 +221,7 @@ impl AsTantivyQuery for Matching {
Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::DmozDescription,
+ text_field::DmozDescription.into(),
schema,
fastfield_reader.clone(),
)),
@@ -232,7 +232,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Content => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::CleanBody,
+ text_field::CleanBody.into(),
schema,
fastfield_reader.clone(),
)),
@@ -241,7 +241,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::MicroformatTag => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::MicroformatTags,
+ text_field::MicroformatTags.into(),
schema,
fastfield_reader.clone(),
)),
@@ -250,7 +250,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Schema => Box::new(ConstQuery::new(
Box::new(PatternQuery::new(
self.pattern.clone(),
- TextField::FlattenedSchemaOrgJson,
+ text_field::FlattenedSchemaOrgJson.into(),
schema,
fastfield_reader.clone(),
)),
diff --git a/crates/core/src/query/parser/as_tantivy.rs b/crates/core/src/query/parser/as_tantivy.rs
index efa90e91..9de8edc1 100644
--- a/crates/core/src/query/parser/as_tantivy.rs
+++ b/crates/core/src/query/parser/as_tantivy.rs
@@ -21,7 +21,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIXES,
- schema::{Field, TextField},
+ schema::{text_field::TextField, Field, TextFieldEnum},
};
use super::{CompoundAwareTerm, SimpleOrPhrase, SimpleTerm, Term, TermCompound};
@@ -53,9 +53,9 @@ fn simple_into_tantivy(
.filter(|field| {
matches!(
Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::AllBody))
- | Some(Field::Text(TextField::Title))
- | Some(Field::Text(TextField::Url))
+ Some(Field::Text(TextFieldEnum::AllBody(_)))
+ | Some(Field::Text(TextFieldEnum::Title(_)))
+ | Some(Field::Text(TextFieldEnum::Url(_)))
)
})
.copied()
@@ -98,7 +98,7 @@ fn phrase_query(
}
if processed_terms.len() == 1 {
- let options = field.as_text().unwrap().index_option();
+ let options = field.as_text().unwrap().record_option();
phrases.push((
Occur::Should,
@@ -140,7 +140,7 @@ impl Term {
.find(|field| {
matches!(
Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::Title))
+ Some(Field::Text(TextFieldEnum::Title(_)))
)
})
.unwrap();
@@ -156,7 +156,7 @@ impl Term {
.find(|field| {
matches!(
Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::AllBody))
+ Some(Field::Text(TextFieldEnum::AllBody(_)))
)
})
.unwrap();
@@ -172,7 +172,7 @@ impl Term {
.find(|field| {
matches!(
Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::Url))
+ Some(Field::Text(TextFieldEnum::Url(_)))
)
})
.unwrap();
@@ -222,7 +222,7 @@ impl Term {
.filter(|field| {
matches!(
Field::get(field.field_id() as usize),
- Some(Field::Text(TextField::UrlForSiteOperator))
+ Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_)))
)
})
.map(|field| {
@@ -258,7 +258,7 @@ impl Term {
.unwrap()
.as_text()
.unwrap()
- .index_option();
+ .record_option();
let processed_query = if processed_terms.len() == 1 {
let term = processed_terms.remove(0);
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index 8968ea94..0e696cbc 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -93,8 +93,9 @@ fn single_bang(input: &str, pref: char) -> nom::IResult<&str, Term> {
fn bang(input: &str) -> nom::IResult<&str, Term> {
for pref in BANG_PREFIXES.iter() {
- let (input, output) = single_bang(input, *pref)?;
- return Ok((input, output));
+ if let Ok((input, output)) = single_bang(input, *pref) {
+ return Ok((input, output));
+ }
}
Err(nom::Err::Error(nom::error::Error::new(
diff --git a/crates/core/src/query/pattern_query/mod.rs b/crates/core/src/query/pattern_query/mod.rs
index 0057690f..6d9d1f7a 100644
--- a/crates/core/src/query/pattern_query/mod.rs
+++ b/crates/core/src/query/pattern_query/mod.rs
@@ -23,7 +23,7 @@ use tantivy::tokenizer::Tokenizer;
use crate::{
fastfield_reader::FastFieldReader,
- schema::{Field, TextField},
+ schema::{text_field::TextField, Field, TextFieldEnum},
};
use self::weight::{FastSiteDomainPatternWeight, PatternWeight};
@@ -50,7 +50,7 @@ impl std::fmt::Debug for PatternQuery {
impl PatternQuery {
pub fn new(
patterns: Vec<PatternPart>,
- field: TextField,
+ field: TextFieldEnum,
schema: &tantivy::schema::Schema,
fastfield_reader: FastFieldReader,
) -> Self {
@@ -165,6 +165,6 @@ fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
&& patterns[1..patterns.len() - 1]
.iter()
.all(|pattern| matches!(pattern, PatternPart::Raw(_)))
- && (matches!(field, Field::Text(TextField::UrlForSiteOperator))
- || matches!(field, Field::Text(TextField::Domain)))
+ && (matches!(field, Field::Text(TextFieldEnum::UrlForSiteOperator(_)))
+ || matches!(field, Field::Text(TextFieldEnum::Domain(_))))
}
diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rs
index 3efafe66..8fe0b55f 100644
--- a/crates/core/src/query/pattern_query/scorer.rs
+++ b/crates/core/src/query/pattern_query/scorer.rs
@@ -24,7 +24,7 @@ use tantivy::{
use crate::{
fastfield_reader::{self, FastFieldReader},
query::intersection::Intersection,
- schema::FastField,
+ schema::FastFieldEnum,
};
use super::SmallPatternPart;
@@ -126,7 +126,7 @@ impl Scorer for AllScorer {
pub struct EmptyFieldScorer {
pub segment_reader: Arc<fastfield_reader::SegmentReader>,
- pub num_tokens_fastfield: FastField,
+ pub num_tokens_fastfield: FastFieldEnum,
pub all_scorer: AllScorer,
}
@@ -210,7 +210,7 @@ pub struct NormalPatternScorer {
left: Vec<u32>,
right: Vec<u32>,
phrase_count: u32,
- num_tokens_field: FastField,
+ num_tokens_field: FastFieldEnum,
segment_reader: Arc<fastfield_reader::SegmentReader>,
}
@@ -219,7 +219,7 @@ impl NormalPatternScorer {
term_postings_list: Vec<SegmentPostings>,
pattern: Vec<SmallPatternPart>,
segment: tantivy::SegmentId,
- num_tokens_field: FastField,
+ num_tokens_field: FastFieldEnum,
fastfield_reader: FastFieldReader,
) -> Self {
let num_query_terms = term_postings_list.len();
diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rs
index 249c723f..50bbb2f7 100644
--- a/crates/core/src/query/pattern_query/weight.rs
+++ b/crates/core/src/query/pattern_query/weight.rs
@@ -24,7 +24,11 @@ use tantivy::{
use crate::{
fastfield_reader::FastFieldReader,
- schema::{FastField, Field, TextField},
+ schema::{
+ fast_field,
+ text_field::{self, TextField},
+ Field, TextFieldEnum,
+ },
};
use super::scorer::{
@@ -52,10 +56,12 @@ impl FastSiteDomainPatternWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let field_no_tokenizer = match Field::get(self.field.field_id() as usize) {
- Some(Field::Text(TextField::UrlForSiteOperator)) => {
- Field::Text(TextField::SiteNoTokenizer)
+ Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {
+ Field::Text(text_field::SiteNoTokenizer.into())
+ }
+ Some(Field::Text(TextFieldEnum::Domain(_))) => {
+ Field::Text(text_field::DomainNoTokenizer.into())
}
- Some(Field::Text(TextField::Domain)) => Field::Text(TextField::DomainNoTokenizer),
_ => unreachable!(),
};
@@ -65,7 +71,7 @@ impl FastSiteDomainPatternWeight {
.unwrap();
let opt = match field_no_tokenizer {
- Field::Text(t) => t.index_option(),
+ Field::Text(t) => t.record_option(),
Field::Fast(_) => unreachable!(),
};
@@ -134,19 +140,23 @@ impl PatternWeight {
}
let num_tokens_fastfield = match Field::get(self.field.field_id() as usize) {
- Some(Field::Text(TextField::Title)) => Ok(FastField::NumTitleTokens),
- Some(Field::Text(TextField::CleanBody)) => Ok(FastField::NumCleanBodyTokens),
- Some(Field::Text(TextField::Url)) => Ok(FastField::NumUrlTokens),
- Some(Field::Text(TextField::Domain)) => Ok(FastField::NumDomainTokens),
- Some(Field::Text(TextField::UrlForSiteOperator)) => {
- Ok(FastField::NumUrlForSiteOperatorTokens)
+ Some(Field::Text(TextFieldEnum::Title(_))) => Ok(fast_field::NumTitleTokens.into()),
+ Some(Field::Text(TextFieldEnum::CleanBody(_))) => {
+ Ok(fast_field::NumCleanBodyTokens.into())
+ }
+ Some(Field::Text(TextFieldEnum::Url(_))) => Ok(fast_field::NumUrlTokens.into()),
+ Some(Field::Text(TextFieldEnum::Domain(_))) => Ok(fast_field::NumDomainTokens.into()),
+ Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {
+ Ok(fast_field::NumUrlForSiteOperatorTokens.into())
+ }
+ Some(Field::Text(TextFieldEnum::Description(_))) => {
+ Ok(fast_field::NumDescriptionTokens.into())
}
- Some(Field::Text(TextField::Description)) => Ok(FastField::NumDescriptionTokens),
- Some(Field::Text(TextField::MicroformatTags)) => {
- Ok(FastField::NumMicroformatTagsTokens)
+ Some(Field::Text(TextFieldEnum::MicroformatTags(_))) => {
+ Ok(fast_field::NumMicroformatTagsTokens.into())
}
- Some(Field::Text(TextField::FlattenedSchemaOrgJson)) => {
- Ok(FastField::NumFlattenedSchemaTokens)
+ Some(Field::Text(TextFieldEnum::FlattenedSchemaOrgJson(_))) => {
+ Ok(fast_field::NumFlattenedSchemaTokens.into())
}
Some(field) => Err(TantivyError::InvalidArgument(format!(
"{} is not supported in pattern query",
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index 56a9f93b..4267b779 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -30,7 +30,7 @@ use crate::{
pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},
Signal, SignalAggregator, SignalScore,
},
- schema::FastField,
+ schema::fast_field,
searcher::SearchQuery,
};
@@ -62,11 +62,11 @@ impl RecallRankingWebpage {
let fastfields = fastfield_reader.get_field_reader(pointer.address.doc_id);
let title_embedding: Option<Vec<u8>> = fastfields
- .get(FastField::TitleEmbeddings)
+ .get(fast_field::TitleEmbeddings.into())
.and_then(|v| v.into());
let keyword_embedding: Option<Vec<u8>> = fastfields
- .get(FastField::KeywordEmbeddings)
+ .get(fast_field::KeywordEmbeddings.into())
.and_then(|v| v.into());
let mut res = RecallRankingWebpage {
diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rs
index 0900d39c..61ed32ef 100644
--- a/crates/core/src/ranking/signal.rs
+++ b/crates/core/src/ranking/signal.rs
@@ -14,13 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::enum_map::InsertEnumMapKey;
use crate::query::optic::AsSearchableRule;
use crate::query::Query;
+use crate::schema::text_field::TextField;
+use crate::schema::{fast_field, text_field};
use crate::Result;
use crate::{
enum_map::EnumMap,
fastfield_reader,
- schema::{FastField, TextField},
+ schema::{FastFieldEnum, TextFieldEnum},
webgraph::NodeID,
webpage::Webpage,
};
@@ -31,10 +34,11 @@ use serde::{Deserialize, Serialize};
use std::cell::RefCell;
use std::str::FromStr;
use std::sync::Arc;
+use strum::VariantArray;
use tantivy::fieldnorm::FieldNormReader;
use tantivy::postings::SegmentPostings;
use tantivy::query::{Query as _, Scorer};
-use tantivy::tokenizer::Tokenizer;
+use tantivy::tokenizer::Tokenizer as _;
use thiserror::Error;
use utoipa::ToSchema;
@@ -56,7 +60,9 @@ pub enum Error {
UnknownSignal(#[from] serde_json::Error),
}
-#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(
+ Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, VariantArray,
+)]
pub enum Signal {
#[serde(rename = "bm25_title")]
Bm25Title,
@@ -140,55 +146,12 @@ pub enum Signal {
KeywordEmbeddingSimilarity,
}
-impl From<Signal> for usize {
- fn from(signal: Signal) -> Self {
- signal as usize
+impl InsertEnumMapKey for Signal {
+ fn into_usize(self) -> usize {
+ self as usize
}
}
-pub const ALL_SIGNALS: [Signal; 40] = [
- Signal::Bm25Title,
- Signal::Bm25TitleBigrams,
- Signal::Bm25TitleTrigrams,
- Signal::Bm25CleanBody,
- Signal::Bm25CleanBodyBigrams,
- Signal::Bm25CleanBodyTrigrams,
- Signal::Bm25StemmedTitle,
- Signal::Bm25StemmedCleanBody,
- Signal::Bm25AllBody,
- Signal::Bm25BacklinkText,
- Signal::Bm25Keywords,
- Signal::IdfSumUrl,
- Signal::IdfSumSite,
- Signal::IdfSumDomain,
- Signal::IdfSumSiteNoTokenizer,
- Signal::IdfSumDomainNoTokenizer,
- Signal::IdfSumDomainNameNoTokenizer,
- Signal::IdfSumDomainIfHomepage,
- Signal::IdfSumDomainNameIfHomepageNoTokenizer,
- Signal::IdfSumDomainIfHomepageNoTokenizer,
- Signal::IdfSumTitleIfHomepage,
- Signal::CrossEncoderSnippet,
- Signal::CrossEncoderTitle,
- Signal::HostCentrality,
- Signal::HostCentralityRank,
- Signal::PageCentrality,
- Signal::PageCentralityRank,
- Signal::IsHomepage,
- Signal::FetchTimeMs,
- Signal::UpdateTimestamp,
- Signal::TrackerScore,
- Signal::Region,
- Signal::QueryCentrality,
- Signal::InboundSimilarity,
- Signal::LambdaMART,
- Signal::UrlDigits,
- Signal::UrlSlashes,
- Signal::LinkDensity,
- Signal::TitleEmbeddingSimilarity,
- Signal::KeywordEmbeddingSimilarity,
-];
-
fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {
if timestamp >= signal_aggregator.current_timestamp.unwrap_or(0) {
return 0.0;
@@ -359,7 +322,7 @@ impl Signal {
let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);
let node_id = fastfield_reader
- .get(FastField::HostNodeID)
+ .get(fast_field::HostNodeID.into())
.and_then(|n| n.as_u64())
.unwrap();
@@ -610,56 +573,60 @@ impl Signal {
})
}
- fn as_fastfield(&self) -> Option<FastField> {
+ fn as_fastfield(&self) -> Option<FastFieldEnum> {
match self {
- Signal::HostCentrality => Some(FastField::HostCentrality),
- Signal::HostCentralityRank => Some(FastField::HostCentralityRank),
- Signal::PageCentrality => Some(FastField::PageCentrality),
- Signal::PageCentralityRank => Some(FastField::PageCentralityRank),
- Signal::IsHomepage => Some(FastField::IsHomepage),
- Signal::FetchTimeMs => Some(FastField::FetchTimeMs),
- Signal::UpdateTimestamp => Some(FastField::LastUpdated),
- Signal::TrackerScore => Some(FastField::TrackerScore),
- Signal::Region => Some(FastField::Region),
- Signal::UrlSlashes => Some(FastField::NumPathAndQuerySlashes),
- Signal::UrlDigits => Some(FastField::NumPathAndQueryDigits),
- Signal::LinkDensity => Some(FastField::LinkDensity),
- Signal::TitleEmbeddingSimilarity => Some(FastField::TitleEmbeddings),
- Signal::KeywordEmbeddingSimilarity => Some(FastField::KeywordEmbeddings),
+ Signal::HostCentrality => Some(fast_field::HostCentrality.into()),
+ Signal::HostCentralityRank => Some(fast_field::HostCentralityRank.into()),
+ Signal::PageCentrality => Some(fast_field::PageCentrality.into()),
+ Signal::PageCentralityRank => Some(fast_field::PageCentralityRank.into()),
+ Signal::IsHomepage => Some(fast_field::IsHomepage.into()),
+ Signal::FetchTimeMs => Some(fast_field::FetchTimeMs.into()),
+ Signal::UpdateTimestamp => Some(fast_field::LastUpdated.into()),
+ Signal::TrackerScore => Some(fast_field::TrackerScore.into()),
+ Signal::Region => Some(fast_field::Region.into()),
+ Signal::UrlSlashes => Some(fast_field::NumPathAndQuerySlashes.into()),
+ Signal::UrlDigits => Some(fast_field::NumPathAndQueryDigits.into()),
+ Signal::LinkDensity => Some(fast_field::LinkDensity.into()),
+ Signal::TitleEmbeddingSimilarity => Some(fast_field::TitleEmbeddings.into()),
+ Signal::KeywordEmbeddingSimilarity => Some(fast_field::KeywordEmbeddings.into()),
_ => None,
}
}
- fn as_textfield(&self) -> Option<TextField> {
+ fn as_textfield(&self) -> Option<TextFieldEnum> {
match self {
- Signal::Bm25Title => Some(TextField::Title),
- Signal::Bm25TitleBigrams => Some(TextField::TitleBigrams),
- Signal::Bm25TitleTrigrams => Some(TextField::TitleTrigrams),
- Signal::Bm25CleanBody => Some(TextField::CleanBody),
- Signal::Bm25CleanBodyBigrams => Some(TextField::CleanBodyBigrams),
- Signal::Bm25CleanBodyTrigrams => Some(TextField::CleanBodyTrigrams),
- Signal::Bm25StemmedTitle => Some(TextField::StemmedTitle),
- Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),
- Signal::Bm25AllBody => Some(TextField::AllBody),
- Signal::Bm25BacklinkText => Some(TextField::BacklinkText),
- Signal::Bm25Keywords => Some(TextField::Keywords),
- Signal::IdfSumUrl => Some(TextField::Url),
- Signal::IdfSumSite => Some(TextField::SiteWithout),
- Signal::IdfSumDomain => Some(TextField::Domain),
- Signal::IdfSumSiteNoTokenizer => Some(TextField::SiteNoTokenizer),
- Signal::IdfSumDomainNoTokenizer => Some(TextField::DomainNoTokenizer),
- Signal::IdfSumDomainNameNoTokenizer => Some(TextField::DomainNameNoTokenizer),
- Signal::IdfSumDomainIfHomepage => Some(TextField::DomainIfHomepage),
+ Signal::Bm25Title => Some(text_field::Title.into()),
+ Signal::Bm25TitleBigrams => Some(text_field::TitleBigrams.into()),
+ Signal::Bm25TitleTrigrams => Some(text_field::TitleTrigrams.into()),
+ Signal::Bm25CleanBody => Some(text_field::CleanBody.into()),
+ Signal::Bm25CleanBodyBigrams => Some(text_field::CleanBodyBigrams.into()),
+ Signal::Bm25CleanBodyTrigrams => Some(text_field::CleanBodyTrigrams.into()),
+ Signal::Bm25StemmedTitle => Some(text_field::StemmedTitle.into()),
+ Signal::Bm25StemmedCleanBody => Some(text_field::StemmedCleanBody.into()),
+ Signal::Bm25AllBody => Some(text_field::AllBody.into()),
+ Signal::Bm25BacklinkText => Some(text_field::BacklinkText.into()),
+ Signal::Bm25Keywords => Some(text_field::Keywords.into()),
+ Signal::IdfSumUrl => Some(text_field::Url.into()),
+ Signal::IdfSumSite => Some(text_field::SiteWithout.into()),
+ Signal::IdfSumDomain => Some(text_field::Domain.into()),
+ Signal::IdfSumSiteNoTokenizer => Some(text_field::SiteNoTokenizer.into()),
+ Signal::IdfSumDomainNoTokenizer => Some(text_field::DomainNoTokenizer.into()),
+ Signal::IdfSumDomainNameNoTokenizer => Some(text_field::DomainNameNoTokenizer.into()),
+ Signal::IdfSumDomainIfHomepage => Some(text_field::DomainIfHomepage.into()),
Signal::IdfSumDomainNameIfHomepageNoTokenizer => {
- Some(TextField::DomainNameIfHomepageNoTokenizer)
+ Some(text_field::DomainNameIfHomepageNoTokenizer.into())
}
- Signal::IdfSumTitleIfHomepage => Some(TextField::TitleIfHomepage),
+ Signal::IdfSumTitleIfHomepage => Some(text_field::TitleIfHomepage.into()),
Signal::IdfSumDomainIfHomepageNoTokenizer => {
- Some(TextField::DomainIfHomepageNoTokenizer)
+ Some(text_field::DomainIfHomepageNoTokenizer.into())
}
_ => None,
}
}
+
+ pub fn all() -> impl Iterator<Item = Self> {
+ Self::VARIANTS.iter().copied()
+ }
}
impl FromStr for Signal {
@@ -706,7 +673,7 @@ impl SignalCoefficient {
}
pub fn merge_into(&mut self, coeffs: SignalCoefficient) {
- for signal in ALL_SIGNALS {
+ for signal in Signal::all() {
if let Some(coeff) = coeffs.map.get(signal).copied() {
match self.map.get_mut(signal) {
Some(existing_coeff) => *existing_coeff += coeff,
@@ -736,7 +703,7 @@ struct OpticBoosts {
}
struct SegmentReader {
- text_fields: EnumMap<TextField, TextFieldData>,
+ text_fields: EnumMap<TextFieldEnum, TextFieldData>,
optic_boosts: OpticBoosts,
fastfield_reader: Arc<fastfield_reader::SegmentReader>,
}
@@ -856,13 +823,13 @@ impl SignalAggregator {
&self,
tv_searcher: &tantivy::Searcher,
segment_reader: &tantivy::SegmentReader,
- ) -> Result<EnumMap<TextField, TextFieldData>> {
+ ) -> Result<EnumMap<TextFieldEnum, TextFieldData>> {
let mut text_fields = EnumMap::new();
let schema = tv_searcher.schema();
if let Some(query) = &self.query_data {
if !query.simple_terms.is_empty() {
- for signal in ALL_SIGNALS {
+ for signal in Signal::all() {
if let Some(text_field) = signal.as_textfield() {
let tv_field = schema.get_field(text_field.name()).unwrap();
let simple_query = itertools::intersperse(
@@ -891,7 +858,7 @@ impl SignalAggregator {
let mut postings = Vec::with_capacity(terms.len());
for term in &terms {
if let Some(p) =
- inverted_index.read_postings(term, text_field.index_option())?
+ inverted_index.read_postings(term, text_field.record_option())?
{
postings.push(p);
matching_terms.push(term.clone());
@@ -1043,8 +1010,7 @@ impl SignalAggregator {
}
pub fn precompute_score(&self, webpage: &Webpage) -> f64 {
- ALL_SIGNALS
- .into_iter()
+ Signal::all()
.filter_map(|signal| signal.precompute(self, webpage))
.map(|computed| computed.score.coefficient * computed.score.value)
.sum()
@@ -1078,7 +1044,7 @@ pub struct SignalScore {
#[derive(Clone)]
pub struct SignalOrder {
- text_signals: EnumMap<TextField, NGramSignalOrder>,
+ text_signals: EnumMap<TextFieldEnum, NGramSignalOrder>,
other_signals: Vec<Signal>,
}
@@ -1094,7 +1060,7 @@ impl SignalOrder {
let mut text_signals = EnumMap::new();
let mut other_signals = Vec::new();
- for signal in ALL_SIGNALS {
+ for signal in Signal::all() {
if signal_aggregator.coefficient(&signal) == 0.0 {
continue;
}
diff --git a/crates/core/src/schema.rs b/crates/core/src/schema.rs
deleted file mode 100644
index c4c4228b..00000000
--- a/crates/core/src/schema.rs
+++ /dev/null
@@ -1,850 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use tantivy::{
- schema::{BytesOptions, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions},
- DateOptions,
-};
-
-use crate::tokenizer::{
- BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,
-};
-
-pub const FLOAT_SCALING: u64 = 1_000_000_000;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum TextField {
- Title,
- CleanBody,
- StemmedTitle,
- StemmedCleanBody,
- AllBody,
- Url,
- UrlNoTokenizer,
- UrlForSiteOperator,
- SiteWithout,
- Domain,
- SiteNoTokenizer,
- DomainNoTokenizer,
- DomainNameNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
- SiteIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
- DomainIfHomepage,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
- DomainNameIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
- DomainIfHomepageNoTokenizer,
- /// this field is only set if the webpage is the homepage for the site. Allows us to boost
- TitleIfHomepage,
- BacklinkText,
- Description,
- DmozDescription,
- SchemaOrgJson,
- FlattenedSchemaOrgJson,
- CleanBodyBigrams,
- TitleBigrams,
- CleanBodyTrigrams,
- TitleTrigrams,
- MicroformatTags,
- /// can either be NSFW or SFW (see safety classifier)
- SafetyClassification,
- InsertionTimestamp,
- RecipeFirstIngredientTagId,
- Keywords,
-}
-
-impl From<TextField> for usize {
- fn from(value: TextField) -> Self {
- value as usize
- }
-}
-
-impl TextField {
- pub fn ngram_size(&self) -> usize {
- match self {
- TextField::Title => 1,
- TextField::CleanBody => 1,
- TextField::StemmedTitle => 1,
- TextField::StemmedCleanBody => 1,
- TextField::AllBody => 1,
- TextField::Url => 1,
- TextField::UrlNoTokenizer => 1,
- TextField::UrlForSiteOperator => 1,
- TextField::SiteWithout => 1,
- TextField::Domain => 1,
- TextField::SiteNoTokenizer => 1,
- TextField::DomainNoTokenizer => 1,
- TextField::DomainNameNoTokenizer => 1,
- TextField::SiteIfHomepageNoTokenizer => 1,
- TextField::DomainIfHomepage => 1,
- TextField::DomainNameIfHomepageNoTokenizer => 1,
- TextField::DomainIfHomepageNoTokenizer => 1,
- TextField::TitleIfHomepage => 1,
- TextField::BacklinkText => 1,
- TextField::Description => 1,
- TextField::DmozDescription => 1,
- TextField::SchemaOrgJson => 1,
- TextField::FlattenedSchemaOrgJson => 1,
- TextField::CleanBodyBigrams => 2,
- TextField::TitleBigrams => 2,
- TextField::CleanBodyTrigrams => 3,
- TextField::TitleTrigrams => 3,
- TextField::MicroformatTags => 1,
- TextField::SafetyClassification => 1,
- TextField::InsertionTimestamp => 1,
- TextField::RecipeFirstIngredientTagId => 1,
- TextField::Keywords => 1,
- }
- }
-
- pub fn monogram_field(&self) -> TextField {
- match self {
- TextField::Title => TextField::Title,
- TextField::CleanBody => TextField::CleanBody,
- TextField::StemmedTitle => TextField::StemmedTitle,
- TextField::StemmedCleanBody => TextField::StemmedCleanBody,
- TextField::AllBody => TextField::AllBody,
- TextField::Url => TextField::Url,
- TextField::UrlNoTokenizer => TextField::UrlNoTokenizer,
- TextField::UrlForSiteOperator => TextField::UrlForSiteOperator,
- TextField::SiteWithout => TextField::SiteWithout,
- TextField::Domain => TextField::Domain,
- TextField::SiteNoTokenizer => TextField::SiteNoTokenizer,
- TextField::DomainNoTokenizer => TextField::DomainNoTokenizer,
- TextField::DomainNameNoTokenizer => TextField::DomainNameNoTokenizer,
- TextField::SiteIfHomepageNoTokenizer => TextField::SiteIfHomepageNoTokenizer,
- TextField::DomainIfHomepage => TextField::DomainIfHomepage,
- TextField::DomainNameIfHomepageNoTokenizer => {
- TextField::DomainNameIfHomepageNoTokenizer
- }
- TextField::DomainIfHomepageNoTokenizer => TextField::DomainIfHomepageNoTokenizer,
- TextField::TitleIfHomepage => TextField::TitleIfHomepage,
- TextField::BacklinkText => TextField::BacklinkText,
- TextField::Description => TextField::Description,
- TextField::DmozDescription => TextField::DmozDescription,
- TextField::SchemaOrgJson => TextField::SchemaOrgJson,
- TextField::FlattenedSchemaOrgJson => TextField::FlattenedSchemaOrgJson,
- TextField::CleanBodyBigrams => TextField::CleanBody,
- TextField::TitleBigrams => TextField::Title,
- TextField::CleanBodyTrigrams => TextField::CleanBody,
- TextField::TitleTrigrams => TextField::Title,
- TextField::MicroformatTags => TextField::MicroformatTags,
- TextField::SafetyClassification => TextField::SafetyClassification,
- TextField::InsertionTimestamp => TextField::InsertionTimestamp,
- TextField::RecipeFirstIngredientTagId => TextField::RecipeFirstIngredientTagId,
- TextField::Keywords => TextField::Keywords,
- }
- }
-
- pub fn query_tokenizer(&self) -> Tokenizer {
- match self {
- TextField::TitleBigrams => Tokenizer::default(),
- TextField::CleanBodyBigrams => Tokenizer::default(),
- TextField::TitleTrigrams => Tokenizer::default(),
- TextField::CleanBodyTrigrams => Tokenizer::default(),
- _ => self.indexing_tokenizer(),
- }
- }
-
- pub fn indexing_tokenizer(&self) -> Tokenizer {
- match self {
- TextField::Title => Tokenizer::default(),
- TextField::CleanBody => Tokenizer::default(),
- TextField::StemmedTitle => Tokenizer::new_stemmed(),
- TextField::StemmedCleanBody => Tokenizer::new_stemmed(),
- TextField::AllBody => Tokenizer::default(),
- TextField::Url => Tokenizer::default(),
- TextField::UrlNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::UrlForSiteOperator => Tokenizer::SiteOperator(SiteOperatorUrlTokenizer),
- TextField::SiteWithout => Tokenizer::default(),
- TextField::Domain => Tokenizer::default(),
- TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::SiteIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::DomainNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::DomainNameNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::DomainIfHomepage => Tokenizer::default(),
- TextField::DomainNameIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::DomainIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::TitleIfHomepage => Tokenizer::default(),
- TextField::BacklinkText => Tokenizer::default(),
- TextField::Description => Tokenizer::default(),
- TextField::DmozDescription => Tokenizer::default(),
- TextField::SchemaOrgJson => Tokenizer::Identity(Identity {}),
- TextField::FlattenedSchemaOrgJson => Tokenizer::Json(JsonField),
- TextField::CleanBodyBigrams => Tokenizer::Bigram(BigramTokenizer::default()),
- TextField::TitleBigrams => Tokenizer::Bigram(BigramTokenizer::default()),
- TextField::CleanBodyTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
- TextField::TitleTrigrams => Tokenizer::Trigram(TrigramTokenizer::default()),
- TextField::MicroformatTags => Tokenizer::default(),
- TextField::SafetyClassification => Tokenizer::Identity(Identity {}),
- TextField::InsertionTimestamp => Tokenizer::Identity(Identity {}),
- TextField::RecipeFirstIngredientTagId => Tokenizer::Identity(Identity {}),
- TextField::Keywords => Tokenizer::default(),
- }
- }
-
- pub fn index_option(&self) -> IndexRecordOption {
- if self.has_pos() {
- IndexRecordOption::WithFreqsAndPositions
- } else {
- IndexRecordOption::WithFreqs
- }
- }
-
- pub fn has_pos(&self) -> bool {
- match self {
- TextField::Title => true,
- TextField::CleanBody => true,
- TextField::StemmedTitle => false,
- TextField::StemmedCleanBody => false,
- TextField::AllBody => false,
- TextField::Url => true,
- TextField::UrlNoTokenizer => false,
- TextField::UrlForSiteOperator => true,
- TextField::SiteWithout => true,
- TextField::Domain => true,
- TextField::SiteNoTokenizer => false,
- TextField::SiteIfHomepageNoTokenizer => false,
- TextField::DomainNoTokenizer => false,
- TextField::DomainNameNoTokenizer => false,
- TextField::DomainIfHomepage => false,
- TextField::DomainNameIfHomepageNoTokenizer => false,
- TextField::DomainIfHomepageNoTokenizer => false,
- TextField::TitleIfHomepage => false,
- TextField::BacklinkText => false,
- TextField::Description => true,
- TextField::DmozDescription => true,
- TextField::SchemaOrgJson => false,
- TextField::FlattenedSchemaOrgJson => true,
- TextField::CleanBodyBigrams => false,
- TextField::TitleBigrams => false,
- TextField::CleanBodyTrigrams => false,
- TextField::TitleTrigrams => false,
- TextField::MicroformatTags => true,
- TextField::SafetyClassification => false,
- TextField::InsertionTimestamp => false,
- TextField::RecipeFirstIngredientTagId => false,
- TextField::Keywords => false,
- }
- }
-
- pub fn name(&self) -> &str {
- match self {
- TextField::Title => "title",
- TextField::CleanBody => "body",
- TextField::Url => "url",
- TextField::UrlNoTokenizer => "url_no_tokenizer",
- TextField::UrlForSiteOperator => "url_for_site_operator",
- TextField::SiteWithout => "site",
- TextField::Domain => "domain",
- TextField::SiteNoTokenizer => "site_no_tokenizer",
- TextField::SiteIfHomepageNoTokenizer => "site_if_homepage_no_tokenizer",
- TextField::DomainNoTokenizer => "domain_no_tokenizer",
- TextField::DomainNameNoTokenizer => "domain_name_no_tokenizer",
- TextField::BacklinkText => "backlink_text",
- TextField::StemmedTitle => "stemmed_title",
- TextField::StemmedCleanBody => "stemmed_body",
- TextField::DomainIfHomepage => "domain_if_homepage",
- TextField::DomainNameIfHomepageNoTokenizer => "domain_name_if_homepage_no_tokenizer",
- TextField::DomainIfHomepageNoTokenizer => "domain_if_homepage_no_tokenizer",
- TextField::Description => "description",
- TextField::TitleIfHomepage => "title_if_homepage",
- TextField::AllBody => "all_body",
- TextField::DmozDescription => "dmoz_description",
- TextField::SchemaOrgJson => "schema_org_json",
- TextField::FlattenedSchemaOrgJson => "flattened_schema_org_json",
- TextField::CleanBodyBigrams => "clean_body_bigrams",
- TextField::TitleBigrams => "title_bigrams",
- TextField::CleanBodyTrigrams => "clean_body_trigrams",
- TextField::TitleTrigrams => "title_trigrams",
- TextField::MicroformatTags => "microformat_tags",
- TextField::SafetyClassification => "safety_classification",
- TextField::InsertionTimestamp => "insertion_timestamp",
- TextField::RecipeFirstIngredientTagId => "recipe_first_ingredient_tag_id",
- TextField::Keywords => "keywords",
- }
- }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum FastField {
- IsHomepage,
- HostCentrality,
- HostCentralityRank,
- PageCentrality,
- PageCentralityRank,
- FetchTimeMs,
- LastUpdated,
- TrackerScore,
- Region,
- NumUrlTokens,
- NumTitleTokens,
- NumCleanBodyTokens,
- NumDescriptionTokens,
- NumUrlForSiteOperatorTokens,
- NumDomainTokens,
- NumMicroformatTagsTokens,
- SiteHash1,
- SiteHash2,
- UrlWithoutQueryHash1,
- UrlWithoutQueryHash2,
- TitleHash1,
- TitleHash2,
- UrlHash1,
- UrlHash2,
- DomainHash1,
- DomainHash2,
- UrlWithoutTldHash1,
- UrlWithoutTldHash2,
- PreComputedScore,
- HostNodeID,
- SimHash,
- NumFlattenedSchemaTokens,
- NumPathAndQuerySlashes,
- NumPathAndQueryDigits,
- LikelyHasAds,
- LikelyHasPaywall,
- LinkDensity,
- TitleEmbeddings,
- KeywordEmbeddings,
-}
-
-impl FastField {
- pub fn name(&self) -> &str {
- match self {
- FastField::HostCentrality => "host_centrality",
- FastField::HostCentralityRank => "host_centrality_rank",
- FastField::PageCentrality => "page_centrality",
- FastField::PageCentralityRank => "page_centrality_rank",
- FastField::IsHomepage => "is_homepage",
- FastField::FetchTimeMs => "fetch_time_ms",
- FastField::LastUpdated => "last_updated",
- FastField::TrackerScore => "tracker_score",
- FastField::Region => "region",
- FastField::NumUrlTokens => "num_url_tokens",
- FastField::NumTitleTokens => "num_title_tokens",
- FastField::NumCleanBodyTokens => "num_clean_body_tokens",
- FastField::NumDescriptionTokens => "num_description_tokens",
- FastField::NumDomainTokens => "num_domain_tokens",
- FastField::NumUrlForSiteOperatorTokens => "num_url_for_site_operator_tokens",
- FastField::NumFlattenedSchemaTokens => "num_flattened_schema_tokens",
- FastField::NumMicroformatTagsTokens => "num_microformat_tags_tokens",
- FastField::SiteHash1 => "site_hash1",
- FastField::SiteHash2 => "site_hash2",
- FastField::UrlWithoutQueryHash1 => "url_without_query_hash1",
- FastField::UrlWithoutQueryHash2 => "url_without_query_hash2",
- FastField::TitleHash1 => "title_hash1",
- FastField::TitleHash2 => "title_hash2",
- FastField::UrlHash1 => "url_hash1",
- FastField::UrlHash2 => "url_hash2",
- FastField::DomainHash1 => "domain_hash1",
- FastField::DomainHash2 => "domain_hash2",
- FastField::UrlWithoutTldHash1 => "url_without_tld_hash1",
- FastField::UrlWithoutTldHash2 => "url_without_tld_hash2",
- FastField::PreComputedScore => "pre_computed_score",
- FastField::HostNodeID => "host_node_id",
- FastField::SimHash => "sim_hash",
- FastField::NumPathAndQuerySlashes => "num_path_and_query_slashes",
- FastField::NumPathAndQueryDigits => "num_path_and_query_digits",
- FastField::LikelyHasAds => "likely_has_ads",
- FastField::LikelyHasPaywall => "likely_has_paywall",
- FastField::LinkDensity => "link_density",
- FastField::TitleEmbeddings => "title_embeddings",
- FastField::KeywordEmbeddings => "keyword_embeddings",
- }
- }
-}
-
-impl From<FastField> for usize {
- fn from(value: FastField) -> Self {
- value as usize
- }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum Field {
- Fast(FastField),
- Text(TextField),
-}
-
-static ALL_FIELDS: [Field; 70] = [
- Field::Text(TextField::Title),
- Field::Text(TextField::CleanBody),
- Field::Text(TextField::StemmedTitle),
- Field::Text(TextField::StemmedCleanBody),
- Field::Text(TextField::AllBody),
- Field::Text(TextField::Url),
- Field::Text(TextField::UrlNoTokenizer),
- Field::Text(TextField::UrlForSiteOperator),
- Field::Text(TextField::SiteWithout),
- Field::Text(TextField::Domain),
- Field::Text(TextField::SiteNoTokenizer),
- Field::Text(TextField::SiteIfHomepageNoTokenizer),
- Field::Text(TextField::DomainNoTokenizer),
- Field::Text(TextField::DomainNameNoTokenizer),
- Field::Text(TextField::DomainIfHomepage),
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer),
- Field::Text(TextField::DomainIfHomepageNoTokenizer),
- Field::Text(TextField::TitleIfHomepage),
- Field::Text(TextField::BacklinkText),
- Field::Text(TextField::Description),
- Field::Text(TextField::DmozDescription),
- Field::Text(TextField::SchemaOrgJson),
- Field::Text(TextField::FlattenedSchemaOrgJson),
- Field::Text(TextField::CleanBodyBigrams),
- Field::Text(TextField::TitleBigrams),
- Field::Text(TextField::CleanBodyTrigrams),
- Field::Text(TextField::TitleTrigrams),
- Field::Text(TextField::MicroformatTags),
- Field::Text(TextField::SafetyClassification),
- Field::Text(TextField::InsertionTimestamp),
- Field::Text(TextField::Keywords),
- // FAST FIELDS
- Field::Fast(FastField::IsHomepage),
- Field::Fast(FastField::HostCentrality),
- Field::Fast(FastField::HostCentralityRank),
- Field::Fast(FastField::PageCentrality),
- Field::Fast(FastField::PageCentralityRank),
- Field::Fast(FastField::FetchTimeMs),
- Field::Fast(FastField::LastUpdated),
- Field::Fast(FastField::TrackerScore),
- Field::Fast(FastField::Region),
- Field::Fast(FastField::NumUrlTokens),
- Field::Fast(FastField::NumTitleTokens),
- Field::Fast(FastField::NumCleanBodyTokens),
- Field::Fast(FastField::NumDescriptionTokens),
- Field::Fast(FastField::NumDomainTokens),
- Field::Fast(FastField::NumUrlForSiteOperatorTokens),
- Field::Fast(FastField::NumFlattenedSchemaTokens),
- Field::Fast(FastField::NumMicroformatTagsTokens),
- Field::Fast(FastField::SiteHash1),
- Field::Fast(FastField::SiteHash2),
- Field::Fast(FastField::UrlWithoutQueryHash1),
- Field::Fast(FastField::UrlWithoutQueryHash2),
- Field::Fast(FastField::TitleHash1),
- Field::Fast(FastField::TitleHash2),
- Field::Fast(FastField::UrlHash1),
- Field::Fast(FastField::UrlHash2),
- Field::Fast(FastField::DomainHash1),
- Field::Fast(FastField::DomainHash2),
- Field::Fast(FastField::UrlWithoutTldHash1),
- Field::Fast(FastField::UrlWithoutTldHash2),
- Field::Fast(FastField::PreComputedScore),
- Field::Fast(FastField::HostNodeID),
- Field::Fast(FastField::SimHash),
- Field::Fast(FastField::NumPathAndQuerySlashes),
- Field::Fast(FastField::NumPathAndQueryDigits),
- Field::Fast(FastField::LikelyHasAds),
- Field::Fast(FastField::LikelyHasPaywall),
- Field::Fast(FastField::LinkDensity),
- Field::Fast(FastField::TitleEmbeddings),
- Field::Fast(FastField::KeywordEmbeddings),
-];
-
-impl Field {
- #[inline]
- pub fn get(field_id: usize) -> Option<&'static Field> {
- ALL_FIELDS.get(field_id)
- }
-
- #[inline]
- pub fn all() -> impl Iterator<Item = &'static Field> {
- ALL_FIELDS.iter()
- }
- fn default_text_options(&self) -> tantivy::schema::TextOptions {
- let tokenizer = self.as_text().unwrap().indexing_tokenizer();
- let option = self.as_text().unwrap().index_option();
-
- TextOptions::default().set_indexing_options(
- TextFieldIndexing::default()
- .set_tokenizer(tokenizer.as_str())
- .set_index_option(option),
- )
- }
-
- pub fn has_pos(&self) -> bool {
- match self {
- Field::Fast(_) => false,
- Field::Text(text) => text.has_pos(),
- }
- }
-
- pub fn options(&self) -> IndexingOption {
- match self {
- Field::Text(TextField::Title) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::CleanBody) => IndexingOption::Text(self.default_text_options()),
- Field::Text(TextField::Url) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::UrlNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::UrlForSiteOperator) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::SiteWithout) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::Domain) => IndexingOption::Text(self.default_text_options()),
- Field::Text(TextField::SiteNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::DomainNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::DomainNameNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::AllBody) => IndexingOption::Text(self.default_text_options()),
- Field::Text(TextField::DomainIfHomepage) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::TitleIfHomepage) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::BacklinkText) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::StemmedTitle) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::StemmedCleanBody) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::Description) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::DmozDescription) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::SchemaOrgJson) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::FlattenedSchemaOrgJson) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::CleanBodyBigrams) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::TitleBigrams) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::CleanBodyTrigrams) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::TitleTrigrams) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::MicroformatTags) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::SafetyClassification) => {
- IndexingOption::Text(self.default_text_options())
- }
- Field::Text(TextField::RecipeFirstIngredientTagId) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Text(TextField::InsertionTimestamp) => {
- IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())
- }
- Field::Text(TextField::Keywords) => {
- IndexingOption::Text(self.default_text_options().set_stored())
- }
- Field::Fast(FastField::IsHomepage) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::HostCentrality) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::HostCentralityRank) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::PageCentrality) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::PageCentralityRank) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::FetchTimeMs) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::TrackerScore) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::LastUpdated) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_stored()
- .set_indexed(),
- ),
- Field::Fast(FastField::Region) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_stored()
- .set_indexed(),
- ),
- Field::Fast(FastField::NumCleanBodyTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumDescriptionTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumTitleTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumMicroformatTagsTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumUrlTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumDomainTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::NumFlattenedSchemaTokens) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
- }
- Field::Fast(FastField::SiteHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::SiteHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlWithoutQueryHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlWithoutQueryHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlWithoutTldHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::UrlWithoutTldHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::DomainHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::DomainHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::TitleHash1) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::TitleHash2) => {
- IndexingOption::Integer(NumericOptions::default().set_fast())
- }
- Field::Fast(FastField::PreComputedScore) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::HostNodeID) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::SimHash) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::NumPathAndQuerySlashes) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::NumPathAndQueryDigits) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::LikelyHasAds) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::LikelyHasPaywall) => IndexingOption::Integer(
- NumericOptions::default()
- .set_fast()
- .set_indexed()
- .set_stored(),
- ),
- Field::Fast(FastField::LinkDensity) => {
- IndexingOption::Integer(NumericOptions::default().set_fast().set_stored())
- }
- Field::Fast(FastField::TitleEmbeddings) => {
- IndexingOption::Bytes(BytesOptions::default().set_fast())
- }
- Field::Fast(FastField::KeywordEmbeddings) => {
- IndexingOption::Bytes(BytesOptions::default().set_fast())
- }
- }
- }
-
- pub fn name(&self) -> &str {
- match self {
- Field::Text(t) => t.name(),
- Field::Fast(f) => f.name(),
- }
- }
-
- /// Whether or not the field should be included
- /// in the fields that the `Query` searches.
- ///
- /// The fields can still be searched by manually
- /// constructing a tantivy query.
- pub fn is_searchable(&self) -> bool {
- !matches!(
- self,
- Field::Text(TextField::BacklinkText)
- | Field::Text(TextField::SchemaOrgJson)
- | Field::Text(TextField::MicroformatTags)
- | Field::Text(TextField::SafetyClassification)
- | Field::Text(TextField::FlattenedSchemaOrgJson)
- | Field::Text(TextField::UrlForSiteOperator)
- | Field::Text(TextField::Description)
- | Field::Text(TextField::DmozDescription)
- | Field::Text(TextField::SiteIfHomepageNoTokenizer)
- | Field::Text(TextField::DomainIfHomepage)
- | Field::Text(TextField::DomainNameIfHomepageNoTokenizer)
- | Field::Text(TextField::DomainIfHomepageNoTokenizer)
- | Field::Text(TextField::TitleIfHomepage)
- | Field::Text(TextField::SiteWithout) // will match url
- | Field::Text(TextField::Domain) // will match url
- | Field::Text(TextField::InsertionTimestamp)
- | Field::Text(TextField::RecipeFirstIngredientTagId)
- ) && !self.is_fast()
- }
-
- pub fn is_fast(&self) -> bool {
- matches!(self, Field::Fast(_))
- }
-
- pub fn as_text(&self) -> Option<TextField> {
- match self {
- Field::Fast(_) => None,
- Field::Text(field) => Some(*field),
- }
- }
-
- pub fn as_fast(&self) -> Option<FastField> {
- match self {
- Field::Fast(field) => Some(*field),
- Field::Text(_) => None,
- }
- }
-}
-
-pub fn create_schema() -> tantivy::schema::Schema {
- let mut builder = tantivy::schema::Schema::builder();
-
- for field in &ALL_FIELDS {
- match field.options() {
- IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
- IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
- IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),
- IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
- };
- }
-
- builder.build()
-}
-
-pub enum IndexingOption {
- Text(TextOptions),
- Integer(NumericOptions),
- DateTime(DateOptions),
- Bytes(BytesOptions),
-}
-
-pub enum DataType {
- U64,
- Bytes,
-}
-
-impl FastField {
- pub fn data_type(&self) -> DataType {
- match self {
- FastField::IsHomepage => DataType::U64,
- FastField::HostCentrality => DataType::U64,
- FastField::HostCentralityRank => DataType::U64,
- FastField::PageCentrality => DataType::U64,
- FastField::PageCentralityRank => DataType::U64,
- FastField::FetchTimeMs => DataType::U64,
- FastField::LastUpdated => DataType::U64,
- FastField::TrackerScore => DataType::U64,
- FastField::Region => DataType::U64,
- FastField::NumUrlTokens => DataType::U64,
- FastField::NumTitleTokens => DataType::U64,
- FastField::NumMicroformatTagsTokens => DataType::U64,
- FastField::NumCleanBodyTokens => DataType::U64,
- FastField::NumDescriptionTokens => DataType::U64,
- FastField::NumDomainTokens => DataType::U64,
- FastField::NumUrlForSiteOperatorTokens => DataType::U64,
- FastField::NumFlattenedSchemaTokens => DataType::U64,
- FastField::SiteHash1 => DataType::U64,
- FastField::SiteHash2 => DataType::U64,
- FastField::UrlWithoutQueryHash1 => DataType::U64,
- FastField::UrlWithoutQueryHash2 => DataType::U64,
- FastField::TitleHash1 => DataType::U64,
- FastField::TitleHash2 => DataType::U64,
- FastField::UrlHash1 => DataType::U64,
- FastField::UrlHash2 => DataType::U64,
- FastField::DomainHash1 => DataType::U64,
- FastField::DomainHash2 => DataType::U64,
- FastField::UrlWithoutTldHash1 => DataType::U64,
- FastField::UrlWithoutTldHash2 => DataType::U64,
- FastField::PreComputedScore => DataType::U64,
- FastField::HostNodeID => DataType::U64,
- FastField::SimHash => DataType::U64,
- FastField::NumPathAndQuerySlashes => DataType::U64,
- FastField::NumPathAndQueryDigits => DataType::U64,
- FastField::LikelyHasAds => DataType::U64,
- FastField::LikelyHasPaywall => DataType::U64,
- FastField::LinkDensity => DataType::U64,
- FastField::TitleEmbeddings => DataType::Bytes,
- FastField::KeywordEmbeddings => DataType::Bytes,
- }
- }
-}
diff --git a/crates/core/src/schema/fast_field.rs b/crates/core/src/schema/fast_field.rs
new file mode 100644
index 00000000..44413bce
--- /dev/null
+++ b/crates/core/src/schema/fast_field.rs
@@ -0,0 +1,1233 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use enum_dispatch::enum_dispatch;
+use strum::{EnumDiscriminants, VariantArray};
+use tantivy::{
+ schema::{BytesOptions, NumericOptions},
+ TantivyDocument,
+};
+
+use crate::{
+ enum_map::InsertEnumMapKey,
+ from_discriminant, simhash,
+ webpage::{html::FnCache, Html, Webpage},
+ Result,
+};
+
+use super::{IndexingOption, FLOAT_SCALING};
+
+#[enum_dispatch]
+pub trait FastField: Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash {
+ fn name(&self) -> &str;
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()>;
+
+ fn add_webpage_tantivy(
+ &self,
+ _webpage: &crate::webpage::Webpage,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn data_type(&self) -> DataType {
+ DataType::U64
+ }
+
+ fn is_stored(&self) -> bool {
+ false
+ }
+
+ fn is_indexed(&self) -> bool {
+ true
+ }
+
+ fn indexing_option(&self) -> IndexingOption {
+ debug_assert!(matches!(self.data_type(), DataType::U64));
+
+ let mut opt = NumericOptions::default().set_fast();
+
+ if self.is_stored() {
+ opt = opt.set_stored();
+ }
+
+ if self.is_indexed() {
+ opt = opt.set_indexed();
+ }
+
+ IndexingOption::Integer(opt)
+ }
+
+ fn tantivy_field(&self, schema: &tantivy::schema::Schema) -> tantivy::schema::Field {
+ schema
+ .get_field(self.name())
+ .unwrap_or_else(|_| unreachable!("Unknown field: {}", self.name()))
+ }
+}
+
+#[enum_dispatch(FastField)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants)]
+#[strum_discriminants(derive(VariantArray))]
+pub enum FastFieldEnum {
+ IsHomepage,
+ HostCentrality,
+ HostCentralityRank,
+ PageCentrality,
+ PageCentralityRank,
+ FetchTimeMs,
+ LastUpdated,
+ TrackerScore,
+ Region,
+ NumUrlTokens,
+ NumTitleTokens,
+ NumCleanBodyTokens,
+ NumDescriptionTokens,
+ NumUrlForSiteOperatorTokens,
+ NumDomainTokens,
+ NumMicroformatTagsTokens,
+ SiteHash1,
+ SiteHash2,
+ UrlWithoutQueryHash1,
+ UrlWithoutQueryHash2,
+ TitleHash1,
+ TitleHash2,
+ UrlHash1,
+ UrlHash2,
+ DomainHash1,
+ DomainHash2,
+ UrlWithoutTldHash1,
+ UrlWithoutTldHash2,
+ PreComputedScore,
+ HostNodeID,
+ SimHash,
+ NumFlattenedSchemaTokens,
+ NumPathAndQuerySlashes,
+ NumPathAndQueryDigits,
+ LikelyHasAds,
+ LikelyHasPaywall,
+ LinkDensity,
+ TitleEmbeddings,
+ KeywordEmbeddings,
+}
+
+from_discriminant!(FastFieldEnumDiscriminants => FastFieldEnum,
+[
+ IsHomepage,
+ HostCentrality,
+ HostCentralityRank,
+ PageCentrality,
+ PageCentralityRank,
+ FetchTimeMs,
+ LastUpdated,
+ TrackerScore,
+ Region,
+ NumUrlTokens,
+ NumTitleTokens,
+ NumCleanBodyTokens,
+ NumDescriptionTokens,
+ NumUrlForSiteOperatorTokens,
+ NumDomainTokens,
+ NumMicroformatTagsTokens,
+ SiteHash1,
+ SiteHash2,
+ UrlWithoutQueryHash1,
+ UrlWithoutQueryHash2,
+ TitleHash1,
+ TitleHash2,
+ UrlHash1,
+ UrlHash2,
+ DomainHash1,
+ DomainHash2,
+ UrlWithoutTldHash1,
+ UrlWithoutTldHash2,
+ PreComputedScore,
+ HostNodeID,
+ SimHash,
+ NumFlattenedSchemaTokens,
+ NumPathAndQuerySlashes,
+ NumPathAndQueryDigits,
+ LikelyHasAds,
+ LikelyHasPaywall,
+ LinkDensity,
+ TitleEmbeddings,
+ KeywordEmbeddings,
+]);
+
+impl FastFieldEnum {
+ pub fn all() -> impl Iterator<Item = FastFieldEnum> {
+ FastFieldEnumDiscriminants::VARIANTS
+ .iter()
+ .copied()
+ .map(|v| v.into())
+ }
+
+ pub fn get(field_id: usize) -> Option<FastFieldEnum> {
+ FastFieldEnumDiscriminants::VARIANTS
+ .get(field_id)
+ .copied()
+ .map(FastFieldEnum::from)
+ }
+
+ pub fn num_variants() -> usize {
+ FastFieldEnumDiscriminants::VARIANTS.len()
+ }
+}
+
+pub enum DataType {
+ U64,
+ Bytes,
+}
+
+impl InsertEnumMapKey for FastFieldEnum {
+ fn into_usize(self) -> usize {
+ FastFieldEnumDiscriminants::from(self) as usize
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct IsHomepage;
+impl FastField for IsHomepage {
+ fn name(&self) -> &str {
+ "is_homepage"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), (html.is_homepage()).into());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct HostCentrality;
+impl FastField for HostCentrality {
+ fn name(&self) -> &str {
+ "host_centrality"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ (webpage.host_centrality * FLOAT_SCALING as f64) as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct HostCentralityRank;
+impl FastField for HostCentralityRank {
+ fn name(&self) -> &str {
+ "host_centrality_rank"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), webpage.host_centrality_rank);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PageCentrality;
+impl FastField for PageCentrality {
+ fn name(&self) -> &str {
+ "page_centrality"
+ }
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ (webpage.page_centrality * FLOAT_SCALING as f64) as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PageCentralityRank;
+impl FastField for PageCentralityRank {
+ fn name(&self) -> &str {
+ "page_centrality_rank"
+ }
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), webpage.page_centrality_rank);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct FetchTimeMs;
+impl FastField for FetchTimeMs {
+ fn name(&self) -> &str {
+ "fetch_time_ms"
+ }
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), webpage.fetch_time_ms);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct LastUpdated;
+impl FastField for LastUpdated {
+ fn name(&self) -> &str {
+ "last_updated"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ html.updated_time()
+ .map_or(0, |time| time.timestamp().max(0) as u64),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TrackerScore;
+impl FastField for TrackerScore {
+ fn name(&self) -> &str {
+ "tracker_score"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), html.trackers().len() as u64);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Region;
+impl FastField for Region {
+ fn name(&self) -> &str {
+ "region"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let region = crate::webpage::region::Region::guess_from(webpage);
+ if let Ok(region) = region {
+ doc.add_u64(self.tantivy_field(schema), region.id());
+ } else {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ crate::webpage::region::Region::All.id(),
+ );
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumUrlTokens;
+impl FastField for NumUrlTokens {
+ fn name(&self) -> &str {
+ "num_url_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_url().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumTitleTokens;
+impl FastField for NumTitleTokens {
+ fn name(&self) -> &str {
+ "num_title_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache
+ .pretokenize_title()
+ .as_ref()
+ .map(|n| n.tokens.len() as u64)
+ .map_err(|e| anyhow::anyhow!("{}", e))?,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumCleanBodyTokens;
+impl FastField for NumCleanBodyTokens {
+ fn name(&self) -> &str {
+ "num_clean_body_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_clean_text().tokens.len() as u64,
+ );
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumDescriptionTokens;
+impl FastField for NumDescriptionTokens {
+ fn name(&self) -> &str {
+ "num_description_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_description().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumUrlForSiteOperatorTokens;
+impl FastField for NumUrlForSiteOperatorTokens {
+ fn name(&self) -> &str {
+ "num_url_for_site_operator_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_url_for_site_operator().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumDomainTokens;
+impl FastField for NumDomainTokens {
+ fn name(&self) -> &str {
+ "num_domain_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_domain().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumMicroformatTagsTokens;
+impl FastField for NumMicroformatTagsTokens {
+ fn name(&self) -> &str {
+ "num_microformat_tags_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenize_microformats().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SiteHash1;
+impl FastField for SiteHash1 {
+ fn name(&self) -> &str {
+ "site_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.site_hash()[0]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SiteHash2;
+impl FastField for SiteHash2 {
+ fn name(&self) -> &str {
+ "site_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.site_hash()[1]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlWithoutQueryHash1;
+impl FastField for UrlWithoutQueryHash1 {
+ fn name(&self) -> &str {
+ "url_without_query_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.url_without_query_hash()[0],
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlWithoutQueryHash2;
+impl FastField for UrlWithoutQueryHash2 {
+ fn name(&self) -> &str {
+ "url_without_query_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.url_without_query_hash()[1],
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleHash1;
+impl FastField for TitleHash1 {
+ fn name(&self) -> &str {
+ "title_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.title_hash()[0]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleHash2;
+impl FastField for TitleHash2 {
+ fn name(&self) -> &str {
+ "title_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.title_hash()[1]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlHash1;
+impl FastField for UrlHash1 {
+ fn name(&self) -> &str {
+ "url_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.url_hash()[0]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlHash2;
+impl FastField for UrlHash2 {
+ fn name(&self) -> &str {
+ "url_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.url_hash()[1]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainHash1;
+impl FastField for DomainHash1 {
+ fn name(&self) -> &str {
+ "domain_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[0]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainHash2;
+impl FastField for DomainHash2 {
+ fn name(&self) -> &str {
+ "domain_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.domain_hash()[1]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlWithoutTldHash1;
+impl FastField for UrlWithoutTldHash1 {
+ fn name(&self) -> &str {
+ "url_without_tld_hash1"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[0]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlWithoutTldHash2;
+impl FastField for UrlWithoutTldHash2 {
+ fn name(&self) -> &str {
+ "url_without_tld_hash2"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), cache.url_without_tld_hash()[1]);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PreComputedScore;
+impl FastField for PreComputedScore {
+ fn name(&self) -> &str {
+ "pre_computed_score"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ (webpage.pre_computed_score * FLOAT_SCALING as f64) as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct HostNodeID;
+impl FastField for HostNodeID {
+ fn name(&self) -> &str {
+ "host_node_id"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ match &webpage.node_id {
+ Some(node_id) => {
+ doc.add_u64(self.tantivy_field(schema), node_id.as_u64());
+ }
+ None => {
+ doc.add_u64(self.tantivy_field(schema), u64::MAX);
+ }
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SimHash;
+impl FastField for SimHash {
+ fn name(&self) -> &str {
+ "sim_hash"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let clean_text = cache.pretokenize_clean_text();
+
+ let hash = if !clean_text.text.is_empty() {
+ simhash::hash(&clean_text.text)
+ } else {
+ 0
+ };
+ doc.add_u64(self.tantivy_field(schema), hash);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumFlattenedSchemaTokens;
+impl FastField for NumFlattenedSchemaTokens {
+ fn name(&self) -> &str {
+ "num_flattened_schema_tokens"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ cache.pretokenized_schema_json().tokens.len() as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumPathAndQuerySlashes;
+impl FastField for NumPathAndQuerySlashes {
+ fn name(&self) -> &str {
+ "num_path_and_query_slashes"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let num_slashes = html
+ .url()
+ .path_segments()
+ .map(|segments| segments.count())
+ .unwrap_or(0);
+
+ doc.add_u64(self.tantivy_field(schema), num_slashes as u64);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct NumPathAndQueryDigits;
+impl FastField for NumPathAndQueryDigits {
+ fn name(&self) -> &str {
+ "num_path_and_query_digits"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let num_digits = html
+ .url()
+ .path()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count()
+ + html
+ .url()
+ .query()
+ .unwrap_or_default()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count();
+
+ doc.add_u64(self.tantivy_field(schema), num_digits as u64);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct LikelyHasAds;
+impl FastField for LikelyHasAds {
+ fn name(&self) -> &str {
+ "likely_has_ads"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), html.likely_has_ads() as u64);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct LikelyHasPaywall;
+impl FastField for LikelyHasPaywall {
+ fn name(&self) -> &str {
+ "likely_has_paywall"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(self.tantivy_field(schema), html.likely_has_paywall() as u64);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct LinkDensity;
+impl FastField for LinkDensity {
+ fn name(&self) -> &str {
+ "link_density"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_u64(
+ self.tantivy_field(schema),
+ (html.link_density() * FLOAT_SCALING as f64) as u64,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleEmbeddings;
+impl FastField for TitleEmbeddings {
+ fn name(&self) -> &str {
+ "title_embeddings"
+ }
+
+ fn data_type(&self) -> DataType {
+ DataType::Bytes
+ }
+
+ fn indexing_option(&self) -> IndexingOption {
+ IndexingOption::Bytes(BytesOptions::default().set_fast().set_stored())
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ if let Some(emb) = &webpage.title_embedding {
+ let mut serialized = Vec::new();
+ emb.write_bytes(&mut serialized)?;
+
+ doc.add_bytes(self.tantivy_field(schema), serialized);
+ } else {
+ doc.add_bytes(self.tantivy_field(schema), Vec::new());
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct KeywordEmbeddings;
+impl FastField for KeywordEmbeddings {
+ fn name(&self) -> &str {
+ "keyword_embeddings"
+ }
+
+ fn data_type(&self) -> DataType {
+ DataType::Bytes
+ }
+
+ fn indexing_option(&self) -> IndexingOption {
+ IndexingOption::Bytes(BytesOptions::default().set_fast().set_stored())
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ if let Some(emb) = &webpage.keyword_embedding {
+ let mut serialized = Vec::new();
+ emb.write_bytes(&mut serialized)?;
+
+ doc.add_bytes(self.tantivy_field(schema), serialized);
+ } else {
+ doc.add_bytes(self.tantivy_field(schema), Vec::new());
+ }
+
+ Ok(())
+ }
+}
diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rs
new file mode 100644
index 00000000..1a14d767
--- /dev/null
+++ b/crates/core/src/schema/mod.rs
@@ -0,0 +1,139 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod fast_field;
+pub mod text_field;
+
+use tantivy::{
+ schema::{BytesOptions, NumericOptions, TextOptions},
+ DateOptions,
+};
+
+pub use fast_field::{DataType, FastFieldEnum};
+pub use text_field::TextFieldEnum;
+
+use self::{fast_field::FastField, text_field::TextField};
+
+pub const FLOAT_SCALING: u64 = 1_000_000_000;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Field {
+ Fast(FastFieldEnum),
+ Text(TextFieldEnum),
+}
+
+impl Field {
+ #[inline]
+ pub fn get(field_id: usize) -> Option<Field> {
+ if field_id < TextFieldEnum::num_variants() {
+ return Some(Field::Text(TextFieldEnum::get(field_id).unwrap()));
+ }
+ let field_id = field_id - TextFieldEnum::num_variants();
+
+ if field_id < FastFieldEnum::num_variants() {
+ return Some(Field::Fast(FastFieldEnum::get(field_id).unwrap()));
+ }
+ let _field_id = field_id - FastFieldEnum::num_variants();
+
+ return None;
+ }
+
+ #[inline]
+ pub fn all() -> impl Iterator<Item = Field> {
+ TextFieldEnum::all()
+ .map(Field::Text)
+ .chain(FastFieldEnum::all().map(Field::Fast))
+ }
+
+ pub fn has_pos(&self) -> bool {
+ match self {
+ Field::Fast(_) => false,
+ Field::Text(text) => text.has_pos(),
+ }
+ }
+
+ pub fn indexing_option(&self) -> IndexingOption {
+ match self {
+ Field::Text(f) => f.indexing_option(),
+ Field::Fast(f) => f.indexing_option(),
+ }
+ }
+
+ pub fn name(&self) -> &str {
+ match self {
+ Field::Text(f) => f.name(),
+ Field::Fast(f) => f.name(),
+ }
+ }
+
+ pub fn is_searchable(&self) -> bool {
+ match self {
+ Field::Text(f) => f.is_searchable(),
+ Field::Fast(_) => false,
+ }
+ }
+
+ pub fn as_text(&self) -> Option<TextFieldEnum> {
+ match self {
+ Field::Fast(_) => None,
+ Field::Text(field) => Some(*field),
+ }
+ }
+
+ pub fn as_fast(&self) -> Option<FastFieldEnum> {
+ match self {
+ Field::Fast(field) => Some(*field),
+ Field::Text(_) => None,
+ }
+ }
+}
+
+pub fn create_schema() -> tantivy::schema::Schema {
+ let mut builder = tantivy::schema::Schema::builder();
+
+ for field in Field::all() {
+ match field.indexing_option() {
+ IndexingOption::Text(options) => builder.add_text_field(field.name(), options),
+ IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),
+ IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),
+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),
+ };
+ }
+
+ builder.build()
+}
+
+pub enum IndexingOption {
+ Text(TextOptions),
+ Integer(NumericOptions),
+ DateTime(DateOptions),
+ Bytes(BytesOptions),
+}
+
+#[macro_export]
+macro_rules! from_discriminant {
+ ($discenum:ident => $enum:ident, [$($disc:ident),*$(,)?]) => {
+ impl From<$discenum> for $enum {
+ fn from(value: $discenum) -> Self {
+ match value {
+ $(
+ $discenum::$disc => $disc.into(),
+ )*
+ }
+ }
+ }
+ };
+}
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
new file mode 100644
index 00000000..a69cbfd0
--- /dev/null
+++ b/crates/core/src/schema/text_field.rs
@@ -0,0 +1,1427 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use enum_dispatch::enum_dispatch;
+use strum::{EnumDiscriminants, VariantArray};
+use tantivy::{
+ schema::{IndexRecordOption, TextFieldIndexing, TextOptions},
+ time::OffsetDateTime,
+ tokenizer::PreTokenizedString,
+ TantivyDocument,
+};
+use whatlang::Lang;
+
+use crate::{
+ enum_map::InsertEnumMapKey,
+ from_discriminant,
+ tokenizer::{
+ BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,
+ },
+ webpage::Html,
+ Result,
+};
+
+use crate::webpage::html::FnCache;
+
+use super::IndexingOption;
+
+#[enum_dispatch]
+pub trait TextField:
+ Clone + Copy + std::fmt::Debug + PartialEq + Eq + std::hash::Hash + Into<TextFieldEnum>
+{
+ fn name(&self) -> &str;
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()>;
+
+ fn add_webpage_tantivy(
+ &self,
+ _webpage: &crate::webpage::Webpage,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::default()
+ }
+
+ fn query_tokenizer(&self) -> Tokenizer {
+ self.indexing_tokenizer()
+ }
+
+ fn ngram_size(&self) -> usize {
+ 1
+ }
+
+ fn monogram_field(&self) -> TextFieldEnum {
+ debug_assert_eq!(self.ngram_size(), 1);
+ (*self).into()
+ }
+
+ /// Whether or not the field should be included
+ /// in the fields that the `Query` searches.
+ ///
+ /// The fields can still be searched by manually
+ /// constructing a tantivy query.
+ fn is_searchable(&self) -> bool {
+ false
+ }
+
+ fn has_pos(&self) -> bool {
+ false
+ }
+
+ fn is_stored(&self) -> bool {
+ false
+ }
+
+ fn record_option(&self) -> IndexRecordOption {
+ if self.has_pos() {
+ IndexRecordOption::WithFreqsAndPositions
+ } else {
+ IndexRecordOption::WithFreqs
+ }
+ }
+
+ fn indexing_option(&self) -> IndexingOption {
+ let tokenizer = self.indexing_tokenizer();
+ let option = self.record_option();
+
+ let mut opt = TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer(tokenizer.as_str())
+ .set_index_option(option),
+ );
+
+ if self.is_stored() {
+ opt = opt.set_stored();
+ }
+
+ IndexingOption::Text(opt)
+ }
+
+ fn tantivy_field(&self, schema: &tantivy::schema::Schema) -> tantivy::schema::Field {
+ schema
+ .get_field(self.name())
+ .unwrap_or_else(|_| unreachable!("Unknown field: {}", self.name()))
+ }
+}
+
+#[enum_dispatch(TextField)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants)]
+#[strum_discriminants(derive(VariantArray))]
+pub enum TextFieldEnum {
+ Title,
+ CleanBody,
+ StemmedTitle,
+ StemmedCleanBody,
+ AllBody,
+ Url,
+ UrlNoTokenizer,
+ UrlForSiteOperator,
+ SiteWithout,
+ Domain,
+ SiteNoTokenizer,
+ DomainNoTokenizer,
+ DomainNameNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ SiteIfHomepageNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainIfHomepage,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainNameIfHomepageNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ DomainIfHomepageNoTokenizer,
+ /// this field is only set if the webpage is the homepage for the site. Allows us to boost
+ TitleIfHomepage,
+ BacklinkText,
+ Description,
+ DmozDescription,
+ SchemaOrgJson,
+ FlattenedSchemaOrgJson,
+ CleanBodyBigrams,
+ TitleBigrams,
+ CleanBodyTrigrams,
+ TitleTrigrams,
+ MicroformatTags,
+ /// can either be NSFW or SFW (see safety classifier)
+ SafetyClassification,
+ InsertionTimestamp,
+ RecipeFirstIngredientTagId,
+ Keywords,
+}
+
+from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
+[
+ Title,
+ CleanBody,
+ StemmedTitle,
+ StemmedCleanBody,
+ AllBody,
+ Url,
+ UrlNoTokenizer,
+ UrlForSiteOperator,
+ SiteWithout,
+ Domain,
+ SiteNoTokenizer,
+ DomainNoTokenizer,
+ DomainNameNoTokenizer,
+ SiteIfHomepageNoTokenizer,
+ DomainIfHomepage,
+ DomainNameIfHomepageNoTokenizer,
+ DomainIfHomepageNoTokenizer,
+ TitleIfHomepage,
+ BacklinkText,
+ Description,
+ DmozDescription,
+ SchemaOrgJson,
+ FlattenedSchemaOrgJson,
+ CleanBodyBigrams,
+ TitleBigrams,
+ CleanBodyTrigrams,
+ TitleTrigrams,
+ MicroformatTags,
+ SafetyClassification,
+ InsertionTimestamp,
+ RecipeFirstIngredientTagId,
+ Keywords,
+]);
+
+impl TextFieldEnum {
+ pub fn num_variants() -> usize {
+ TextFieldEnumDiscriminants::VARIANTS.len()
+ }
+
+ pub fn all() -> impl Iterator<Item = TextFieldEnum> {
+ TextFieldEnumDiscriminants::VARIANTS
+ .iter()
+ .copied()
+ .map(|v| v.into())
+ }
+
+ pub fn get(field_id: usize) -> Option<TextFieldEnum> {
+ TextFieldEnumDiscriminants::VARIANTS
+ .get(field_id)
+ .copied()
+ .map(TextFieldEnum::from)
+ }
+}
+
+impl InsertEnumMapKey for TextFieldEnum {
+ fn into_usize(self) -> usize {
+ TextFieldEnumDiscriminants::from(self) as usize
+ }
+}
+
+fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
+ match lang {
+ Lang::Ara => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Arabic),
+ Lang::Dan => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Danish),
+ Lang::Nld => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Dutch),
+ Lang::Fin => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Finnish),
+ Lang::Fra => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::French),
+ Lang::Deu => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::German),
+ Lang::Ell => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Greek),
+ Lang::Hun => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Hungarian),
+ Lang::Ita => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Italian),
+ Lang::Por => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Portuguese),
+ Lang::Ron => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Romanian),
+ Lang::Rus => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Russian),
+ Lang::Spa => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Spanish),
+ Lang::Swe => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Swedish),
+ Lang::Tam => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Tamil),
+ Lang::Tur => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Turkish),
+ _ => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::English),
+ }
+}
+
+fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {
+ let stemmer = stemmer_from_lang(&lang);
+ for token in tokens {
+ // TODO remove allocation
+ if let Ok(stemmed_str) = std::panic::catch_unwind(|| stemmer.stem(&token.text).into_owned())
+ {
+ token.text.clear();
+ token.text.push_str(&stemmed_str);
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Title;
+impl TextField for Title {
+ fn name(&self) -> &str {
+ "title"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache
+ .pretokenize_title()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?,
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct CleanBody;
+impl TextField for CleanBody {
+ fn name(&self) -> &str {
+ "body"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache.pretokenize_clean_text().clone(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct StemmedTitle;
+impl TextField for StemmedTitle {
+ fn name(&self) -> &str {
+ "stemmed_title"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::new_stemmed()
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let title = cache
+ .pretokenize_title()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?;
+ let mut tokens = title.tokens.clone();
+ stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: title.text.clone(),
+ tokens,
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct StemmedCleanBody;
+impl TextField for StemmedCleanBody {
+ fn name(&self) -> &str {
+ "stemmed_body"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::new_stemmed()
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let clean_text = cache.pretokenize_clean_text();
+ let mut tokens = clean_text.tokens.clone();
+ stem_tokens(&mut tokens, html.lang().copied().unwrap_or(Lang::Eng));
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: clean_text.text.clone(),
+ tokens,
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct AllBody;
+impl TextField for AllBody {
+ fn name(&self) -> &str {
+ "all_body"
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let all_text = cache
+ .pretokenize_all_text()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+ doc.add_pre_tokenized_text(self.tantivy_field(schema), all_text.clone());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Url;
+impl TextField for Url {
+ fn name(&self) -> &str {
+ "url"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let url = cache.pretokenize_url();
+ doc.add_pre_tokenized_text(self.tantivy_field(schema), url.clone());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlNoTokenizer;
+impl TextField for UrlNoTokenizer {
+ fn name(&self) -> &str {
+ "url_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let url = html.url().to_string();
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: url.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: url.len(),
+ position: 0,
+ text: url,
+ position_length: 1,
+ }],
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UrlForSiteOperator;
+impl TextField for UrlForSiteOperator {
+ fn name(&self) -> &str {
+ "url_for_site_operator"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::SiteOperator(SiteOperatorUrlTokenizer)
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache.pretokenize_url_for_site_operator().clone(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SiteWithout;
+impl TextField for SiteWithout {
+ fn name(&self) -> &str {
+ "site"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(self.tantivy_field(schema), cache.pretokenize_site().clone());
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Domain;
+impl TextField for Domain {
+ fn name(&self) -> &str {
+ "domain"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache.pretokenize_domain().clone(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SiteNoTokenizer;
+impl TextField for SiteNoTokenizer {
+ fn name(&self) -> &str {
+ "site_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let site = cache.pretokenize_site();
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: site.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: site.text.len(),
+ position: 0,
+ text: site.text.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainNoTokenizer;
+impl TextField for DomainNoTokenizer {
+ fn name(&self) -> &str {
+ "domain_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let domain = cache.pretokenize_domain();
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: domain.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain.text.len(),
+ position: 0,
+ text: domain.text.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainNameNoTokenizer;
+impl TextField for DomainNameNoTokenizer {
+ fn name(&self) -> &str {
+ "domain_name_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let domain_name = cache.domain_name();
+
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: domain_name.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain_name.len(),
+ position: 0,
+ text: domain_name.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SiteIfHomepageNoTokenizer;
+impl TextField for SiteIfHomepageNoTokenizer {
+ fn name(&self) -> &str {
+ "site_if_homepage_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let site = cache.pretokenize_site();
+
+ if html.is_homepage() {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: site.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: site.text.len(),
+ position: 0,
+ text: site.text.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+ } else {
+ doc.add_text(self.tantivy_field(schema), "");
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainIfHomepage;
+impl TextField for DomainIfHomepage {
+ fn name(&self) -> &str {
+ "domain_if_homepage"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let domain = cache.pretokenize_domain();
+ if html.is_homepage() {
+ doc.add_text(self.tantivy_field(schema), domain.text.clone());
+ } else {
+ doc.add_text(self.tantivy_field(schema), "");
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainNameIfHomepageNoTokenizer;
+impl TextField for DomainNameIfHomepageNoTokenizer {
+ fn name(&self) -> &str {
+ "domain_name_if_homepage_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let domain_name = cache.domain_name();
+
+ if html.is_homepage() {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: domain_name.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain_name.len(),
+ position: 0,
+ text: domain_name.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+ } else {
+ doc.add_text(self.tantivy_field(schema), "");
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DomainIfHomepageNoTokenizer;
+impl TextField for DomainIfHomepageNoTokenizer {
+ fn name(&self) -> &str {
+ "domain_if_homepage_no_tokenizer"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let domain = cache.pretokenize_domain();
+
+ if html.is_homepage() {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ PreTokenizedString {
+ text: domain.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain.text.len(),
+ position: 0,
+ text: domain.text.clone(),
+ position_length: 1,
+ }],
+ },
+ );
+ } else {
+ doc.add_text(self.tantivy_field(schema), "");
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleIfHomepage;
+impl TextField for TitleIfHomepage {
+ fn name(&self) -> &str {
+ "title_if_homepage"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let title = cache
+ .pretokenize_title()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+ if html.is_homepage() {
+ doc.add_pre_tokenized_text(self.tantivy_field(schema), title);
+ } else {
+ doc.add_text(self.tantivy_field(schema), "");
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct BacklinkText;
+impl TextField for BacklinkText {
+ fn name(&self) -> &str {
+ "backlink_text"
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema),
+ webpage.backlink_labels.join("\n"),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Description;
+impl TextField for Description {
+ fn name(&self) -> &str {
+ "description"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let description = cache.pretokenize_description();
+ doc.add_pre_tokenized_text(self.tantivy_field(schema), description.clone());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DmozDescription;
+impl TextField for DmozDescription {
+ fn name(&self) -> &str {
+ "dmoz_description"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema),
+ webpage.dmoz_description().unwrap_or_default(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SchemaOrgJson;
+impl TextField for SchemaOrgJson {
+ fn name(&self) -> &str {
+ "schema_org_json"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(self.tantivy_field(schema), cache.schema_json());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct FlattenedSchemaOrgJson;
+impl TextField for FlattenedSchemaOrgJson {
+ fn name(&self) -> &str {
+ "flattened_schema_org_json"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Json(JsonField)
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache.pretokenized_schema_json().clone(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct CleanBodyBigrams;
+impl TextField for CleanBodyBigrams {
+ fn name(&self) -> &str {
+ "clean_body_bigrams"
+ }
+
+ fn ngram_size(&self) -> usize {
+ 2
+ }
+
+ fn monogram_field(&self) -> TextFieldEnum {
+ CleanBody.into()
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Bigram(BigramTokenizer::default())
+ }
+
+ fn query_tokenizer(&self) -> Tokenizer {
+ Tokenizer::default()
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema),
+ html.clean_text().cloned().unwrap_or_default(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleBigrams;
+impl TextField for TitleBigrams {
+ fn name(&self) -> &str {
+ "title_bigrams"
+ }
+
+ fn ngram_size(&self) -> usize {
+ 2
+ }
+
+ fn monogram_field(&self) -> TextFieldEnum {
+ Title.into()
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Bigram(BigramTokenizer::default())
+ }
+
+ fn query_tokenizer(&self) -> Tokenizer {
+ Tokenizer::default()
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let title = cache
+ .pretokenize_title()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+ doc.add_text(self.tantivy_field(schema), title.text.clone());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct CleanBodyTrigrams;
+impl TextField for CleanBodyTrigrams {
+ fn name(&self) -> &str {
+ "clean_body_trigrams"
+ }
+
+ fn ngram_size(&self) -> usize {
+ 3
+ }
+
+ fn monogram_field(&self) -> TextFieldEnum {
+ CleanBody.into()
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Trigram(TrigramTokenizer::default())
+ }
+
+ fn query_tokenizer(&self) -> Tokenizer {
+ Tokenizer::default()
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ html: &Html,
+ _cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema),
+ html.clean_text().cloned().unwrap_or_default(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct TitleTrigrams;
+impl TextField for TitleTrigrams {
+ fn name(&self) -> &str {
+ "title_trigrams"
+ }
+
+ fn ngram_size(&self) -> usize {
+ 3
+ }
+
+ fn monogram_field(&self) -> TextFieldEnum {
+ Title.into()
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Trigram(TrigramTokenizer::default())
+ }
+
+ fn query_tokenizer(&self) -> Tokenizer {
+ Tokenizer::default()
+ }
+
+ fn is_searchable(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let title = cache
+ .pretokenize_title()
+ .as_ref()
+ .map(Clone::clone)
+ .map_err(|e| anyhow::anyhow!("{}", e))?;
+
+ doc.add_text(self.tantivy_field(schema), title.text.clone());
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct MicroformatTags;
+impl TextField for MicroformatTags {
+ fn name(&self) -> &str {
+ "microformat_tags"
+ }
+
+ fn has_pos(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_pre_tokenized_text(
+ self.tantivy_field(schema),
+ cache.pretokenize_microformats().clone(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SafetyClassification;
+impl TextField for SafetyClassification {
+ fn name(&self) -> &str {
+ "safety_classification"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ let safety = webpage
+ .safety_classification
+ .map(|label| label.to_string())
+ .unwrap_or_default();
+
+ doc.add_text(self.tantivy_field(schema), safety);
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct InsertionTimestamp;
+impl TextField for InsertionTimestamp {
+ fn name(&self) -> &str {
+ "insertion_timestamp"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn indexing_option(&self) -> IndexingOption {
+ IndexingOption::DateTime(tantivy::schema::DateOptions::default().set_indexed())
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_date(
+ self.tantivy_field(schema),
+ tantivy::DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
+ webpage.inserted_at.timestamp(),
+ )?),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct RecipeFirstIngredientTagId;
+impl TextField for RecipeFirstIngredientTagId {
+ fn name(&self) -> &str {
+ "recipe_first_ingredient_tag_id"
+ }
+
+ fn indexing_tokenizer(&self) -> Tokenizer {
+ Tokenizer::Identity(Identity {})
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ cache: &mut FnCache,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(
+ self.tantivy_field(schema),
+ cache.first_ingredient_tag_id().cloned().unwrap_or_default(),
+ );
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Keywords;
+impl TextField for Keywords {
+ fn name(&self) -> &str {
+ "keywords"
+ }
+
+ fn is_stored(&self) -> bool {
+ true
+ }
+
+ fn add_html_tantivy(
+ &self,
+ _html: &Html,
+ _cache: &mut FnCache,
+ _doc: &mut TantivyDocument,
+ _schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ Ok(())
+ }
+
+ fn add_webpage_tantivy(
+ &self,
+ webpage: &crate::webpage::Webpage,
+ doc: &mut TantivyDocument,
+ schema: &tantivy::schema::Schema,
+ ) -> Result<()> {
+ doc.add_text(self.tantivy_field(schema), webpage.keywords.join("\n"));
+
+ Ok(())
+ }
+}
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 00e9e809..c733222f 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -32,7 +32,7 @@ use crate::inverted_index::RetrievedWebpage;
use crate::models::dual_encoder::DualEncoder;
use crate::ranking::models::cross_encoder::CrossEncoderModel;
use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};
-use crate::ranking::ALL_SIGNALS;
+use crate::ranking::Signal;
use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
use crate::widgets::{Widget, Widgets};
@@ -145,9 +145,9 @@ pub fn combine_results(
}
pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebpagePointer]) {
for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {
- let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
+ let mut signals = HashMap::new();
- for signal in ALL_SIGNALS {
+ for signal in Signal::all() {
if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {
signals.insert(signal, *signal_value);
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 42dde1b4..df372fb4 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -29,7 +29,7 @@ use crate::ranking::inbound_similarity::InboundSimilarity;
use crate::ranking::models::lambdamart::LambdaMART;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage};
-use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};
+use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator};
use crate::search_ctx::Ctx;
use crate::search_prettifier::DisplayedWebpage;
use crate::webgraph::Node;
@@ -409,7 +409,7 @@ where
for (webpage, ranking) in webpages.iter_mut().zip(top_websites) {
let mut ranking_signals = HashMap::new();
- for signal in ALL_SIGNALS {
+ for signal in Signal::all() {
if let Some(score) = ranking.ranking.signals.get(signal) {
ranking_signals.insert(signal, *score);
}
diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rs
new file mode 100644
index 00000000..fee3abae
--- /dev/null
+++ b/crates/core/src/webpage/html/fn_cache.rs
@@ -0,0 +1,136 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{webpage::schema_org, Result};
+use tantivy::tokenizer::PreTokenizedString;
+
+use super::{find_recipe_first_ingredient_tag_id, Html};
+
+#[macro_export]
+macro_rules! cache {
+ ($($fn:ident -> $res:ty),*$(,)?) => {
+ /// Dynamically compute the different webpage functions
+ /// and cache the results for subsequent calls.
+ ///
+ /// Used during indexing as some of the fields use
+ /// the same data from the webpage and we don't want to
+ /// recompute the same data multiple times.
+ pub struct FnCache<'a> {
+ html: &'a Html,
+ first_ingredient_tag_id: Option<String>,
+ schema_json: Option<String>,
+ pretokenized_schema_json: Option<PreTokenizedString>,
+ $($fn: Option<$res>,)*
+ }
+
+ impl<'a> FnCache<'a> {
+ /// Create a new instance of the IndexingCacher
+ pub fn new(html: &'a Html) -> Self {
+ Self {
+ html,
+ first_ingredient_tag_id: None,
+ schema_json: None,
+ pretokenized_schema_json: None,
+ $($fn: None,)*
+ }
+ }
+
+ $(
+ /// Compute $fn from webpage and cache the result
+ pub fn $fn(&mut self) -> &$res {
+ if self.$fn.is_none() {
+ self.$fn = Some(self.html.$fn());
+ }
+
+ self.$fn.as_ref().unwrap()
+ }
+ )*
+ }
+ };
+}
+
+cache! {
+ pretokenize_title -> Result<PreTokenizedString>,
+ pretokenize_all_text -> Result<PreTokenizedString>,
+ pretokenize_clean_text -> PreTokenizedString,
+ pretokenize_url -> PreTokenizedString,
+ pretokenize_url_for_site_operator -> PreTokenizedString,
+ pretokenize_domain -> PreTokenizedString,
+ pretokenize_site -> PreTokenizedString,
+ pretokenize_description -> PreTokenizedString,
+ pretokenize_microformats -> PreTokenizedString,
+ domain_name -> String,
+ schema_org -> Vec<schema_org::Item>,
+ site_hash -> [u64; 2],
+ url_without_query_hash -> [u64; 2],
+ url_hash -> [u64; 2],
+ url_without_tld_hash -> [u64; 2],
+ domain_hash -> [u64; 2],
+ title_hash -> [u64; 2],
+}
+
+/// Some manual implementations so we can use previously cached data
+/// to compute the next field.
+impl<'a> FnCache<'a> {
+ pub fn first_ingredient_tag_id(&mut self) -> Option<&String> {
+ if self.first_ingredient_tag_id.is_none() {
+ let root = self.html.root.clone(); // Node is just a NodeRef, so it's cheap to clone
+
+ self.first_ingredient_tag_id =
+ find_recipe_first_ingredient_tag_id(self.schema_org().as_slice(), &root);
+ }
+
+ self.first_ingredient_tag_id.as_ref()
+ }
+
+ pub fn schema_json(&mut self) -> &String {
+ if self.schema_json.is_none() {
+ self.schema_json = Some(serde_json::to_string(self.schema_org()).unwrap());
+ }
+
+ self.schema_json.as_ref().unwrap()
+ }
+
+ pub fn pretokenized_schema_json(&mut self) -> &PreTokenizedString {
+ if self.pretokenized_schema_json.is_none() {
+ self.pretokenized_schema_json =
+ match schema_org::flattened_json(self.schema_org().clone()) {
+ Ok(mut f) => {
+ let mut tokens = Vec::new();
+
+ {
+ let mut stream = f.token_stream();
+
+ while let Some(token) = stream.next() {
+ tokens.push(token.clone());
+ }
+ }
+
+ Some(PreTokenizedString {
+ text: f.text().to_string(),
+ tokens,
+ })
+ }
+ Err(_) => Some(PreTokenizedString {
+ text: String::new(),
+ tokens: Vec::new(),
+ }),
+ };
+ }
+
+ self.pretokenized_schema_json.as_ref().unwrap()
+ }
+}
diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rs
index 3263e75f..383da490 100644
--- a/crates/core/src/webpage/html/into_tantivy.rs
+++ b/crates/core/src/webpage/html/into_tantivy.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -18,8 +18,12 @@ use crate::{
ceil_char_boundary,
prehashed::hash,
rake::RakeModel,
- schema::{FastField, TextField},
- simhash, split_u128, tokenizer,
+ schema::{
+ fast_field::FastField,
+ text_field::{self, TextField},
+ TextFieldEnum,
+ },
+ split_u128, tokenizer,
webpage::url_ext::UrlExt,
Error, Result,
};
@@ -29,12 +33,12 @@ use tantivy::{
};
use whatlang::Lang;
-use super::{find_recipe_first_ingredient_tag_id, schema_org, Html};
+use super::{fn_cache::FnCache, Html};
-use crate::schema::{Field, FLOAT_SCALING};
+use crate::schema::Field;
impl Html {
- fn pretokenize_title(&self) -> Result<PreTokenizedString> {
+ pub fn pretokenize_title(&self) -> Result<PreTokenizedString> {
let title = self.title();
if title.is_none() {
@@ -42,10 +46,10 @@ impl Html {
}
let title = title.unwrap();
- Ok(self.pretokenize_string(title, TextField::Title))
+ Ok(self.pretokenize_string(title, text_field::Title.into()))
}
- fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
+ pub fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
let all_text = self.all_text();
if all_text.is_none() {
@@ -53,38 +57,45 @@ impl Html {
}
let all_text = all_text.unwrap();
- Ok(self.pretokenize_string(all_text, TextField::AllBody))
+ Ok(self.pretokenize_string(all_text, text_field::AllBody.into()))
}
- fn pretokenize_clean_text(&self) -> PreTokenizedString {
+ pub fn pretokenize_clean_text(&self) -> PreTokenizedString {
let clean_text = self.clean_text().cloned().unwrap_or_default();
- self.pretokenize_string(clean_text, TextField::CleanBody)
+ self.pretokenize_string(clean_text, text_field::CleanBody.into())
}
- fn pretokenize_url(&self) -> PreTokenizedString {
+ pub fn pretokenize_url(&self) -> PreTokenizedString {
let url = self.url().to_string();
- self.pretokenize_string(url, TextField::Url)
+ self.pretokenize_string(url, text_field::Url.into())
}
- fn pretokenize_domain(&self) -> PreTokenizedString {
+ pub fn pretokenize_url_for_site_operator(&self) -> PreTokenizedString {
+ self.pretokenize_string_with(
+ self.url().to_string(),
+ tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
+ )
+ }
+
+ pub fn pretokenize_domain(&self) -> PreTokenizedString {
let domain = self.url().root_domain().unwrap_or_default().to_string();
- self.pretokenize_string(domain, TextField::Domain)
+ self.pretokenize_string(domain, text_field::Domain.into())
}
- fn pretokenize_site(&self) -> PreTokenizedString {
+ pub fn pretokenize_site(&self) -> PreTokenizedString {
let site = self.url().normalized_host().unwrap_or_default().to_string();
- self.pretokenize_string(site, TextField::SiteWithout)
+ self.pretokenize_string(site, text_field::SiteWithout.into())
}
- fn pretokenize_description(&self) -> PreTokenizedString {
+ pub fn pretokenize_description(&self) -> PreTokenizedString {
let text = self.description().unwrap_or_default();
- self.pretokenize_string(text, TextField::Description)
+ self.pretokenize_string(text, text_field::Description.into())
}
- fn pretokenize_microformats(&self) -> PreTokenizedString {
+ pub fn pretokenize_microformats(&self) -> PreTokenizedString {
let mut text = String::new();
for microformat in self.microformats().iter() {
@@ -92,10 +103,10 @@ impl Html {
text.push(' ');
}
- self.pretokenize_string(text, TextField::MicroformatTags)
+ self.pretokenize_string(text, text_field::MicroformatTags.into())
}
- fn pretokenize_string(&self, text: String, field: TextField) -> PreTokenizedString {
+ fn pretokenize_string(&self, text: String, field: TextFieldEnum) -> PreTokenizedString {
self.pretokenize_string_with(text, field.indexing_tokenizer())
}
@@ -118,6 +129,17 @@ impl Html {
PreTokenizedString { text, tokens }
}
+ pub fn domain_name(&self) -> String {
+ let domain = self.url().domain().unwrap_or_default();
+ self.url()
+ .root_domain()
+ .unwrap_or_default()
+ .find('.')
+ .map(|index| &domain[..ceil_char_boundary(&domain, index).min(domain.len())])
+ .unwrap_or_default()
+ .to_string()
+ }
+
pub fn keywords(&self, rake: &RakeModel) -> Vec<String> {
self.clean_text()
.map(|text| {
@@ -129,70 +151,18 @@ impl Html {
.unwrap_or_default()
}
- pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
- let mut doc = TantivyDocument::new();
-
- let title = self.pretokenize_title()?;
- let all_text = self.pretokenize_all_text()?;
- let clean_text = self.pretokenize_clean_text();
- let url = self.pretokenize_url();
- let domain = self.pretokenize_domain();
- let site = self.pretokenize_site();
- let description = self.pretokenize_description();
- let microformats = self.pretokenize_microformats();
- let url_for_site_operator = self.pretokenize_string_with(
- self.url().to_string(),
- tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
- );
-
- let domain_name = self
- .url()
- .root_domain()
- .unwrap_or_default()
- .find('.')
- .map(|index| {
- &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]
- })
- .unwrap_or_default()
- .to_string();
-
- let schemas: Vec<_> = self.schema_org();
- let first_ingredient_tag_id =
- find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();
-
- let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
-
- let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
- Ok(mut f) => {
- let mut tokens = Vec::new();
-
- {
- let mut stream = f.token_stream();
-
- while let Some(token) = stream.next() {
- tokens.push(token.clone());
- }
- }
-
- PreTokenizedString {
- text: f.text().to_string(),
- tokens,
- }
- }
- Err(_) => PreTokenizedString {
- text: String::new(),
- tokens: Vec::new(),
- },
- };
-
- let site_hash = split_u128(hash(self.url().normalized_host().unwrap_or_default()).0);
+ pub fn site_hash(&self) -> [u64; 2] {
+ split_u128(hash(self.url().normalized_host().unwrap_or_default()).0)
+ }
+ pub fn url_without_query_hash(&self) -> [u64; 2] {
let mut url_without_query = self.url().clone();
url_without_query.set_query(None);
- let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
- let url_hash = split_u128(hash(self.url().as_str()).0);
+ split_u128(hash(url_without_query.as_str()).0)
+ }
+ pub fn url_without_tld_hash(&self) -> [u64; 2] {
let tld = self.url().tld().unwrap_or_default();
let url_without_tld = self
.url()
@@ -205,396 +175,35 @@ impl Html {
+ "?"
+ self.url().query().unwrap_or_default();
- let url_without_tld_hash = split_u128(hash(url_without_tld).0);
+ split_u128(hash(url_without_tld).0)
+ }
+
+ pub fn url_hash(&self) -> [u64; 2] {
+ split_u128(hash(self.url().as_str()).0)
+ }
- let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
- let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
+ pub fn domain_hash(&self) -> [u64; 2] {
+ split_u128(hash(self.url().root_domain().unwrap_or_default()).0)
+ }
+
+ pub fn title_hash(&self) -> [u64; 2] {
+ split_u128(hash(self.title().unwrap_or_default()).0)
+ }
+
+ pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+ let mut doc = TantivyDocument::new();
+ let mut cache = FnCache::new(self);
for field in schema
.fields()
.filter_map(|(field, _)| Field::get(field.field_id() as usize))
{
- let tantivy_field = schema
- .get_field(field.name())
- .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));
-
match field {
- Field::Text(TextField::Title) => {
- doc.add_pre_tokenized_text(tantivy_field, title.clone())
- }
- Field::Text(TextField::StemmedTitle) => {
- let mut tokens = title.tokens.clone();
- stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: title.text.clone(),
- tokens,
- },
- );
- }
- Field::Text(TextField::CleanBody) => {
- doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())
- }
- Field::Text(TextField::StemmedCleanBody) => {
- let mut tokens = clean_text.tokens.clone();
- stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: clean_text.text.clone(),
- tokens,
- },
- );
- }
- Field::Text(TextField::CleanBodyBigrams) => {
- doc.add_text(
- tantivy_field,
- self.clean_text().cloned().unwrap_or_default(),
- );
- }
- Field::Text(TextField::CleanBodyTrigrams) => {
- doc.add_text(
- tantivy_field,
- self.clean_text().cloned().unwrap_or_default(),
- );
- }
- Field::Text(TextField::TitleBigrams) => {
- doc.add_text(tantivy_field, title.text.clone());
- }
- Field::Text(TextField::TitleTrigrams) => {
- doc.add_text(tantivy_field, title.text.clone());
- }
- Field::Text(TextField::Description) => {
- doc.add_pre_tokenized_text(tantivy_field, description.clone());
- }
- Field::Text(TextField::Url) => {
- doc.add_pre_tokenized_text(tantivy_field, url.clone())
- }
- Field::Text(TextField::UrlForSiteOperator) => {
- doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())
- }
- Field::Text(TextField::UrlNoTokenizer) => {
- let url = self.url().to_string();
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: url.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: url.len(),
- position: 0,
- text: url,
- position_length: 1,
- }],
- },
- );
- }
- Field::Text(TextField::SiteWithout) => {
- doc.add_pre_tokenized_text(tantivy_field, site.clone())
- }
- Field::Text(TextField::Domain) => {
- doc.add_pre_tokenized_text(tantivy_field, domain.clone())
- }
- Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: site.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: site.text.len(),
- position: 0,
- text: site.text.clone(),
- position_length: 1,
- }],
- },
- ),
- Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: site.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: site.text.len(),
- position: 0,
- text: site.text.clone(),
- position_length: 1,
- }],
- },
- )
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain.text.len(),
- position: 0,
- text: domain.text.clone(),
- position_length: 1,
- }],
- },
- ),
- Field::Text(TextField::TitleIfHomepage) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(tantivy_field, title.clone());
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainIfHomepage) => {
- if self.is_homepage() {
- doc.add_text(tantivy_field, domain.text.clone());
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainNameNoTokenizer) => {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain_name.to_string(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain_name.len(),
- position: 0,
- text: domain_name.to_string(),
- position_length: 1,
- }],
- },
- );
- }
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain_name.to_string(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain_name.len(),
- position: 0,
- text: domain_name.to_string(),
- position_length: 1,
- }],
- },
- );
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain.text.len(),
- position: 0,
- text: domain.text.clone(),
- position_length: 1,
- }],
- },
- );
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::AllBody) => {
- doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
- }
- Field::Text(TextField::RecipeFirstIngredientTagId) => {
- doc.add_text(tantivy_field, first_ingredient_tag_id.clone());
- }
- Field::Text(TextField::SchemaOrgJson) => {
- doc.add_text(tantivy_field, schema_json.clone());
- }
- Field::Text(TextField::FlattenedSchemaOrgJson) => {
- doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
- }
- Field::Text(TextField::MicroformatTags) => {
- doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
- }
- Field::Fast(FastField::IsHomepage) => {
- doc.add_u64(tantivy_field, (self.is_homepage()).into());
- }
- Field::Fast(FastField::LastUpdated) => doc.add_u64(
- tantivy_field,
- self.updated_time()
- .map_or(0, |time| time.timestamp().max(0) as u64),
- ),
- Field::Fast(FastField::TrackerScore) => {
- doc.add_u64(tantivy_field, self.trackers().len() as u64)
- }
- Field::Fast(FastField::NumUrlTokens) => {
- doc.add_u64(tantivy_field, url.tokens.len() as u64)
- }
- Field::Fast(FastField::NumMicroformatTagsTokens) => {
- doc.add_u64(tantivy_field, microformats.tokens.len() as u64)
- }
- Field::Fast(FastField::NumTitleTokens) => {
- doc.add_u64(tantivy_field, title.tokens.len() as u64)
- }
- Field::Fast(FastField::NumCleanBodyTokens) => {
- doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)
- }
- Field::Fast(FastField::NumDescriptionTokens) => {
- doc.add_u64(tantivy_field, description.tokens.len() as u64)
- }
- Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
- doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)
- }
- Field::Fast(FastField::NumDomainTokens) => {
- doc.add_u64(tantivy_field, domain.tokens.len() as u64)
- }
- Field::Fast(FastField::NumFlattenedSchemaTokens) => {
- doc.add_u64(tantivy_field, pretokenized_schema_json.tokens.len() as u64)
- }
- Field::Fast(FastField::SiteHash1) => {
- doc.add_u64(tantivy_field, site_hash[0]);
- }
- Field::Fast(FastField::SiteHash2) => {
- doc.add_u64(tantivy_field, site_hash[1]);
- }
- Field::Fast(FastField::UrlWithoutQueryHash1) => {
- doc.add_u64(tantivy_field, url_without_query_hash[0]);
- }
- Field::Fast(FastField::UrlWithoutQueryHash2) => {
- doc.add_u64(tantivy_field, url_without_query_hash[1]);
- }
- Field::Fast(FastField::UrlHash1) => {
- doc.add_u64(tantivy_field, url_hash[0]);
- }
- Field::Fast(FastField::UrlHash2) => {
- doc.add_u64(tantivy_field, url_hash[1]);
- }
- Field::Fast(FastField::UrlWithoutTldHash1) => {
- doc.add_u64(tantivy_field, url_without_tld_hash[0]);
- }
- Field::Fast(FastField::UrlWithoutTldHash2) => {
- doc.add_u64(tantivy_field, url_without_tld_hash[1]);
- }
- Field::Fast(FastField::DomainHash1) => {
- doc.add_u64(tantivy_field, domain_hash[0]);
- }
- Field::Fast(FastField::DomainHash2) => {
- doc.add_u64(tantivy_field, domain_hash[1]);
- }
- Field::Fast(FastField::TitleHash1) => {
- doc.add_u64(tantivy_field, title_hash[0]);
- }
- Field::Fast(FastField::TitleHash2) => {
- doc.add_u64(tantivy_field, title_hash[1]);
- }
- Field::Fast(FastField::SimHash) => {
- let hash = if !clean_text.text.is_empty() {
- simhash::hash(&clean_text.text)
- } else {
- 0
- };
- doc.add_u64(tantivy_field, hash);
- }
- Field::Fast(FastField::NumPathAndQuerySlashes) => {
- let num_slashes = self
- .url()
- .path_segments()
- .map(|segments| segments.count())
- .unwrap_or(0);
-
- doc.add_u64(tantivy_field, num_slashes as u64);
- }
- Field::Fast(FastField::NumPathAndQueryDigits) => {
- let num_digits = self
- .url()
- .path()
- .chars()
- .filter(|c| c.is_ascii_digit())
- .count()
- + self
- .url()
- .query()
- .unwrap_or_default()
- .chars()
- .filter(|c| c.is_ascii_digit())
- .count();
-
- doc.add_u64(tantivy_field, num_digits as u64);
- }
- Field::Fast(FastField::LikelyHasAds) => {
- doc.add_u64(tantivy_field, self.likely_has_ads() as u64);
- }
- Field::Fast(FastField::LikelyHasPaywall) => {
- doc.add_u64(tantivy_field, self.likely_has_paywall() as u64);
- }
- Field::Fast(FastField::LinkDensity) => {
- doc.add_u64(
- tantivy_field,
- (self.link_density() * FLOAT_SCALING as f64) as u64,
- );
- }
- Field::Text(TextField::BacklinkText)
- | Field::Text(TextField::SafetyClassification)
- | Field::Text(TextField::InsertionTimestamp)
- | Field::Fast(FastField::HostCentrality)
- | Field::Fast(FastField::HostCentralityRank)
- | Field::Fast(FastField::PageCentrality)
- | Field::Fast(FastField::PageCentralityRank)
- | Field::Fast(FastField::FetchTimeMs)
- | Field::Fast(FastField::PreComputedScore)
- | Field::Fast(FastField::Region)
- | Field::Fast(FastField::HostNodeID)
- | Field::Fast(FastField::TitleEmbeddings)
- | Field::Fast(FastField::KeywordEmbeddings)
- | Field::Text(TextField::Keywords)
- | Field::Text(TextField::DmozDescription) => {}
+ Field::Text(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
+ Field::Fast(f) => f.add_html_tantivy(self, &mut cache, &mut doc, schema)?,
}
}
Ok(doc)
}
}
-
-fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
- match lang {
- Lang::Ara => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Arabic),
- Lang::Dan => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Danish),
- Lang::Nld => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Dutch),
- Lang::Fin => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Finnish),
- Lang::Fra => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::French),
- Lang::Deu => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::German),
- Lang::Ell => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Greek),
- Lang::Hun => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Hungarian),
- Lang::Ita => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Italian),
- Lang::Por => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Portuguese),
- Lang::Ron => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Romanian),
- Lang::Rus => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Russian),
- Lang::Spa => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Spanish),
- Lang::Swe => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Swedish),
- Lang::Tam => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Tamil),
- Lang::Tur => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Turkish),
- _ => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::English),
- }
-}
-
-fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {
- let stemmer = stemmer_from_lang(&lang);
- for token in tokens {
- // TODO remove allocation
- if let Ok(stemmed_str) = std::panic::catch_unwind(|| stemmer.stem(&token.text).into_owned())
- {
- token.text.clear();
- token.text.push_str(&stemmed_str);
- }
- }
-}
diff --git a/crates/core/src/webpage/html/microformats.rs b/crates/core/src/webpage/html/microformats.rs
index 9ecba223..8e310e14 100644
--- a/crates/core/src/webpage/html/microformats.rs
+++ b/crates/core/src/webpage/html/microformats.rs
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{enum_map::EnumSet, Result};
+use crate::enum_map::{EnumSet, GetEnumMapKey, InsertEnumMapKey};
use super::Html;
@@ -49,9 +49,9 @@ impl Microformat {
}
}
-impl From<Microformat> for usize {
- fn from(value: Microformat) -> Self {
- match value {
+impl InsertEnumMapKey for Microformat {
+ fn into_usize(self) -> usize {
+ match self {
Microformat::HCard => 0,
Microformat::HEvent => 1,
Microformat::HEntry => 2,
@@ -62,18 +62,16 @@ impl From<Microformat> for usize {
}
}
-impl TryFrom<usize> for Microformat {
- type Error = anyhow::Error;
-
- fn try_from(value: usize) -> Result<Self> {
+impl GetEnumMapKey for Microformat {
+ fn from_usize(value: usize) -> Option<Self> {
match value {
- 0 => Ok(Microformat::HCard),
- 1 => Ok(Microformat::HEvent),
- 2 => Ok(Microformat::HEntry),
- 3 => Ok(Microformat::HRecipe),
- 4 => Ok(Microformat::HReview),
- 5 => Ok(Microformat::HProduct),
- _ => Err(anyhow::anyhow!("Unknown microformat")),
+ 0 => Some(Microformat::HCard),
+ 1 => Some(Microformat::HEvent),
+ 2 => Some(Microformat::HEntry),
+ 3 => Some(Microformat::HRecipe),
+ 4 => Some(Microformat::HReview),
+ 5 => Some(Microformat::HProduct),
+ _ => None,
}
}
}
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 0b93b31e..b7faf546 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -28,6 +28,9 @@ use super::{adservers::AD_SERVERS, schema_org, Meta, Script};
use super::url_ext::UrlExt;
+pub use fn_cache::FnCache;
+
+mod fn_cache;
mod into_tantivy;
mod links;
mod microformats;
diff --git a/crates/core/src/webpage/html/robots_meta.rs b/crates/core/src/webpage/html/robots_meta.rs
index 08707624..b65adb11 100644
--- a/crates/core/src/webpage/html/robots_meta.rs
+++ b/crates/core/src/webpage/html/robots_meta.rs
@@ -16,7 +16,10 @@
use std::str::FromStr;
-use crate::{enum_map::EnumSet, Error, Result};
+use crate::{
+ enum_map::{EnumSet, InsertEnumMapKey},
+ Error, Result,
+};
use super::Html;
@@ -38,9 +41,9 @@ impl FromStr for RobotsMeta {
}
}
-impl From<RobotsMeta> for usize {
- fn from(val: RobotsMeta) -> Self {
- match val {
+impl InsertEnumMapKey for RobotsMeta {
+ fn into_usize(self) -> usize {
+ match self {
RobotsMeta::NoIndex => 0,
RobotsMeta::NoFollow => 1,
}
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index 03c88a8b..ea507f42 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- schema::{FastField, TextField},
+ schema::{fast_field::FastField, text_field::TextField, Field},
webgraph::NodeID,
Result,
};
@@ -23,15 +23,11 @@ use candle_core::Tensor;
use chrono::{DateTime, Utc};
use std::collections::HashMap;
-use tantivy::{time::OffsetDateTime, TantivyDocument};
+use tantivy::TantivyDocument;
use url::Url;
-use crate::schema::{Field, FLOAT_SCALING};
-
-use self::region::Region;
-
mod adservers;
-mod html;
+pub mod html;
mod just_text;
pub mod region;
pub mod safety_classifier;
@@ -114,7 +110,7 @@ impl Webpage {
})
}
- fn dmoz_description(&self) -> Option<String> {
+ pub fn dmoz_description(&self) -> Option<String> {
self.dmoz_description.as_ref().and_then(|desc| {
if !self.html.metadata().iter().any(|metadata| {
if let Some(content) = metadata.get(&"content".to_string()) {
@@ -131,172 +127,15 @@ impl Webpage {
}
pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
- let region = Region::guess_from(self);
-
- let dmoz_description = self.dmoz_description();
-
let mut doc = self.html.as_tantivy(schema)?;
- if let Ok(region) = region {
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::Region).name())
- .expect("Failed to get region field"),
- region.id(),
- );
- } else {
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::Region).name())
- .expect("Failed to get region field"),
- Region::All.id(),
- );
- }
-
- let backlink_text: String =
- itertools::intersperse(self.backlink_labels.clone(), "\n".to_string()).collect();
-
- doc.add_text(
- schema
- .get_field(Field::Text(TextField::BacklinkText).name())
- .expect("Failed to get backlink-text field"),
- backlink_text,
- );
-
- doc.add_text(
- schema
- .get_field(Field::Text(TextField::Keywords).name())
- .expect("Failed to get keywords field"),
- self.keywords.join("\n"),
- );
-
- doc.add_date(
- schema
- .get_field(Field::Text(TextField::InsertionTimestamp).name())
- .expect("Failed to get insertion-timestamp field"),
- tantivy::DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
- self.inserted_at.timestamp(),
- )?),
- );
-
- let safety = self
- .safety_classification
- .map(|label| label.to_string())
- .unwrap_or_default();
-
- doc.add_text(
- schema
- .get_field(Field::Text(TextField::SafetyClassification).name())
- .expect("Failed to get safety_classification field"),
- safety,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostCentrality).name())
- .expect("Failed to get host_centrality field"),
- (self.host_centrality * FLOAT_SCALING as f64) as u64,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostCentralityRank).name())
- .expect("Failed to get host_centrality_rank field"),
- self.host_centrality_rank,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::PageCentrality).name())
- .expect("Failed to get page_centrality field"),
- (self.page_centrality * FLOAT_SCALING as f64) as u64,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::PageCentralityRank).name())
- .expect("Failed to get page_centrality_rank field"),
- self.page_centrality_rank,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::FetchTimeMs).name())
- .expect("Failed to get fetch_time_ms field"),
- self.fetch_time_ms,
- );
-
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::PreComputedScore).name())
- .expect("failed to get pre_computed_score field"),
- (self.pre_computed_score * FLOAT_SCALING as f64) as u64,
- );
-
- if let Some(emb) = &self.title_embedding {
- let mut serialized = Vec::new();
- emb.write_bytes(&mut serialized)?;
-
- doc.add_bytes(
- schema
- .get_field(Field::Fast(FastField::TitleEmbeddings).name())
- .expect("Failed to get title_embeddings field"),
- serialized,
- );
- } else {
- doc.add_bytes(
- schema
- .get_field(Field::Fast(FastField::TitleEmbeddings).name())
- .expect("Failed to get title_embeddings field"),
- Vec::new(),
- );
- }
-
- if let Some(emb) = &self.keyword_embedding {
- let mut serialized = Vec::new();
- emb.write_bytes(&mut serialized)?;
-
- doc.add_bytes(
- schema
- .get_field(Field::Fast(FastField::KeywordEmbeddings).name())
- .expect("Failed to get keyword_embeddings field"),
- serialized,
- );
- } else {
- doc.add_bytes(
- schema
- .get_field(Field::Fast(FastField::KeywordEmbeddings).name())
- .expect("Failed to get keyword_embeddings field"),
- Vec::new(),
- );
- }
-
- match &self.node_id {
- Some(node_id) => {
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostNodeID).name())
- .expect("Failed to get node_id field"),
- node_id.as_u64(),
- );
- }
- None => {
- doc.add_u64(
- schema
- .get_field(Field::Fast(FastField::HostNodeID).name())
- .expect("Failed to get node_id field"),
- u64::MAX,
- );
+ for field in Field::all() {
+ match field {
+ Field::Fast(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
+ Field::Text(f) => f.add_webpage_tantivy(self, &mut doc, schema)?,
}
}
- doc.add_text(
- schema
- .get_field(Field::Text(TextField::DmozDescription).name())
- .expect("failed to get dmoz_description field"),
- dmoz_description.unwrap_or_default(),
- );
-
Ok(doc)
}
}