Skip to content

web search done right

remove 'FieldMapping' and move functionality into non-self functions under 'Field' as this is more intutive

Mikkel Denker9d3650d344ad
diff --git a/core/src/fastfield_reader.rs b/core/src/fastfield_reader.rs
index 311804dc..acf67408 100644
--- a/core/src/fastfield_reader.rs
+++ b/core/src/fastfield_reader.rs
@@ -20,7 +20,7 @@ use tantivy::{columnar::ColumnValues, DocId, SegmentId};
use crate::{
enum_map::EnumMap,
- schema::{DataType, FastField, FieldMapping},
+ schema::{DataType, FastField, Field},
};
#[derive(Default, Clone)]
@@ -48,7 +48,7 @@ impl FastFieldReader {
let mut field_readers = Vec::new();
- for field in FieldMapping::all().filter_map(|field| field.as_fast()) {
+ for field in Field::all().filter_map(|field| field.as_fast()) {
let field_reader = match field.data_type() {
DataType::U64 => {
let reader = fastfield_readers.u64(field.name()).unwrap();
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 5b55d8f9..5e61f64b 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -43,7 +43,7 @@ use crate::query::Query;
use crate::ranking::initial::Score;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::SignalAggregator;
-use crate::schema::{FastField, Field, FieldMapping, TextField};
+use crate::schema::{FastField, Field, TextField};
use crate::search_ctx::Ctx;
use crate::snippet::TextSnippet;
use crate::tokenizer::{
@@ -605,7 +605,7 @@ impl From<TantivyDocument> for RetrievedWebpage {
let mut webpage = RetrievedWebpage::default();
for value in doc.field_values() {
- match FieldMapping::get(value.field.field_id() as usize).copied() {
+ match Field::get(value.field.field_id() as usize).copied() {
Some(Field::Text(TextField::Title)) => {
webpage.title = value
.value()
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 7da75358..60ab8f18 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -22,7 +22,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIXES,
floor_char_boundary,
- schema::{Field, FieldMapping, TextField},
+ schema::{Field, TextField},
};
#[derive(Debug, Clone)]
@@ -103,7 +103,7 @@ fn simple_into_tantivy(
.iter()
.filter(|field| {
matches!(
- FieldMapping::get(field.field_id() as usize),
+ Field::get(field.field_id() as usize),
Some(Field::Text(TextField::AllBody))
| Some(Field::Text(TextField::Title))
| Some(Field::Text(TextField::Url))
@@ -153,8 +153,7 @@ impl Term {
for (field, tv_field) in fields
.iter()
.filter_map(|tv_field| {
- FieldMapping::get(tv_field.field_id() as usize)
- .map(|mapped| (mapped, *tv_field))
+ Field::get(tv_field.field_id() as usize).map(|mapped| (mapped, *tv_field))
})
.filter(|(field, _)| field.is_searchable())
.filter(|(field, _)| field.has_pos())
@@ -197,7 +196,7 @@ impl Term {
.iter()
.find(|field| {
matches!(
- FieldMapping::get(field.field_id() as usize),
+ Field::get(field.field_id() as usize),
Some(Field::Text(TextField::Title))
)
})
@@ -210,7 +209,7 @@ impl Term {
.iter()
.find(|field| {
matches!(
- FieldMapping::get(field.field_id() as usize),
+ Field::get(field.field_id() as usize),
Some(Field::Text(TextField::AllBody))
)
})
@@ -223,7 +222,7 @@ impl Term {
.iter()
.find(|field| {
matches!(
- FieldMapping::get(field.field_id() as usize),
+ Field::get(field.field_id() as usize),
Some(Field::Text(TextField::Url))
)
})
@@ -249,7 +248,7 @@ impl Term {
fields
.iter()
.filter_map(|tv_field| {
- FieldMapping::get(tv_field.field_id() as usize)
+ Field::get(tv_field.field_id() as usize)
.filter(|field| field.is_searchable())
.map(|_| tv_field)
})
@@ -265,7 +264,7 @@ impl Term {
.iter()
.filter(|field| {
matches!(
- FieldMapping::get(field.field_id() as usize),
+ Field::get(field.field_id() as usize),
Some(Field::Text(TextField::UrlForSiteOperator))
)
})
@@ -298,7 +297,7 @@ impl Term {
) -> Box<dyn tantivy::query::Query + 'static> {
let mut processed_terms = Term::process_tantivy_term(term, *field);
- let option = FieldMapping::get(field.field_id() as usize)
+ let option = Field::get(field.field_id() as usize)
.unwrap()
.as_text()
.unwrap()
@@ -329,7 +328,7 @@ impl Term {
term: &str,
tantivy_field: tantivy::schema::Field,
) -> Vec<tantivy::Term> {
- match FieldMapping::get(tantivy_field.field_id() as usize) {
+ match Field::get(tantivy_field.field_id() as usize) {
Some(Field::Fast(_)) => vec![tantivy::Term::from_field_text(tantivy_field, term)],
Some(Field::Text(text_field)) => {
let mut terms: Vec<tantivy::Term> = Vec::new();
diff --git a/core/src/query/pattern_query/weight.rs b/core/src/query/pattern_query/weight.rs
index 90b9d08a..b5d5b5ae 100644
--- a/core/src/query/pattern_query/weight.rs
+++ b/core/src/query/pattern_query/weight.rs
@@ -25,7 +25,7 @@ use tantivy::{
use crate::{
fastfield_reader::FastFieldReader,
ranking::bm25::Bm25Weight,
- schema::{FastField, Field, FieldMapping, TextField},
+ schema::{FastField, Field, TextField},
};
use super::scorer::{
@@ -64,7 +64,7 @@ impl FastSiteDomainPatternWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- let field_no_tokenizer = match FieldMapping::get(self.field.field_id() as usize) {
+ let field_no_tokenizer = match Field::get(self.field.field_id() as usize) {
Some(Field::Text(TextField::UrlForSiteOperator)) => {
Field::Text(TextField::SiteNoTokenizer)
}
@@ -170,7 +170,7 @@ impl PatternWeight {
return Ok(None);
}
- let num_tokens_fastfield = match FieldMapping::get(self.field.field_id() as usize) {
+ let num_tokens_fastfield = match Field::get(self.field.field_id() as usize) {
Some(Field::Text(TextField::Title)) => Ok(FastField::NumTitleTokens),
Some(Field::Text(TextField::CleanBody)) => Ok(FastField::NumCleanBodyTokens),
Some(Field::Text(TextField::Url)) => Ok(FastField::NumUrlTokens),
diff --git a/core/src/schema.rs b/core/src/schema.rs
index a714799b..e9853ee8 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -361,9 +361,7 @@ static ALL_FIELDS: [Field; 65] = [
Field::Fast(FastField::LikelyHasPaywall),
];
-pub struct FieldMapping;
-
-impl FieldMapping {
+impl Field {
#[inline]
pub fn get(field_id: usize) -> Option<&'static Field> {
ALL_FIELDS.get(field_id)
@@ -373,9 +371,6 @@ impl FieldMapping {
pub fn all() -> impl Iterator<Item = &'static Field> {
ALL_FIELDS.iter()
}
-}
-
-impl Field {
fn default_text_options(&self) -> tantivy::schema::TextOptions {
let tokenizer = self.as_text().unwrap().indexing_tokenizer();
let option = self.as_text().unwrap().index_option();
diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rs
index 9c09c57f..06f58910 100644
--- a/core/src/webpage/html/into_tantivy.rs
+++ b/core/src/webpage/html/into_tantivy.rs
@@ -17,7 +17,7 @@
use crate::{
ceil_char_boundary,
prehashed::hash,
- schema::{FastField, FieldMapping, TextField},
+ schema::{FastField, TextField},
simhash, split_u128, tokenizer,
webpage::url_ext::UrlExt,
Error, Result,
@@ -199,7 +199,7 @@ impl Html {
for field in schema
.fields()
- .filter_map(|(field, _)| FieldMapping::get(field.field_id() as usize))
+ .filter_map(|(field, _)| Field::get(field.field_id() as usize))
{
let tantivy_field = schema
.get_field(field.name())