Skip to content

web search done right

reduce fastfield indirections

Mikkel Denker10043e7db6a5
diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/fastfield_reader.rs
index c11a2f26..bbb94d5a 100644
--- a/crates/core/src/fastfield_reader.rs
+++ b/crates/core/src/fastfield_reader.rs
@@ -52,7 +52,9 @@ impl FastFieldReader {
let field_reader = match field.data_type() {
DataType::U64 => {
let reader = fastfield_readers.u64(field.name()).unwrap();
- FieldReader::U64(Arc::new(reader.values.iter().collect()))
+ FieldReader {
+ data: Arc::new(reader.values.iter().collect()),
+ }
}
};
@@ -75,39 +77,14 @@ impl FastFieldReader {
}
}
-pub enum FieldValue {
- U64(u64),
- U64s(Vec<u64>),
-}
-
-impl From<FieldValue> for Option<Vec<u64>> {
- fn from(val: FieldValue) -> Self {
- match val {
- FieldValue::U64s(vec) => Some(vec),
- _ => None,
- }
- }
-}
-
-impl From<FieldValue> for Option<u64> {
- fn from(val: FieldValue) -> Self {
- match val {
- FieldValue::U64(res) => Some(res),
- _ => None,
- }
- }
-}
-
#[derive(Clone)]
-pub enum FieldReader {
- U64(Arc<Vec<u64>>),
+pub struct FieldReader {
+ data: Arc<Vec<u64>>,
}
impl FieldReader {
- pub fn get(&self, doc: &DocId) -> FieldValue {
- match self {
- FieldReader::U64(reader) => FieldValue::U64(reader[*doc as usize]),
- }
+ pub fn get(&self, doc: &DocId) -> u64 {
+ self.data[*doc as usize]
}
}
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index bca36f20..0790c245 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -449,7 +449,7 @@ impl InvertedIndex {
let snippet = if let Some(description) = page.description.as_deref() {
let snip = description.split_whitespace().take(50).join(" ");
- if snip.split_ascii_whitespace().count() < 10 {
+ if snip.split_whitespace().count() < 10 {
page.body.split_whitespace().take(50).join(" ")
} else {
snip
@@ -464,8 +464,14 @@ impl InvertedIndex {
} else {
let min_body_len = if url.is_homepage() { 1024 } else { 256 };
- if page.body.len() < min_body_len
- && !page.description.as_deref().unwrap_or_default().is_empty()
+ if page.body.split_whitespace().count() < min_body_len
+ && page
+ .description
+ .as_deref()
+ .unwrap_or_default()
+ .split_whitespace()
+ .count()
+ >= 8
{
page.snippet = snippet::generate(
query,
diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rs
index 26d813d1..6dc5ddfe 100644
--- a/crates/core/src/ranking/signal.rs
+++ b/crates/core/src/ranking/signal.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::fastfield_reader::FieldValue;
use crate::query::optic::AsSearchableRule;
use crate::query::Query;
use crate::Result;
@@ -330,14 +329,14 @@ impl Signal {
})
}
- fn fastfield_value(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<FieldValue> {
+ fn fastfield_value(&self, aggregator: &SignalAggregator, doc: &DocId) -> Option<u64> {
aggregator.segment_reader.as_ref().and_then(|reader| {
- self.as_fastfield().map(|fast_field| {
+ self.as_fastfield().as_ref().map(|fast_field| {
reader
- .borrow_mut()
+ .borrow()
.fastfield_reader
- .get_field_reader(&fast_field)
- .get(&doc)
+ .get_field_reader(fast_field)
+ .get(doc)
})
})
}
@@ -351,28 +350,28 @@ impl Signal {
let value: Option<f64> = match self {
Signal::HostCentrality | Signal::PageCentrality => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|val| val as f64 / FLOAT_SCALING as f64)
}
Signal::HostCentralityRank | Signal::PageCentralityRank => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|val| val as f64).map(score_rank)
}
Signal::IsHomepage => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|val| val as f64)
}
Signal::LinkDensity => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value
@@ -381,7 +380,7 @@ impl Signal {
}
Signal::FetchTimeMs => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|v| v as usize).map(|fetch_time_ms| {
@@ -394,7 +393,7 @@ impl Signal {
}
Signal::UpdateTimestamp => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value
@@ -403,28 +402,28 @@ impl Signal {
}
Signal::TrackerScore => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|num_trackers| score_trackers(num_trackers as f64))
}
Signal::UrlDigits => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|num_digits| score_digits(num_digits as f64))
}
Signal::UrlSlashes => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value.map(|num_slashes| score_slashes(num_slashes as f64))
}
Signal::Region => {
let field_value: Option<u64> = self
- .fastfield_value(signal_aggregator, doc)
+ .fastfield_value(signal_aggregator, &doc)
.and_then(|val| val.into());
field_value