Skip to content

web search done right

edges get sorted by their segment id for performance when retrieving. assigning sort_scores with .zip would therefore be incorrect

Mikkel Denker91544e314d79
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 2ed132bc..226e5a55 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use itertools::Itertools;
use tantivy::query::ShortCircuitQuery;
use super::{
@@ -27,7 +26,7 @@ use crate::{
webgraph::{
doc_address::DocAddress,
document::Edge,
- schema::{Field, FromHostId, FromId, RelFlags, ToHostId, ToId},
+ schema::{Field, FromHostId, FromId, RelFlags, SortScore, ToHostId, ToId},
searcher::Searcher,
EdgeLimit, Node, NodeID, SmallEdge, SmallEdgeWithLabel,
},
@@ -38,10 +37,11 @@ pub fn fetch_small_edges<F: Field>(
searcher: &Searcher,
mut doc_ids: Vec<DocAddress>,
node_id_field: F,
-) -> Result<Vec<(NodeID, crate::webpage::RelFlags)>> {
+) -> Result<Vec<(NodeID, crate::webpage::RelFlags, f64)>> {
doc_ids.sort_unstable_by_key(|doc| doc.segment_ord);
let mut prev_segment_id = None;
let mut field_column = None;
+ let mut score_column = None;
let mut rel_flags_column = None;
let mut edges = Vec::with_capacity(doc_ids.len());
@@ -57,6 +57,7 @@ pub fn fetch_small_edges<F: Field>(
);
field_column = Some(segment_column_fields.u64(node_id_field).unwrap());
rel_flags_column = Some(segment_column_fields.u64(RelFlags).unwrap());
+ score_column = Some(segment_column_fields.f64(SortScore).unwrap());
}
let Some(id) = field_column.as_ref().unwrap().first(doc.doc_id) else {
@@ -65,8 +66,15 @@ pub fn fetch_small_edges<F: Field>(
let Some(rel_flags) = rel_flags_column.as_ref().unwrap().first(doc.doc_id) else {
continue;
};
+ let Some(sort_score) = score_column.as_ref().unwrap().first(doc.doc_id) else {
+ continue;
+ };
- edges.push((NodeID::from(id), crate::webpage::RelFlags::from(rel_flags)));
+ edges.push((
+ NodeID::from(id),
+ crate::webpage::RelFlags::from(rel_flags),
+ sort_score,
+ ));
}
Ok(edges)
@@ -109,7 +117,7 @@ impl BacklinksQuery {
impl Query for BacklinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
@@ -136,15 +144,20 @@ impl Query for BacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let nodes = fetch_small_edges(searcher, docs, FromId)?;
- Ok(scores
+ Ok(nodes
.into_iter()
- .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
- from: node,
- to: self.node,
- rel_flags,
- }))
+ .map(|(node, rel_flags, sort_score)| {
+ (
+ sort_score,
+ SmallEdge {
+ from: node,
+ to: self.node,
+ rel_flags,
+ },
+ )
+ })
.collect())
}
@@ -189,7 +202,7 @@ impl HostBacklinksQuery {
impl Query for HostBacklinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
@@ -229,15 +242,20 @@ impl Query for HostBacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let nodes = fetch_small_edges(searcher, docs, FromHostId)?;
- Ok(scores
+ Ok(nodes
.into_iter()
- .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
- from: node,
- to: self.node,
- rel_flags,
- }))
+ .map(|(node, rel_flags, score)| {
+ (
+ score,
+ SmallEdge {
+ from: node,
+ to: self.node,
+ rel_flags,
+ },
+ )
+ })
.collect())
}
@@ -284,7 +302,7 @@ impl FullBacklinksQuery {
impl Query for FullBacklinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, Edge)>;
+ type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
@@ -314,9 +332,9 @@ impl Query for FullBacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let edges = fetch_edges(searcher, docs)?;
- Ok(scores.into_iter().zip_eq(edges).collect())
+ Ok(edges)
}
fn filter_fruit_shards(
@@ -332,8 +350,8 @@ impl Query for FullBacklinksQuery {
fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
let mut edges: Vec<_> = results.into_iter().flatten().collect();
- edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
- edges.into_iter().map(|(_, e)| e).collect()
+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));
+ edges
}
}
@@ -362,7 +380,7 @@ impl FullHostBacklinksQuery {
impl Query for FullHostBacklinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, Edge)>;
+ type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
@@ -402,17 +420,16 @@ impl Query for FullHostBacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let edges = fetch_edges(searcher, docs)?;
- Ok(scores
+
+ Ok(edges
.into_iter()
- .zip_eq(edges.into_iter().map(|e| Edge {
+ .map(|e| Edge {
from: e.from.into_host(),
to: e.to.into_host(),
- rel_flags: e.rel_flags,
- label: e.label,
- sort_score: e.sort_score,
- }))
+ ..e
+ })
.collect())
}
@@ -429,8 +446,8 @@ impl Query for FullHostBacklinksQuery {
fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
let mut edges: Vec<_> = results.into_iter().flatten().collect();
- edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
- edges.into_iter().map(|(_, e)| e).collect()
+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));
+ edges
}
}
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
@@ -458,7 +475,7 @@ impl BacklinksWithLabelsQuery {
impl Query for BacklinksWithLabelsQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, SmallEdgeWithLabel)>;
+ type IntermediateOutput = Vec<(f64, SmallEdgeWithLabel)>;
type Output = Vec<SmallEdgeWithLabel>;
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
@@ -480,16 +497,21 @@ impl Query for BacklinksWithLabelsQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let edges = fetch_edges(searcher, docs)?;
- Ok(scores
+ Ok(edges
.into_iter()
- .zip_eq(edges.into_iter().map(|e| SmallEdgeWithLabel {
- from: e.from.id(),
- to: e.to.id(),
- rel_flags: e.rel_flags,
- label: e.label,
- }))
+ .map(|e| {
+ (
+ e.sort_score,
+ SmallEdgeWithLabel {
+ from: e.from.id(),
+ to: e.to.id(),
+ rel_flags: e.rel_flags,
+ label: e.label,
+ },
+ )
+ })
.collect())
}
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 689805c9..bef7dc2d 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -30,7 +30,6 @@ use crate::{
Result,
};
-use itertools::Itertools;
use tantivy::query::ShortCircuitQuery;
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
@@ -58,7 +57,7 @@ impl ForwardlinksQuery {
impl Query for ForwardlinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
@@ -96,15 +95,20 @@ impl Query for ForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let nodes = fetch_small_edges(searcher, docs, ToId)?;
- Ok(scores
+ Ok(nodes
.into_iter()
- .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
- from: self.node,
- to: node,
- rel_flags,
- }))
+ .map(|(node, rel_flags, score)| {
+ (
+ score,
+ SmallEdge {
+ from: self.node,
+ to: node,
+ rel_flags,
+ },
+ )
+ })
.collect())
}
@@ -140,7 +144,7 @@ impl HostForwardlinksQuery {
impl Query for HostForwardlinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
@@ -191,15 +195,20 @@ impl Query for HostForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let nodes = fetch_small_edges(searcher, docs, ToHostId)?;
- Ok(scores
+ Ok(nodes
.into_iter()
- .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
- from: self.node,
- to: node,
- rel_flags,
- }))
+ .map(|(node, rel_flags, score)| {
+ (
+ score,
+ SmallEdge {
+ from: self.node,
+ to: node,
+ rel_flags,
+ },
+ )
+ })
.collect())
}
@@ -235,7 +244,7 @@ impl FullForwardlinksQuery {
impl Query for FullForwardlinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, Edge)>;
+ type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
@@ -276,15 +285,15 @@ impl Query for FullForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let edges = fetch_edges(searcher, docs)?;
- Ok(scores.into_iter().zip_eq(edges).collect())
+ Ok(edges)
}
fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
let mut edges: Vec<_> = results.into_iter().flatten().collect();
- edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
- edges.into_iter().map(|(_, e)| e).collect()
+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));
+ edges
}
}
@@ -313,7 +322,7 @@ impl FullHostForwardlinksQuery {
impl Query for FullHostForwardlinksQuery {
type Collector = TopDocsCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = Vec<(f32, Edge)>;
+ type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
@@ -364,23 +373,21 @@ impl Query for FullHostForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
let edges = fetch_edges(searcher, docs)?;
- Ok(scores
+ Ok(edges
.into_iter()
- .zip_eq(edges.into_iter().map(|e| Edge {
+ .map(|e| Edge {
from: e.from.into_host(),
to: e.to.into_host(),
- rel_flags: e.rel_flags,
- label: e.label,
- sort_score: e.sort_score,
- }))
+ ..e
+ })
.collect())
}
fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
let mut edges: Vec<_> = results.into_iter().flatten().collect();
- edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
- edges.into_iter().map(|(_, e)| e).collect()
+ edges.sort_by(|a, b| b.sort_score.total_cmp(&a.sort_score));
+ edges
}
}
diff --git a/crates/core/src/webgraph/warmed_column_fields.rs b/crates/core/src/webgraph/warmed_column_fields.rs
index a2afc9fe..fd9980a0 100644
--- a/crates/core/src/webgraph/warmed_column_fields.rs
+++ b/crates/core/src/webgraph/warmed_column_fields.rs
@@ -24,6 +24,7 @@ use crate::{enum_map::EnumMap, Result};
#[derive(Debug, Clone)]
pub struct SegmentColumnFields {
u64_fields: EnumMap<FieldEnumDiscriminants, Column<u64>>,
+ f64_fields: EnumMap<FieldEnumDiscriminants, Column<f64>>,
}
impl SegmentColumnFields {
@@ -33,6 +34,13 @@ impl SegmentColumnFields {
.get(FieldEnumDiscriminants::from(field_enum))
.cloned()
}
+
+ pub fn f64<F: Field>(&self, field: F) -> Option<Column<f64>> {
+ let field_enum: FieldEnum = field.into();
+ self.f64_fields
+ .get(FieldEnumDiscriminants::from(field_enum))
+ .cloned()
+ }
}
#[derive(Debug, Clone)]
@@ -45,6 +53,7 @@ impl WarmedColumnFields {
let mut segments = HashMap::new();
for segment in tantivy_searcher.segment_readers() {
let mut u64_fields = EnumMap::new();
+ let mut f64_fields = EnumMap::new();
for field in FieldEnum::iter() {
match field.field_type() {
tantivy::schema::FieldType::U64(numeric_options)
@@ -53,10 +62,22 @@ impl WarmedColumnFields {
let column = segment.column_fields().u64(field.name())?;
u64_fields.insert(field.into(), column);
}
+ tantivy::schema::FieldType::F64(numeric_options)
+ if numeric_options.is_columnar() =>
+ {
+ let column = segment.column_fields().f64(field.name())?;
+ f64_fields.insert(field.into(), column);
+ }
_ => {}
}
}
- segments.insert(segment.segment_id(), SegmentColumnFields { u64_fields });
+ segments.insert(
+ segment.segment_id(),
+ SegmentColumnFields {
+ u64_fields,
+ f64_fields,
+ },
+ );
}
Ok(Self { segments })