Skip to content

web search done right

[ranking] add a host_centrality_rank and page_centrality_rank signal it might be easier to score pages based on their rank of the sorted their centralities. for instance the centralities for page A and page B might be very similar numerically, but if a lot of pages are between A and B when looking at the sorted list, the highest ranking page might in reality be a better result than the lower ranking one.

Mikkel Denker7ea3dbcca41f
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 85806988..f004beaf 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,7 +31,6 @@ The process is simple and fast. Upon your first pull request, you will be prompt
* Update ulimit. RocksDB tends to exceed the max number of allowed open files, so you will have to run `ulimit -n 10240` to increase the allowed max number of open files.
* Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile. Since libtorch requires some specific environment variables to be set, you need to preface all `cargo run` and `cargo test` commands with `just` (so they becom `just cargo run` and `just cargo test`) .
* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.
-* (Optional) run `just configure --alice` to download the weights for Alice (the AI assistant that can search on the web).
* (Optional) Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`. This makes frontend development easier.
* (Optional) Install [abeye](https://github.com/oeb25/abeye) by running `cargo install --git https://github.com/oeb25/abeye --locked`. This is used for generating the API client used by the frontend by running `just openapi`.
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 2bc3ae9c..bd040fea 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -15,9 +15,11 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use anyhow::Result;
-use std::{cmp::Reverse, collections::BinaryHeap, fs::File, path::Path};
+use serde::{Deserialize, Serialize};
+use std::{cmp::Reverse, fs::File, path::Path};
use crate::{
+ external_sort::ExternalSorter,
kv::{rocksdb_store::RocksDbStore, Kv},
ranking::inbound_similarity::InboundSimilarity,
webgraph::{
@@ -44,6 +46,7 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {
wtr.flush().unwrap();
}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
struct SortableFloat(f64);
impl PartialEq for SortableFloat {
@@ -70,7 +73,10 @@ pub struct Centrality;
impl Centrality {
pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) {
- tracing::info!("Building harmonic centrality");
+ tracing::info!(
+ "Building harmonic centrality for {}",
+ webgraph_path.as_ref().to_str().unwrap()
+ );
let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
let harmonic_centrality = HarmonicCentrality::calculate(&graph);
let store = RocksDbStore::open(base_output.as_ref().join("harmonic"));
@@ -80,29 +86,36 @@ impl Centrality {
}
store.flush();
- let mut top_harmonics = BinaryHeap::new();
+ let rank_store = RocksDbStore::open(base_output.as_ref().join("harmonic_rank"));
+ let mut top_harmonics = Vec::new();
+ for (rank, node, centrality) in ExternalSorter::new()
+ .with_chunk_size(100_000_000)
+ .sort(
+ harmonic_centrality
+ .iter()
+ .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), *node_id)),
+ )
+ .unwrap()
+ .enumerate()
+ .map(|(rank, (Reverse(SortableFloat(centrality)), node_id))| {
+ (rank, node_id, centrality)
+ })
+ {
+ rank_store.insert(node, rank as f64);
- for (node_id, centrality) in harmonic_centrality.iter() {
if top_harmonics.len() < 1_000_000 {
- top_harmonics.push((Reverse(SortableFloat(centrality)), *node_id));
- } else {
- let mut min = top_harmonics.peek_mut().unwrap();
- if min.0 .0 < SortableFloat(centrality) {
- *min = (Reverse(SortableFloat(centrality)), *node_id);
- }
+ top_harmonics.push((graph.id2node(&node).unwrap(), centrality));
}
}
- let harmonics: Vec<_> = top_harmonics
- .into_iter()
- .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))
- .collect();
-
- store_csv(harmonics, base_output.as_ref().join("harmonic.csv"));
+ store_csv(top_harmonics, base_output.as_ref().join("harmonic.csv"));
}
pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, base_output: P) {
- tracing::info!("Building inbound similarity");
+ tracing::info!(
+ "Building inbound similarity for {}",
+ webgraph_path.as_ref().to_str().unwrap()
+ );
let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
let sim = InboundSimilarity::build(&graph);
@@ -112,30 +125,37 @@ impl Centrality {
}
pub fn build_approx_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) -> Result<()> {
- tracing::info!("Building approximated harmonic centrality for page graph");
+ tracing::info!(
+ "Building approximated harmonic centrality for {}",
+ webgraph_path.as_ref().to_str().unwrap()
+ );
+
let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("approx_harmonic"));
-
- let mut top_nodes = BinaryHeap::new();
-
- for (node_id, centrality) in approx.iter() {
+ let approx_rank = RocksDbStore::open(base_output.as_ref().join("approx_harmonic_rank"));
+
+ let mut top_nodes = Vec::new();
+
+ for (rank, node, centrality) in ExternalSorter::new()
+ .with_chunk_size(100_000_000)
+ .sort(
+ approx
+ .iter()
+ .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), node_id)),
+ )?
+ .enumerate()
+ .map(|(rank, (Reverse(SortableFloat(centrality)), node_id))| {
+ (rank, node_id, centrality)
+ })
+ {
+ approx_rank.insert(node, rank as f64);
if top_nodes.len() < 1_000_000 {
- top_nodes.push((Reverse(SortableFloat(centrality)), node_id));
- } else {
- let mut min = top_nodes.peek_mut().unwrap();
- if min.0 .0 < SortableFloat(centrality) {
- *min = (Reverse(SortableFloat(centrality)), node_id);
- }
+ top_nodes.push((graph.id2node(&node).unwrap(), centrality));
}
}
- let derived: Vec<_> = top_nodes
- .into_iter()
- .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))
- .collect();
-
- store_csv(derived, base_output.as_ref().join("approx_harmonic.csv"));
+ store_csv(top_nodes, base_output.as_ref().join("approx_harmonic.csv"));
Ok(())
}
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 6b6caac6..68c3a840 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -52,7 +52,9 @@ pub struct JobSettings {
pub struct IndexingWorker {
host_centrality_store: RocksDbStore<NodeID, f64>,
+ host_centrality_rank_store: RocksDbStore<NodeID, f64>,
page_centrality_store: Option<RocksDbStore<NodeID, f64>>,
+ page_centrality_rank_store: Option<RocksDbStore<NodeID, f64>>,
page_webgraph: Option<Webgraph>,
topics: Option<human_website_annotations::Mapper>,
safety_classifier: Option<safety_classifier::Model>,
@@ -71,8 +73,15 @@ impl IndexingWorker {
host_centrality_store: RocksDbStore::open(
Path::new(&host_centrality_store_path).join("harmonic"),
),
+ host_centrality_rank_store: RocksDbStore::open(
+ Path::new(&host_centrality_store_path).join("harmonic_rank"),
+ ),
page_centrality_store: page_centrality_store_path
+ .as_ref()
.map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),
+ page_centrality_rank_store: page_centrality_store_path
+ .as_ref()
+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic_rank"))),
page_webgraph: page_webgraph_path
.map(|path| WebgraphBuilder::new(path).single_threaded().open()),
topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),
@@ -112,6 +121,11 @@ impl IndexingWorker {
.get(&host_node_id)
.unwrap_or_default();
+ let mut host_centrality_rank = self
+ .host_centrality_rank_store
+ .get(&host_node_id)
+ .unwrap_or(u64::MAX as f64);
+
if let Some(host_centrality_threshold) =
self.job_settings.and_then(|s| s.host_centrality_threshold)
{
@@ -182,6 +196,14 @@ impl IndexingWorker {
page_centrality = store.get(&node_id).unwrap_or_default();
}
+ let mut page_centrality_rank = u64::MAX as f64;
+
+ if let Some(store) = self.page_centrality_rank_store.as_ref() {
+ let node_id = node.id();
+
+ page_centrality_rank = store.get(&node_id).unwrap_or(u64::MAX as f64);
+ }
+
if !page_centrality.is_finite() {
page_centrality = 0.0;
}
@@ -190,6 +212,14 @@ impl IndexingWorker {
host_centrality = 0.0;
}
+ if !page_centrality_rank.is_finite() {
+ page_centrality_rank = u64::MAX as f64;
+ }
+
+ if !host_centrality_rank.is_finite() {
+ host_centrality_rank = u64::MAX as f64;
+ }
+
let mut dmoz_description = None;
if let Some(mapper) = self.topics.as_ref() {
@@ -202,7 +232,9 @@ impl IndexingWorker {
html,
backlink_labels,
page_centrality,
+ page_centrality_rank,
host_centrality,
+ host_centrality_rank,
fetch_time_ms,
pre_computed_score: 0.0,
node_id: Some(host_node_id),
diff --git a/core/src/external_sort.rs b/core/src/external_sort.rs
new file mode 100644
index 00000000..cba11672
--- /dev/null
+++ b/core/src/external_sort.rs
@@ -0,0 +1,357 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::Result;
+use std::{
+ cmp::Reverse,
+ collections::BinaryHeap,
+ fs::{File, OpenOptions},
+ io::{BufReader, BufWriter, Read, Seek, Write},
+};
+
+struct Chunk<T> {
+ data: Vec<T>,
+}
+
+impl<T> Chunk<T> {
+ fn new() -> Self {
+ Self { data: Vec::new() }
+ }
+
+ fn push(&mut self, item: T) {
+ self.data.push(item);
+ }
+
+ fn len(&self) -> usize {
+ self.data.len()
+ }
+
+ fn is_empty(&self) -> bool {
+ self.data.is_empty()
+ }
+
+ fn sort(&mut self)
+ where
+ T: Ord,
+ {
+ self.data.sort();
+ }
+
+ fn write_into(self, file: &mut File) -> Result<()>
+ where
+ T: serde::Serialize,
+ {
+ let mut wrt = BufWriter::new(file);
+
+ for item in self.data {
+ let bytes = bincode::serialize(&item)?;
+ let size = bytes.len() as u64;
+
+ wrt.write_all(&size.to_le_bytes())?;
+ wrt.write_all(&bytes)?;
+ }
+
+ wrt.flush()?;
+
+ Ok(())
+ }
+
+ fn store(mut self, mut file: TempFile) -> Result<StoredChunk<T>>
+ where
+ T: serde::Serialize + Ord,
+ {
+ self.sort();
+ self.write_into(&mut file.inner)?;
+
+ StoredChunk::new(file)
+ }
+}
+
+struct TempDir {
+ path: std::path::PathBuf,
+}
+
+impl TempDir {
+ fn new() -> Result<Self> {
+ let path = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());
+
+ std::fs::create_dir(&path)?;
+
+ Ok(Self { path })
+ }
+
+ fn as_ref(&self) -> &std::path::Path {
+ self.path.as_ref()
+ }
+}
+
+impl Drop for TempDir {
+ fn drop(&mut self) {
+ std::fs::remove_dir_all(&self.path).unwrap();
+ }
+}
+
+struct TempFile {
+ inner: File,
+ path: std::path::PathBuf,
+}
+
+impl TempFile {
+ fn new(dir: &TempDir) -> Result<Self> {
+ let path = dir.as_ref().join(uuid::Uuid::new_v4().to_string());
+
+ Ok(Self {
+ inner: OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(&path)?,
+ path,
+ })
+ }
+}
+
+impl Drop for TempFile {
+ fn drop(&mut self) {
+ std::fs::remove_file(&self.path).unwrap();
+ }
+}
+
+impl Read for TempFile {
+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+ self.inner.read(buf)
+ }
+}
+
+impl Seek for TempFile {
+ fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
+ self.inner.seek(pos)
+ }
+}
+
+struct StoredChunk<T> {
+ buf: Vec<u8>,
+ data: BufReader<TempFile>,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl<T> StoredChunk<T> {
+ fn new(mut file: TempFile) -> Result<Self> {
+ file.seek(std::io::SeekFrom::Start(0))?;
+ Ok(Self {
+ buf: Vec::new(),
+ data: BufReader::new(file),
+ _marker: std::marker::PhantomData,
+ })
+ }
+
+ fn next(&mut self) -> Option<T>
+ where
+ T: serde::de::DeserializeOwned,
+ {
+ self.buf.clear();
+ self.buf.resize(u64::BITS as usize / 8, 0);
+
+ self.data.read_exact(&mut self.buf).ok()?;
+
+ let next_size: u64 = u64::from_le_bytes(self.buf.as_slice().try_into().ok()?);
+
+ self.buf.clear();
+ self.buf.resize(next_size as usize, 0);
+
+ self.data.read_exact(&mut self.buf).ok()?;
+
+ let next = bincode::deserialize(&self.buf).ok()?;
+
+ Some(next)
+ }
+}
+
+pub struct ExternalSorter<T> {
+ chunk_size: usize,
+ _marker: std::marker::PhantomData<T>,
+}
+
+impl Default for ExternalSorter<()> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<T> ExternalSorter<T> {
+ pub fn new() -> Self {
+ Self {
+ chunk_size: 1_000_000,
+ _marker: std::marker::PhantomData,
+ }
+ }
+
+ pub fn with_chunk_size(mut self, chunk_size: usize) -> Self {
+ self.chunk_size = chunk_size;
+ self
+ }
+
+ pub fn sort<I>(self, iter: I) -> Result<SortedIterator<T>>
+ where
+ I: Iterator<Item = T>,
+ T: serde::Serialize + serde::de::DeserializeOwned + Ord,
+ {
+ let dir = TempDir::new()?;
+ let mut stored_chunks = Vec::new();
+ let mut chunk = Chunk::new();
+
+ for item in iter {
+ chunk.push(item);
+
+ if chunk.len() >= self.chunk_size {
+ let file = TempFile::new(&dir)?;
+ let stored_chunk = chunk.store(file)?;
+ stored_chunks.push(stored_chunk);
+ chunk = Chunk::new();
+ }
+ }
+
+ if !chunk.is_empty() {
+ let file = TempFile::new(&dir)?;
+ let stored_chunk = chunk.store(file)?;
+ stored_chunks.push(stored_chunk);
+ }
+
+ let heads = stored_chunks
+ .into_iter()
+ .filter_map(|chunk| Head::new(chunk))
+ .map(Reverse)
+ .collect::<BinaryHeap<_>>();
+
+ Ok(SortedIterator {
+ _dir: dir,
+ chunks: heads,
+ })
+ }
+}
+
+struct Head<T> {
+ item: T,
+ rest: StoredChunk<T>,
+}
+
+impl<T> Head<T>
+where
+ T: serde::de::DeserializeOwned,
+{
+ fn new(mut chunk: StoredChunk<T>) -> Option<Self> {
+ let item = chunk.next()?;
+
+ Some(Self { item, rest: chunk })
+ }
+}
+
+impl<T> Ord for Head<T>
+where
+ T: Ord,
+{
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.item.cmp(&other.item)
+ }
+}
+
+impl<T> PartialOrd for Head<T>
+where
+ T: Ord,
+{
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl<T> Eq for Head<T> where T: Eq {}
+
+impl<T> PartialEq for Head<T>
+where
+ T: PartialEq,
+{
+ fn eq(&self, other: &Self) -> bool {
+ self.item.eq(&other.item)
+ }
+}
+
+impl<T> std::fmt::Debug for Head<T>
+where
+ T: std::fmt::Debug,
+{
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("Head").field("item", &self.item).finish()
+ }
+}
+
+pub struct SortedIterator<T> {
+ // We need to keep the temp dir alive for the lifetime of the iterator
+ _dir: TempDir,
+ chunks: BinaryHeap<Reverse<Head<T>>>,
+}
+
+impl<T> Iterator for SortedIterator<T>
+where
+ T: serde::de::DeserializeOwned + Ord,
+{
+ type Item = T;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let mut head = self.chunks.pop()?.0;
+
+ if let Some(next) = head.rest.next() {
+ self.chunks.push(Reverse(Head {
+ item: next,
+ rest: head.rest,
+ }));
+ }
+
+ Some(head.item)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use rand::Rng;
+
+ use super::*;
+
+ #[test]
+ fn test_external_sort() {
+ let mut rng = rand::thread_rng();
+
+ let mut expected = Vec::new();
+ let mut data = Vec::new();
+
+ for _ in 0..1_000_000 {
+ let item = rng.gen_range(0..1_000_000);
+
+ expected.push(item);
+ data.push(item);
+ }
+
+ expected.sort();
+
+ let sorted = ExternalSorter::new()
+ .with_chunk_size(100_000)
+ .sort(data.into_iter())
+ .unwrap()
+ .collect::<Vec<_>>();
+
+ assert_eq!(sorted.len(), 1_000_000);
+ assert_eq!(sorted, expected);
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index f825a70b..c6cba649 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -44,6 +44,7 @@ mod distributed;
mod entity_index;
mod enum_map;
mod executor;
+mod external_sort;
mod fastfield_reader;
pub mod feed;
mod human_website_annotations;
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 8985291a..1e4a5ee1 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -104,8 +104,12 @@ pub enum Signal {
CrossEncoderTitle,
#[serde(rename = "host_centrality")]
HostCentrality,
+ #[serde(rename = "host_centrality_rank")]
+ HostCentralityRank,
#[serde(rename = "page_centrality")]
PageCentrality,
+ #[serde(rename = "page_centrality_rank")]
+ PageCentralityRank,
#[serde(rename = "is_homepage")]
IsHomepage,
#[serde(rename = "fetch_time_ms")]
@@ -136,7 +140,7 @@ impl From<Signal> for usize {
}
}
-pub const ALL_SIGNALS: [Signal; 35] = [
+pub const ALL_SIGNALS: [Signal; 37] = [
Signal::Bm25Title,
Signal::Bm25TitleBigrams,
Signal::Bm25TitleTrigrams,
@@ -160,7 +164,9 @@ pub const ALL_SIGNALS: [Signal; 35] = [
Signal::CrossEncoderSnippet,
Signal::CrossEncoderTitle,
Signal::HostCentrality,
+ Signal::HostCentralityRank,
Signal::PageCentrality,
+ Signal::PageCentralityRank,
Signal::IsHomepage,
Signal::FetchTimeMs,
Signal::UpdateTimestamp,
@@ -189,6 +195,11 @@ fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f6
}
}
+#[inline]
+fn score_rank(rank: f64) -> f64 {
+ 1.0 / (rank + 1.0)
+}
+
#[inline]
fn score_trackers(num_trackers: f64) -> f64 {
1.0 / (num_trackers + 1.0)
@@ -284,7 +295,9 @@ impl Signal {
Signal::CrossEncoderSnippet => 0.17,
Signal::CrossEncoderTitle => 0.17,
Signal::HostCentrality => 0.5,
+ Signal::HostCentralityRank => 0.0,
Signal::PageCentrality => 0.25,
+ Signal::PageCentralityRank => 0.0,
Signal::QueryCentrality => 0.0,
Signal::IsHomepage => 0.0005,
Signal::FetchTimeMs => 0.001,
@@ -345,6 +358,13 @@ impl Signal {
field_value.map(|val| val as f64 / FLOAT_SCALING as f64)
}
+ Signal::HostCentralityRank | Signal::PageCentralityRank => {
+ let field_value: Option<u64> = self
+ .fastfield_value(signal_aggregator, doc)
+ .and_then(|val| val.into());
+
+ field_value.map(|val| val as f64).map(score_rank)
+ }
Signal::IsHomepage => {
let field_value: Option<u64> = self
.fastfield_value(signal_aggregator, doc)
@@ -470,7 +490,9 @@ impl Signal {
let value = match self {
Signal::HostCentrality => Some(webpage.host_centrality),
+ Signal::HostCentralityRank => Some(webpage.host_centrality_rank),
Signal::PageCentrality => Some(webpage.page_centrality),
+ Signal::PageCentralityRank => Some(webpage.page_centrality_rank),
Signal::IsHomepage => Some(webpage.html.is_homepage().into()),
Signal::FetchTimeMs => {
let fetch_time_ms = webpage.fetch_time_ms as usize;
@@ -571,7 +593,9 @@ impl Signal {
fn as_fastfield(&self) -> Option<FastField> {
match self {
Signal::HostCentrality => Some(FastField::HostCentrality),
+ Signal::HostCentralityRank => Some(FastField::HostCentralityRank),
Signal::PageCentrality => Some(FastField::PageCentrality),
+ Signal::PageCentralityRank => Some(FastField::PageCentralityRank),
Signal::IsHomepage => Some(FastField::IsHomepage),
Signal::FetchTimeMs => Some(FastField::FetchTimeMs),
Signal::UpdateTimestamp => Some(FastField::LastUpdated),
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 99cf3ffd..8ca974f7 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -201,7 +201,9 @@ impl TextField {
pub enum FastField {
IsHomepage,
HostCentrality,
+ HostCentralityRank,
PageCentrality,
+ PageCentralityRank,
FetchTimeMs,
LastUpdated,
TrackerScore,
@@ -240,7 +242,9 @@ impl FastField {
pub fn name(&self) -> &str {
match self {
FastField::HostCentrality => "host_centrality",
+ FastField::HostCentralityRank => "host_centrality_rank",
FastField::PageCentrality => "page_centrality",
+ FastField::PageCentralityRank => "page_centrality_rank",
FastField::IsHomepage => "is_homepage",
FastField::FetchTimeMs => "fetch_time_ms",
FastField::LastUpdated => "last_updated",
@@ -290,7 +294,7 @@ pub enum Field {
Text(TextField),
}
-static ALL_FIELDS: [Field; 64] = [
+static ALL_FIELDS: [Field; 66] = [
Field::Text(TextField::Title),
Field::Text(TextField::CleanBody),
Field::Text(TextField::StemmedTitle),
@@ -324,7 +328,9 @@ static ALL_FIELDS: [Field; 64] = [
// FAST FIELDS
Field::Fast(FastField::IsHomepage),
Field::Fast(FastField::HostCentrality),
+ Field::Fast(FastField::HostCentralityRank),
Field::Fast(FastField::PageCentrality),
+ Field::Fast(FastField::PageCentralityRank),
Field::Fast(FastField::FetchTimeMs),
Field::Fast(FastField::LastUpdated),
Field::Fast(FastField::TrackerScore),
@@ -481,9 +487,15 @@ impl Field {
Field::Fast(FastField::HostCentrality) => {
IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
}
+ Field::Fast(FastField::HostCentralityRank) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
Field::Fast(FastField::PageCentrality) => {
IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
}
+ Field::Fast(FastField::PageCentralityRank) => {
+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
+ }
Field::Fast(FastField::FetchTimeMs) => {
IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())
}
@@ -693,7 +705,9 @@ impl FastField {
match self {
FastField::IsHomepage => DataType::U64,
FastField::HostCentrality => DataType::U64,
+ FastField::HostCentralityRank => DataType::U64,
FastField::PageCentrality => DataType::U64,
+ FastField::PageCentralityRank => DataType::U64,
FastField::FetchTimeMs => DataType::U64,
FastField::LastUpdated => DataType::U64,
FastField::TrackerScore => DataType::U64,
diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rs
index 8252ae36..4ac86024 100644
--- a/core/src/webgraph/centrality/approx_harmonic.rs
+++ b/core/src/webgraph/centrality/approx_harmonic.rs
@@ -48,7 +48,7 @@ impl ApproxHarmonic {
let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));
sampled.into_par_iter().progress().for_each(|source| {
- let dists = graph.raw_distances(source);
+ let dists = graph.raw_distances_with_max(source, 5);
let res = res.lock().unwrap();
for (target, dist) in dists {
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 06bb286f..f8b02237 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -335,6 +335,7 @@ impl WebgraphBuilder {
pub trait ShortestPaths {
fn distances(&self, source: Node) -> BTreeMap<Node, u8>;
fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
+ fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8>;
fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;
}
@@ -343,6 +344,7 @@ fn dijkstra_multi<F1, F2, L>(
sources: &[NodeID],
node_edges: F1,
edge_node: F2,
+ max_dist: Option<u8>,
) -> BTreeMap<NodeID, u8>
where
L: EdgeLabel,
@@ -367,6 +369,12 @@ where
continue;
}
+ if let Some(max_dist) = max_dist {
+ if cost > max_dist {
+ return distances;
+ }
+ }
+
for edge in node_edges(v) {
if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {
let d = cost + 1;
@@ -389,11 +397,21 @@ impl ShortestPaths for Webgraph {
.collect()
}
+ fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8> {
+ dijkstra_multi(
+ &[source],
+ |node| self.raw_outgoing_edges(&node),
+ |edge| edge.to,
+ Some(max_dist),
+ )
+ }
+
fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
|node| self.raw_outgoing_edges(&node),
|edge| edge.to,
+ None,
)
}
@@ -402,6 +420,7 @@ impl ShortestPaths for Webgraph {
&[source],
|node| self.raw_ingoing_edges(&node),
|edge| edge.from,
+ None,
)
}
diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rs
index 1c32cf4c..4ca386cb 100644
--- a/core/src/webpage/html/into_tantivy.rs
+++ b/core/src/webpage/html/into_tantivy.rs
@@ -522,7 +522,9 @@ impl Html {
| Field::Text(TextField::SafetyClassification)
| Field::Text(TextField::InsertionTimestamp)
| Field::Fast(FastField::HostCentrality)
+ | Field::Fast(FastField::HostCentralityRank)
| Field::Fast(FastField::PageCentrality)
+ | Field::Fast(FastField::PageCentralityRank)
| Field::Fast(FastField::FetchTimeMs)
| Field::Fast(FastField::PreComputedScore)
| Field::Fast(FastField::Region)
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 1f4926a2..588c1e99 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -45,7 +45,9 @@ pub struct Webpage {
pub html: Html,
pub backlink_labels: Vec<String>,
pub host_centrality: f64,
+ pub host_centrality_rank: f64,
pub page_centrality: f64,
+ pub page_centrality_rank: f64,
pub fetch_time_ms: u64,
pub pre_computed_score: f64,
pub node_id: Option<NodeID>,
@@ -68,7 +70,9 @@ impl Default for Webpage {
},
backlink_labels: Default::default(),
host_centrality: Default::default(),
+ host_centrality_rank: u64::MAX as f64,
page_centrality: Default::default(),
+ page_centrality_rank: u64::MAX as f64,
fetch_time_ms: Default::default(),
pre_computed_score: Default::default(),
node_id: Default::default(),
@@ -86,15 +90,7 @@ impl Webpage {
Ok(Self {
html,
- backlink_labels: Vec::new(),
- host_centrality: 0.0,
- page_centrality: 0.0,
- fetch_time_ms: 0,
- pre_computed_score: 0.0,
- node_id: None,
- dmoz_description: None,
- safety_classification: None,
- inserted_at: Utc::now(),
+ ..Default::default()
})
}
@@ -175,6 +171,13 @@ impl Webpage {
(self.host_centrality * FLOAT_SCALING as f64) as u64,
);
+ doc.add_u64(
+ schema
+ .get_field(Field::Fast(FastField::HostCentralityRank).name())
+ .expect("Failed to get host_centrality_rank field"),
+ self.host_centrality_rank as u64,
+ );
+
doc.add_u64(
schema
.get_field(Field::Fast(FastField::PageCentrality).name())
@@ -182,6 +185,13 @@ impl Webpage {
(self.page_centrality * FLOAT_SCALING as f64) as u64,
);
+ doc.add_u64(
+ schema
+ .get_field(Field::Fast(FastField::PageCentralityRank).name())
+ .expect("Failed to get page_centrality_rank field"),
+ self.page_centrality_rank as u64,
+ );
+
doc.add_u64(
schema
.get_field(Field::Fast(FastField::FetchTimeMs).name())
diff --git a/justfile b/justfile
index 77b67785..d29ea7be 100644
--- a/justfile
+++ b/justfile
@@ -12,8 +12,6 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
@dev-webgraph:
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
-@dev-alice:
- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- alice serve configs/alice.toml"
@dev-frontend:
cd frontend && npm run dev
diff --git a/scripts/run_dev.py b/scripts/run_dev.py
index 080674ef..42813287 100755
--- a/scripts/run_dev.py
+++ b/scripts/run_dev.py
@@ -10,7 +10,6 @@ os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"
parser = argparse.ArgumentParser()
parser.add_argument("--release", action="store_true")
-parser.add_argument("--alice", action="store_true")
args = parser.parse_args()
@@ -24,9 +23,6 @@ processes.append(subprocess.Popen(["just", "dev-search-server"]))
processes.append(subprocess.Popen(["just", "dev-webgraph"]))
processes.append(subprocess.Popen(["just", "dev-frontend"]))
-if args.alice:
- processes.append(subprocess.Popen(["just", "dev-alice"]))
-
# kill processes on ctrl-c
import time