web search done right
[ranking] add a host_centrality_rank and page_centrality_rank signal it might be easier to score pages based on their rank of the sorted their centralities. for instance the centralities for page A and page B might be very similar numerically, but if a lot of pages are between A and B when looking at the sorted list, the highest ranking page might in reality be a better result than the lower ranking one.
Mikkel Denker7ea3dbcca41f
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 85806988..f004beaf 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -31,7 +31,6 @@ The process is simple and fast. Upon your first pull request, you will be prompt* Update ulimit. RocksDB tends to exceed the max number of allowed open files, so you will have to run `ulimit -n 10240` to increase the allowed max number of open files.* Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile. Since libtorch requires some specific environment variables to be set, you need to preface all `cargo run` and `cargo test` commands with `just` (so they becom `just cargo run` and `just cargo test`) .* Run the command `just configure` which should automatically configures the rest of your dev environment. The script creates a python virtual environment, installs relevant dependencies, traces and exports the ML models and creates a small local index which you can use for development.-* (Optional) run `just configure --alice` to download the weights for Alice (the AI assistant that can search on the web).* (Optional) Install [cargo-watch](https://github.com/watchexec/cargo-watch) by running `cargo install cargo-watch`. This makes frontend development easier.* (Optional) Install [abeye](https://github.com/oeb25/abeye) by running `cargo install --git https://github.com/oeb25/abeye --locked`. This is used for generating the API client used by the frontend by running `just openapi`.diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rsindex 2bc3ae9c..bd040fea 100644--- a/core/src/entrypoint/centrality.rs+++ b/core/src/entrypoint/centrality.rs@@ -15,9 +15,11 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;-use std::{cmp::Reverse, collections::BinaryHeap, fs::File, path::Path};+use serde::{Deserialize, Serialize};+use std::{cmp::Reverse, fs::File, path::Path};use crate::{+ external_sort::ExternalSorter,kv::{rocksdb_store::RocksDbStore, Kv},ranking::inbound_similarity::InboundSimilarity,webgraph::{@@ -44,6 +46,7 @@ fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {wtr.flush().unwrap();}+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]struct SortableFloat(f64);impl PartialEq for SortableFloat {@@ -70,7 +73,10 @@ pub struct Centrality;impl Centrality {pub fn build_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) {- tracing::info!("Building harmonic centrality");+ tracing::info!(+ "Building harmonic centrality for {}",+ webgraph_path.as_ref().to_str().unwrap()+ );let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let harmonic_centrality = HarmonicCentrality::calculate(&graph);let store = RocksDbStore::open(base_output.as_ref().join("harmonic"));@@ -80,29 +86,36 @@ impl Centrality {}store.flush();- let mut top_harmonics = BinaryHeap::new();+ let rank_store = RocksDbStore::open(base_output.as_ref().join("harmonic_rank"));+ let mut top_harmonics = Vec::new();+ for (rank, node, centrality) in ExternalSorter::new()+ .with_chunk_size(100_000_000)+ .sort(+ harmonic_centrality+ .iter()+ .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), *node_id)),+ )+ .unwrap()+ .enumerate()+ .map(|(rank, (Reverse(SortableFloat(centrality)), node_id))| {+ (rank, node_id, centrality)+ })+ {+ rank_store.insert(node, rank as f64);- for (node_id, centrality) in harmonic_centrality.iter() {if top_harmonics.len() < 1_000_000 {- top_harmonics.push((Reverse(SortableFloat(centrality)), *node_id));- } else {- let mut min = top_harmonics.peek_mut().unwrap();- if min.0 .0 < SortableFloat(centrality) {- *min = (Reverse(SortableFloat(centrality)), *node_id);- }+ top_harmonics.push((graph.id2node(&node).unwrap(), centrality));}}- let harmonics: Vec<_> = top_harmonics- .into_iter()- .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))- .collect();-- store_csv(harmonics, base_output.as_ref().join("harmonic.csv"));+ store_csv(top_harmonics, base_output.as_ref().join("harmonic.csv"));}pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, base_output: P) {- tracing::info!("Building inbound similarity");+ tracing::info!(+ "Building inbound similarity for {}",+ webgraph_path.as_ref().to_str().unwrap()+ );let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let sim = InboundSimilarity::build(&graph);@@ -112,30 +125,37 @@ impl Centrality {}pub fn build_approx_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) -> Result<()> {- tracing::info!("Building approximated harmonic centrality for page graph");+ tracing::info!(+ "Building approximated harmonic centrality for {}",+ webgraph_path.as_ref().to_str().unwrap()+ );+let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("approx_harmonic"));-- let mut top_nodes = BinaryHeap::new();-- for (node_id, centrality) in approx.iter() {+ let approx_rank = RocksDbStore::open(base_output.as_ref().join("approx_harmonic_rank"));++ let mut top_nodes = Vec::new();++ for (rank, node, centrality) in ExternalSorter::new()+ .with_chunk_size(100_000_000)+ .sort(+ approx+ .iter()+ .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), node_id)),+ )?+ .enumerate()+ .map(|(rank, (Reverse(SortableFloat(centrality)), node_id))| {+ (rank, node_id, centrality)+ })+ {+ approx_rank.insert(node, rank as f64);if top_nodes.len() < 1_000_000 {- top_nodes.push((Reverse(SortableFloat(centrality)), node_id));- } else {- let mut min = top_nodes.peek_mut().unwrap();- if min.0 .0 < SortableFloat(centrality) {- *min = (Reverse(SortableFloat(centrality)), node_id);- }+ top_nodes.push((graph.id2node(&node).unwrap(), centrality));}}- let derived: Vec<_> = top_nodes- .into_iter()- .map(|(score, id)| (graph.id2node(&id).unwrap(), score.0 .0))- .collect();-- store_csv(derived, base_output.as_ref().join("approx_harmonic.csv"));+ store_csv(top_nodes, base_output.as_ref().join("approx_harmonic.csv"));Ok(())}diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 6b6caac6..68c3a840 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -52,7 +52,9 @@ pub struct JobSettings {pub struct IndexingWorker {host_centrality_store: RocksDbStore<NodeID, f64>,+ host_centrality_rank_store: RocksDbStore<NodeID, f64>,page_centrality_store: Option<RocksDbStore<NodeID, f64>>,+ page_centrality_rank_store: Option<RocksDbStore<NodeID, f64>>,page_webgraph: Option<Webgraph>,topics: Option<human_website_annotations::Mapper>,safety_classifier: Option<safety_classifier::Model>,@@ -71,8 +73,15 @@ impl IndexingWorker {host_centrality_store: RocksDbStore::open(Path::new(&host_centrality_store_path).join("harmonic"),),+ host_centrality_rank_store: RocksDbStore::open(+ Path::new(&host_centrality_store_path).join("harmonic_rank"),+ ),page_centrality_store: page_centrality_store_path+ .as_ref().map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),+ page_centrality_rank_store: page_centrality_store_path+ .as_ref()+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic_rank"))),page_webgraph: page_webgraph_path.map(|path| WebgraphBuilder::new(path).single_threaded().open()),topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),@@ -112,6 +121,11 @@ impl IndexingWorker {.get(&host_node_id).unwrap_or_default();+ let mut host_centrality_rank = self+ .host_centrality_rank_store+ .get(&host_node_id)+ .unwrap_or(u64::MAX as f64);+if let Some(host_centrality_threshold) =self.job_settings.and_then(|s| s.host_centrality_threshold){@@ -182,6 +196,14 @@ impl IndexingWorker {page_centrality = store.get(&node_id).unwrap_or_default();}+ let mut page_centrality_rank = u64::MAX as f64;++ if let Some(store) = self.page_centrality_rank_store.as_ref() {+ let node_id = node.id();++ page_centrality_rank = store.get(&node_id).unwrap_or(u64::MAX as f64);+ }+if !page_centrality.is_finite() {page_centrality = 0.0;}@@ -190,6 +212,14 @@ impl IndexingWorker {host_centrality = 0.0;}+ if !page_centrality_rank.is_finite() {+ page_centrality_rank = u64::MAX as f64;+ }++ if !host_centrality_rank.is_finite() {+ host_centrality_rank = u64::MAX as f64;+ }+let mut dmoz_description = None;if let Some(mapper) = self.topics.as_ref() {@@ -202,7 +232,9 @@ impl IndexingWorker {html,backlink_labels,page_centrality,+ page_centrality_rank,host_centrality,+ host_centrality_rank,fetch_time_ms,pre_computed_score: 0.0,node_id: Some(host_node_id),diff --git a/core/src/external_sort.rs b/core/src/external_sort.rsnew file mode 100644index 00000000..cba11672--- /dev/null+++ b/core/src/external_sort.rs@@ -0,0 +1,357 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::Result;+use std::{+ cmp::Reverse,+ collections::BinaryHeap,+ fs::{File, OpenOptions},+ io::{BufReader, BufWriter, Read, Seek, Write},+};++struct Chunk<T> {+ data: Vec<T>,+}++impl<T> Chunk<T> {+ fn new() -> Self {+ Self { data: Vec::new() }+ }++ fn push(&mut self, item: T) {+ self.data.push(item);+ }++ fn len(&self) -> usize {+ self.data.len()+ }++ fn is_empty(&self) -> bool {+ self.data.is_empty()+ }++ fn sort(&mut self)+ where+ T: Ord,+ {+ self.data.sort();+ }++ fn write_into(self, file: &mut File) -> Result<()>+ where+ T: serde::Serialize,+ {+ let mut wrt = BufWriter::new(file);++ for item in self.data {+ let bytes = bincode::serialize(&item)?;+ let size = bytes.len() as u64;++ wrt.write_all(&size.to_le_bytes())?;+ wrt.write_all(&bytes)?;+ }++ wrt.flush()?;++ Ok(())+ }++ fn store(mut self, mut file: TempFile) -> Result<StoredChunk<T>>+ where+ T: serde::Serialize + Ord,+ {+ self.sort();+ self.write_into(&mut file.inner)?;++ StoredChunk::new(file)+ }+}++struct TempDir {+ path: std::path::PathBuf,+}++impl TempDir {+ fn new() -> Result<Self> {+ let path = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());++ std::fs::create_dir(&path)?;++ Ok(Self { path })+ }++ fn as_ref(&self) -> &std::path::Path {+ self.path.as_ref()+ }+}++impl Drop for TempDir {+ fn drop(&mut self) {+ std::fs::remove_dir_all(&self.path).unwrap();+ }+}++struct TempFile {+ inner: File,+ path: std::path::PathBuf,+}++impl TempFile {+ fn new(dir: &TempDir) -> Result<Self> {+ let path = dir.as_ref().join(uuid::Uuid::new_v4().to_string());++ Ok(Self {+ inner: OpenOptions::new()+ .read(true)+ .write(true)+ .create(true)+ .open(&path)?,+ path,+ })+ }+}++impl Drop for TempFile {+ fn drop(&mut self) {+ std::fs::remove_file(&self.path).unwrap();+ }+}++impl Read for TempFile {+ fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {+ self.inner.read(buf)+ }+}++impl Seek for TempFile {+ fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {+ self.inner.seek(pos)+ }+}++struct StoredChunk<T> {+ buf: Vec<u8>,+ data: BufReader<TempFile>,+ _marker: std::marker::PhantomData<T>,+}++impl<T> StoredChunk<T> {+ fn new(mut file: TempFile) -> Result<Self> {+ file.seek(std::io::SeekFrom::Start(0))?;+ Ok(Self {+ buf: Vec::new(),+ data: BufReader::new(file),+ _marker: std::marker::PhantomData,+ })+ }++ fn next(&mut self) -> Option<T>+ where+ T: serde::de::DeserializeOwned,+ {+ self.buf.clear();+ self.buf.resize(u64::BITS as usize / 8, 0);++ self.data.read_exact(&mut self.buf).ok()?;++ let next_size: u64 = u64::from_le_bytes(self.buf.as_slice().try_into().ok()?);++ self.buf.clear();+ self.buf.resize(next_size as usize, 0);++ self.data.read_exact(&mut self.buf).ok()?;++ let next = bincode::deserialize(&self.buf).ok()?;++ Some(next)+ }+}++pub struct ExternalSorter<T> {+ chunk_size: usize,+ _marker: std::marker::PhantomData<T>,+}++impl Default for ExternalSorter<()> {+ fn default() -> Self {+ Self::new()+ }+}++impl<T> ExternalSorter<T> {+ pub fn new() -> Self {+ Self {+ chunk_size: 1_000_000,+ _marker: std::marker::PhantomData,+ }+ }++ pub fn with_chunk_size(mut self, chunk_size: usize) -> Self {+ self.chunk_size = chunk_size;+ self+ }++ pub fn sort<I>(self, iter: I) -> Result<SortedIterator<T>>+ where+ I: Iterator<Item = T>,+ T: serde::Serialize + serde::de::DeserializeOwned + Ord,+ {+ let dir = TempDir::new()?;+ let mut stored_chunks = Vec::new();+ let mut chunk = Chunk::new();++ for item in iter {+ chunk.push(item);++ if chunk.len() >= self.chunk_size {+ let file = TempFile::new(&dir)?;+ let stored_chunk = chunk.store(file)?;+ stored_chunks.push(stored_chunk);+ chunk = Chunk::new();+ }+ }++ if !chunk.is_empty() {+ let file = TempFile::new(&dir)?;+ let stored_chunk = chunk.store(file)?;+ stored_chunks.push(stored_chunk);+ }++ let heads = stored_chunks+ .into_iter()+ .filter_map(|chunk| Head::new(chunk))+ .map(Reverse)+ .collect::<BinaryHeap<_>>();++ Ok(SortedIterator {+ _dir: dir,+ chunks: heads,+ })+ }+}++struct Head<T> {+ item: T,+ rest: StoredChunk<T>,+}++impl<T> Head<T>+where+ T: serde::de::DeserializeOwned,+{+ fn new(mut chunk: StoredChunk<T>) -> Option<Self> {+ let item = chunk.next()?;++ Some(Self { item, rest: chunk })+ }+}++impl<T> Ord for Head<T>+where+ T: Ord,+{+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.item.cmp(&other.item)+ }+}++impl<T> PartialOrd for Head<T>+where+ T: Ord,+{+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl<T> Eq for Head<T> where T: Eq {}++impl<T> PartialEq for Head<T>+where+ T: PartialEq,+{+ fn eq(&self, other: &Self) -> bool {+ self.item.eq(&other.item)+ }+}++impl<T> std::fmt::Debug for Head<T>+where+ T: std::fmt::Debug,+{+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("Head").field("item", &self.item).finish()+ }+}++pub struct SortedIterator<T> {+ // We need to keep the temp dir alive for the lifetime of the iterator+ _dir: TempDir,+ chunks: BinaryHeap<Reverse<Head<T>>>,+}++impl<T> Iterator for SortedIterator<T>+where+ T: serde::de::DeserializeOwned + Ord,+{+ type Item = T;++ fn next(&mut self) -> Option<Self::Item> {+ let mut head = self.chunks.pop()?.0;++ if let Some(next) = head.rest.next() {+ self.chunks.push(Reverse(Head {+ item: next,+ rest: head.rest,+ }));+ }++ Some(head.item)+ }+}++#[cfg(test)]+mod tests {+ use rand::Rng;++ use super::*;++ #[test]+ fn test_external_sort() {+ let mut rng = rand::thread_rng();++ let mut expected = Vec::new();+ let mut data = Vec::new();++ for _ in 0..1_000_000 {+ let item = rng.gen_range(0..1_000_000);++ expected.push(item);+ data.push(item);+ }++ expected.sort();++ let sorted = ExternalSorter::new()+ .with_chunk_size(100_000)+ .sort(data.into_iter())+ .unwrap()+ .collect::<Vec<_>>();++ assert_eq!(sorted.len(), 1_000_000);+ assert_eq!(sorted, expected);+ }+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex f825a70b..c6cba649 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -44,6 +44,7 @@ mod distributed;mod entity_index;mod enum_map;mod executor;+mod external_sort;mod fastfield_reader;pub mod feed;mod human_website_annotations;diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 8985291a..1e4a5ee1 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -104,8 +104,12 @@ pub enum Signal {CrossEncoderTitle,#[serde(rename = "host_centrality")]HostCentrality,+ #[serde(rename = "host_centrality_rank")]+ HostCentralityRank,#[serde(rename = "page_centrality")]PageCentrality,+ #[serde(rename = "page_centrality_rank")]+ PageCentralityRank,#[serde(rename = "is_homepage")]IsHomepage,#[serde(rename = "fetch_time_ms")]@@ -136,7 +140,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 35] = [+pub const ALL_SIGNALS: [Signal; 37] = [Signal::Bm25Title,Signal::Bm25TitleBigrams,Signal::Bm25TitleTrigrams,@@ -160,7 +164,9 @@ pub const ALL_SIGNALS: [Signal; 35] = [Signal::CrossEncoderSnippet,Signal::CrossEncoderTitle,Signal::HostCentrality,+ Signal::HostCentralityRank,Signal::PageCentrality,+ Signal::PageCentralityRank,Signal::IsHomepage,Signal::FetchTimeMs,Signal::UpdateTimestamp,@@ -189,6 +195,11 @@ fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f6}}+#[inline]+fn score_rank(rank: f64) -> f64 {+ 1.0 / (rank + 1.0)+}+#[inline]fn score_trackers(num_trackers: f64) -> f64 {1.0 / (num_trackers + 1.0)@@ -284,7 +295,9 @@ impl Signal {Signal::CrossEncoderSnippet => 0.17,Signal::CrossEncoderTitle => 0.17,Signal::HostCentrality => 0.5,+ Signal::HostCentralityRank => 0.0,Signal::PageCentrality => 0.25,+ Signal::PageCentralityRank => 0.0,Signal::QueryCentrality => 0.0,Signal::IsHomepage => 0.0005,Signal::FetchTimeMs => 0.001,@@ -345,6 +358,13 @@ impl Signal {field_value.map(|val| val as f64 / FLOAT_SCALING as f64)}+ Signal::HostCentralityRank | Signal::PageCentralityRank => {+ let field_value: Option<u64> = self+ .fastfield_value(signal_aggregator, doc)+ .and_then(|val| val.into());++ field_value.map(|val| val as f64).map(score_rank)+ }Signal::IsHomepage => {let field_value: Option<u64> = self.fastfield_value(signal_aggregator, doc)@@ -470,7 +490,9 @@ impl Signal {let value = match self {Signal::HostCentrality => Some(webpage.host_centrality),+ Signal::HostCentralityRank => Some(webpage.host_centrality_rank),Signal::PageCentrality => Some(webpage.page_centrality),+ Signal::PageCentralityRank => Some(webpage.page_centrality_rank),Signal::IsHomepage => Some(webpage.html.is_homepage().into()),Signal::FetchTimeMs => {let fetch_time_ms = webpage.fetch_time_ms as usize;@@ -571,7 +593,9 @@ impl Signal {fn as_fastfield(&self) -> Option<FastField> {match self {Signal::HostCentrality => Some(FastField::HostCentrality),+ Signal::HostCentralityRank => Some(FastField::HostCentralityRank),Signal::PageCentrality => Some(FastField::PageCentrality),+ Signal::PageCentralityRank => Some(FastField::PageCentralityRank),Signal::IsHomepage => Some(FastField::IsHomepage),Signal::FetchTimeMs => Some(FastField::FetchTimeMs),Signal::UpdateTimestamp => Some(FastField::LastUpdated),diff --git a/core/src/schema.rs b/core/src/schema.rsindex 99cf3ffd..8ca974f7 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -201,7 +201,9 @@ impl TextField {pub enum FastField {IsHomepage,HostCentrality,+ HostCentralityRank,PageCentrality,+ PageCentralityRank,FetchTimeMs,LastUpdated,TrackerScore,@@ -240,7 +242,9 @@ impl FastField {pub fn name(&self) -> &str {match self {FastField::HostCentrality => "host_centrality",+ FastField::HostCentralityRank => "host_centrality_rank",FastField::PageCentrality => "page_centrality",+ FastField::PageCentralityRank => "page_centrality_rank",FastField::IsHomepage => "is_homepage",FastField::FetchTimeMs => "fetch_time_ms",FastField::LastUpdated => "last_updated",@@ -290,7 +294,7 @@ pub enum Field {Text(TextField),}-static ALL_FIELDS: [Field; 64] = [+static ALL_FIELDS: [Field; 66] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -324,7 +328,9 @@ static ALL_FIELDS: [Field; 64] = [// FAST FIELDSField::Fast(FastField::IsHomepage),Field::Fast(FastField::HostCentrality),+ Field::Fast(FastField::HostCentralityRank),Field::Fast(FastField::PageCentrality),+ Field::Fast(FastField::PageCentralityRank),Field::Fast(FastField::FetchTimeMs),Field::Fast(FastField::LastUpdated),Field::Fast(FastField::TrackerScore),@@ -481,9 +487,15 @@ impl Field {Field::Fast(FastField::HostCentrality) => {IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())}+ Field::Fast(FastField::HostCentralityRank) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }Field::Fast(FastField::PageCentrality) => {IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())}+ Field::Fast(FastField::PageCentralityRank) => {+ IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())+ }Field::Fast(FastField::FetchTimeMs) => {IndexingOption::Integer(NumericOptions::default().set_fast().set_indexed())}@@ -693,7 +705,9 @@ impl FastField {match self {FastField::IsHomepage => DataType::U64,FastField::HostCentrality => DataType::U64,+ FastField::HostCentralityRank => DataType::U64,FastField::PageCentrality => DataType::U64,+ FastField::PageCentralityRank => DataType::U64,FastField::FetchTimeMs => DataType::U64,FastField::LastUpdated => DataType::U64,FastField::TrackerScore => DataType::U64,diff --git a/core/src/webgraph/centrality/approx_harmonic.rs b/core/src/webgraph/centrality/approx_harmonic.rsindex 8252ae36..4ac86024 100644--- a/core/src/webgraph/centrality/approx_harmonic.rs+++ b/core/src/webgraph/centrality/approx_harmonic.rs@@ -48,7 +48,7 @@ impl ApproxHarmonic {let norm = num_nodes as f64 / (num_samples as f64 * (num_nodes as f64 - 1.0));sampled.into_par_iter().progress().for_each(|source| {- let dists = graph.raw_distances(source);+ let dists = graph.raw_distances_with_max(source, 5);let res = res.lock().unwrap();for (target, dist) in dists {diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 06bb286f..f8b02237 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -335,6 +335,7 @@ impl WebgraphBuilder {pub trait ShortestPaths {fn distances(&self, source: Node) -> BTreeMap<Node, u8>;fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;+ fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8>;fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;}@@ -343,6 +344,7 @@ fn dijkstra_multi<F1, F2, L>(sources: &[NodeID],node_edges: F1,edge_node: F2,+ max_dist: Option<u8>,) -> BTreeMap<NodeID, u8>whereL: EdgeLabel,@@ -367,6 +369,12 @@ wherecontinue;}+ if let Some(max_dist) = max_dist {+ if cost > max_dist {+ return distances;+ }+ }+for edge in node_edges(v) {if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&u8::MAX) {let d = cost + 1;@@ -389,11 +397,21 @@ impl ShortestPaths for Webgraph {.collect()}+ fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8> {+ dijkstra_multi(+ &[source],+ |node| self.raw_outgoing_edges(&node),+ |edge| edge.to,+ Some(max_dist),+ )+ }+fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {dijkstra_multi(&[source],|node| self.raw_outgoing_edges(&node),|edge| edge.to,+ None,)}@@ -402,6 +420,7 @@ impl ShortestPaths for Webgraph {&[source],|node| self.raw_ingoing_edges(&node),|edge| edge.from,+ None,)}diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rsindex 1c32cf4c..4ca386cb 100644--- a/core/src/webpage/html/into_tantivy.rs+++ b/core/src/webpage/html/into_tantivy.rs@@ -522,7 +522,9 @@ impl Html {| Field::Text(TextField::SafetyClassification)| Field::Text(TextField::InsertionTimestamp)| Field::Fast(FastField::HostCentrality)+ | Field::Fast(FastField::HostCentralityRank)| Field::Fast(FastField::PageCentrality)+ | Field::Fast(FastField::PageCentralityRank)| Field::Fast(FastField::FetchTimeMs)| Field::Fast(FastField::PreComputedScore)| Field::Fast(FastField::Region)diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 1f4926a2..588c1e99 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -45,7 +45,9 @@ pub struct Webpage {pub html: Html,pub backlink_labels: Vec<String>,pub host_centrality: f64,+ pub host_centrality_rank: f64,pub page_centrality: f64,+ pub page_centrality_rank: f64,pub fetch_time_ms: u64,pub pre_computed_score: f64,pub node_id: Option<NodeID>,@@ -68,7 +70,9 @@ impl Default for Webpage {},backlink_labels: Default::default(),host_centrality: Default::default(),+ host_centrality_rank: u64::MAX as f64,page_centrality: Default::default(),+ page_centrality_rank: u64::MAX as f64,fetch_time_ms: Default::default(),pre_computed_score: Default::default(),node_id: Default::default(),@@ -86,15 +90,7 @@ impl Webpage {Ok(Self {html,- backlink_labels: Vec::new(),- host_centrality: 0.0,- page_centrality: 0.0,- fetch_time_ms: 0,- pre_computed_score: 0.0,- node_id: None,- dmoz_description: None,- safety_classification: None,- inserted_at: Utc::now(),+ ..Default::default()})}@@ -175,6 +171,13 @@ impl Webpage {(self.host_centrality * FLOAT_SCALING as f64) as u64,);+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostCentralityRank).name())+ .expect("Failed to get host_centrality_rank field"),+ self.host_centrality_rank as u64,+ );+doc.add_u64(schema.get_field(Field::Fast(FastField::PageCentrality).name())@@ -182,6 +185,13 @@ impl Webpage {(self.page_centrality * FLOAT_SCALING as f64) as u64,);+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::PageCentralityRank).name())+ .expect("Failed to get page_centrality_rank field"),+ self.page_centrality_rank as u64,+ );+doc.add_u64(schema.get_field(Field::Fast(FastField::FetchTimeMs).name())diff --git a/justfile b/justfileindex 77b67785..d29ea7be 100644--- a/justfile+++ b/justfile@@ -12,8 +12,6 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"@dev-webgraph:cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"-@dev-alice:- cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- alice serve configs/alice.toml"@dev-frontend:cd frontend && npm run devdiff --git a/scripts/run_dev.py b/scripts/run_dev.pyindex 080674ef..42813287 100755--- a/scripts/run_dev.py+++ b/scripts/run_dev.py@@ -10,7 +10,6 @@ os.environ["DYLD_LIBRARY_PATH"] = "libtorch/lib"parser = argparse.ArgumentParser()parser.add_argument("--release", action="store_true")-parser.add_argument("--alice", action="store_true")args = parser.parse_args()@@ -24,9 +23,6 @@ processes.append(subprocess.Popen(["just", "dev-search-server"]))processes.append(subprocess.Popen(["just", "dev-webgraph"]))processes.append(subprocess.Popen(["just", "dev-frontend"]))-if args.alice:- processes.append(subprocess.Popen(["just", "dev-alice"]))-# kill processes on ctrl-cimport time