Skip to content

web search done right

Webgraph inverted index (#232)

Mikkel Denker658ac6f682a3
diff --git a/Cargo.lock b/Cargo.lock
index e656609c..583ac02c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5203,6 +5203,7 @@ dependencies = [
"num-rational",
"openraft",
"optics",
+ "paste",
"percent-encoding",
"postcard",
"proptest",
@@ -5459,6 +5460,7 @@ dependencies = [
"arc-swap",
"async-trait",
"base64 0.22.1",
+ "bincode",
"binggan",
"bitpacking",
"byteorder 1.5.0",
diff --git a/Cargo.toml b/Cargo.toml
index cb4ab77b..6f11f62e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,7 +30,7 @@ debug = true
[workspace.dependencies]
ahash = "0.8.11"
aho-corasick = "1.0"
-anyhow = {version = "1.0.72", features = ["backtrace"]}
+anyhow = { version = "1.0.72", features = ["backtrace"] }
arc-swap = "1.5.0"
async-channel = "1.8.0"
async-stream = "0.3.3"
@@ -38,66 +38,66 @@ async-trait = "0.1"
axum = "0.7.2"
axum-macros = "0.4.0"
base64 = "0.22.0"
-bincode = {git = "https://github.com/bincode-org/bincode", rev = "100685b", features = [
+bincode = { git = "https://github.com/bincode-org/bincode", rev = "100685b", features = [
"serde",
-]}
+] }
bindgen = "0.69.2"
binggan = "0.8.0"
bitflags = "2.5.0"
-bitpacking = {version = "0.9.2", default-features = false, features = [
+bitpacking = { version = "0.9.2", default-features = false, features = [
"bitpacker4x",
"bitpacker1x",
-]}
-bitvec = {version = "1.0.1", features = ["serde"]}
+] }
+bitvec = { version = "1.0.1", features = ["serde"] }
bytecount = "0.6.7"
-bytemuck = {version = "1.13.1", features = ["derive"]}
+bytemuck = { version = "1.13.1", features = ["derive"] }
byteorder = "1.4.3"
candle-core = "0.3.3"
candle-nn = "0.3.3"
candle-transformers = "0.3.3"
-cc = {version = "1", features = ["parallel"]}
+cc = { version = "1", features = ["parallel"] }
census = "0.4.2"
chardetng = "0.1.17"
chitchat = "0.5.0"
-chrono = {version = "0.4.23", features = ["serde"]}
-clap = {version = "4.4.6", features = ["derive"]}
+chrono = { version = "0.4.23", features = ["serde"] }
+clap = { version = "4.4.6", features = ["derive"] }
cmake = "0.1"
-common = {path = "crates/common"}
+common = { path = "crates/common" }
crc32fast = "1.3.2"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
csv = "1.1.6"
-dashmap = {version = "5.4.0", features = ["rayon"]}
+dashmap = { version = "5.4.0", features = ["rayon"] }
deadpool = "0.12.1"
dirs = "5.0.1"
downcast-rs = "1.2.0"
encoding_rs = "0.8.31"
enum_dispatch = "0.3.12"
eventsource-stream = "0.2.3"
-fail = {version = "0.5.0", features = ["failpoints"]}
+fail = { version = "0.5.0", features = ["failpoints"] }
fastdivide = "0.4.0"
fend-core = "1.4.8"
flate2 = "1.0.28"
fnv = "1.0.3"
-fs4 = {version = "0.8.0"}
-fst = {version = "0.4.7", features = ["levenshtein"]}
+fs4 = { version = "0.8.0" }
+fst = { version = "0.4.7", features = ["levenshtein"] }
futures = "0.3.21"
-futures-util = {version = "0.3.28"}
-half = {version = "2.2.1", features = ["serde"]}
-hashbrown = {version = "0.14.0", features = ["serde"]}
+futures-util = { version = "0.3.28" }
+half = { version = "2.2.1", features = ["serde"] }
+hashbrown = { version = "0.14.0", features = ["serde"] }
htmlescape = "0.3.1"
http = "1.0.0"
image = "0.25.1"
-indicatif = {version = "0.17.7", features = ["rayon"]}
+indicatif = { version = "0.17.7", features = ["rayon"] }
insta = "1.31"
itertools = "0.13.0"
js-sys = "0.3.60"
-lalrpop = {version = "0.20.0", features = ["lexer"]}
-lalrpop-util = {version = "0.20.0", features = ["lexer"]}
-lending-iter = {path = "crates/lending-iter"}
+lalrpop = { version = "0.20.0", features = ["lexer"] }
+lalrpop-util = { version = "0.20.0", features = ["lexer"] }
+lending-iter = { path = "crates/lending-iter" }
levenshtein_automata = "0.2.1"
libc = "0.2.142"
-log = {version = "0.4", features = ["release_max_level_info"]}
+log = { version = "0.4", features = ["release_max_level_info"] }
logos = "0.13.0"
lru = "0.12.0"
lz-str = "0.2.1"
@@ -116,15 +116,19 @@ nom = "7.1.3"
num-bigint = "0.4.1"
num-rational = "0.4.1"
oneshot = "0.1.7"
-openraft = {version = "0.9.1", features = [
+openraft = { version = "0.9.1", features = [
"storage-v2",
"serde",
"tracing-log",
-]}
-ownedbytes = {path = "crates/ownedbytes"}
+] }
+ownedbytes = { path = "crates/ownedbytes" }
paste = "1.0.11"
percent-encoding = "2.3.0"
-postcard = {version = "1.0.8", features = ["experimental-derive", "alloc", "use-std"]}
+postcard = { version = "1.0.8", features = [
+ "experimental-derive",
+ "alloc",
+ "use-std",
+] }
pretty_assertions = "1.2.1"
proptest = "1.2.0"
proptest-derive = "0.4.0"
@@ -134,54 +138,54 @@ rand = "0.8.5"
rand_distr = "0.4.3"
rayon = "1.5.3"
redb = "2.0.0"
-regex = {version = "1.6.0", features = ["std", "unicode"]}
-reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
+regex = { version = "1.6.0", features = ["std", "unicode"] }
+reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
ring = "0.17.3"
rio_api = "0.8.4"
rio_turtle = "0.8.4"
-rkyv = {version = "0.7.44", features = ["validation", "bytecheck"]}
-rusqlite = {version = "0.29.0", features = [
+rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }
+rusqlite = { version = "0.29.0", features = [
"bundled",
"modern-full",
"buildtime_bindgen",
-]}
-rust-s3 = {version = "0.34.0", features = ["blocking", "tokio"]}
+] }
+rust-s3 = { version = "0.34.0", features = ["blocking", "tokio"] }
rust-stemmers = "1.2.0"
rustc-hash = "1.1"
safetensors = "0.3.1"
-scylla = {version = "0.12.0", features = ["chrono"]}
-serde = {version = "1.0.137", features = ["rc", "derive"]}
+scylla = { version = "0.12.0", features = ["chrono"] }
+serde = { version = "1.0.137", features = ["rc", "derive"] }
serde-wasm-bindgen = "0.6.3"
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
smallvec = "1.8.0"
stable_deref_trait = "1.2.0"
-strum = {version = "0.26.2", features = ["derive"]}
-tantivy = {path = "crates/tantivy"}
-tempfile = {version = "3.3.0"}
+strum = { version = "0.26.2", features = ["derive"] }
+tantivy = { path = "crates/tantivy" }
+tempfile = { version = "3.3.0" }
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
-time = {version = "0.3.10", features = ["serde-well-known", "macros"]}
+time = { version = "0.3.10", features = ["serde-well-known", "macros"] }
tokenizers = "0.13.2"
-tokio = {version = "1.23.1", features = ["full"]}
+tokio = { version = "1.23.1", features = ["full"] }
tokio-stream = "0.1.11"
toml = "0.8.2"
-tower = {version = "0.4.13", features = ["limit"]}
-tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}
-tracing = {version = "0.1.34", features = ["release_max_level_info"]}
-tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
+tower = { version = "0.4.13", features = ["limit"] }
+tower-http = { version = "0.5.0", features = ["compression-gzip", "cors"] }
+tracing = { version = "0.1.34", features = ["release_max_level_info"] }
+tracing-subscriber = { version = "0.3.11", features = ["env-filter"] }
tracing-test = "0.2.4"
unicode-normalization = "0.1.23"
unicode-segmentation = "1.11.0"
-url = {version = "2.4.0", features = ["serde"]}
-utoipa = {version = "4.2.3", features = ["axum_extras"]}
-utoipa-swagger-ui = {version = "7.0.0", features = ["axum"]}
-uuid = {version = "1.1.2", features = ["serde", "v4"]}
+url = { version = "2.4.0", features = ["serde"] }
+utoipa = { version = "4.2.3", features = ["axum_extras"] }
+utoipa-swagger-ui = { version = "7.0.0", features = ["axum"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
wasm-bindgen = "0.2.83"
-whatlang = {version = "0.16.0", features = ["serde"]}
-xxhash-rust = {version = "0.8.10", features = ["xxh3", "const_xxh3"]}
+whatlang = { version = "0.16.0", features = ["serde"] }
+xxhash-rust = { version = "0.8.10", features = ["xxh3", "const_xxh3"] }
zipf = "7.0.0"
-zstd = {version = "0.13", features = ["experimental"]}
+zstd = { version = "0.13", features = ["experimental"] }
[profile.test.package]
flate2.opt-level = 3
diff --git a/configs/webgraph/create.toml b/configs/webgraph/create.toml
index d3e651cd..4fd25320 100644
--- a/configs/webgraph/create.toml
+++ b/configs/webgraph/create.toml
@@ -1,6 +1,5 @@
limit_warc_files = 5
-host_graph_base_path = "data/webgraph"
-page_graph_base_path = "data/webgraph_page"
+graph_base_path = "data/webgraph"
batch_size = 1
canonical_index_path = "data/canonical"
diff --git a/configs/webgraph/page_server.toml b/configs/webgraph/page_server.toml
deleted file mode 100644
index 7c934091..00000000
--- a/configs/webgraph/page_server.toml
+++ /dev/null
@@ -1,8 +0,0 @@
-cluster_id = "dev_webgraph"
-gossip_addr = "0.0.0.0:3012"
-gossip_seed_nodes = ["0.0.0.0:3005"]
-granularity = "page"
-graph_path = "data/webgraph_page"
-host = "0.0.0.0:3011"
-shard = 0
-inbound_similarity_path = "data/centrality/inbound_similarity"
diff --git a/configs/webgraph/profile.toml b/configs/webgraph/profile.toml
index 880693f4..5959188c 100644
--- a/configs/webgraph/profile.toml
+++ b/configs/webgraph/profile.toml
@@ -1,5 +1,4 @@
graph_base_path = "data/webgraph"
-level = "host"
[warc_source]
folder = "data"
diff --git a/configs/webgraph/host_server.toml b/configs/webgraph/server.toml
similarity index 77%
rename from configs/webgraph/host_server.toml
rename to configs/webgraph/server.toml
index f59efb26..6545d6fe 100644
--- a/configs/webgraph/host_server.toml
+++ b/configs/webgraph/server.toml
@@ -1,8 +1,7 @@
cluster_id = "dev_webgraph"
gossip_addr = "0.0.0.0:3007"
gossip_seed_nodes = ["0.0.0.0:3005"]
-granularity = "host"
-graph_path = "data/webgraph_host"
+graph_path = "data/webgraph"
host = "0.0.0.0:3003"
shard = 0
inbound_similarity_path = "data/centrality/inbound_similarity"
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 2e64548b..e1791bae 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -73,6 +73,7 @@ num-bigint.workspace = true
num-rational.workspace = true
openraft.workspace = true
optics = { path = "../optics" }
+paste.workspace = true
percent-encoding.workspace = true
postcard.workspace = true
publicsuffix.workspace = true
diff --git a/crates/core/benches/harmonic_centrality.rs b/crates/core/benches/harmonic_centrality.rs
index ba4c4dda..21f6b8d0 100644
--- a/crates/core/benches/harmonic_centrality.rs
+++ b/crates/core/benches/harmonic_centrality.rs
@@ -4,7 +4,9 @@ use stract::webgraph::{centrality::harmonic::HarmonicCentrality, WebgraphBuilder
const WEBGRAPH_PATH: &str = "data/webgraph";
pub fn criterion_benchmark(c: &mut Criterion) {
- let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH).open();
+ let webgraph = WebgraphBuilder::new(WEBGRAPH_PATH, 0u64.into())
+ .open()
+ .unwrap();
c.bench_function("Harmonic centrality calculation", |b| {
b.iter(|| {
for _ in 0..10 {
diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rs
index 28eb6d9a..7292bb9b 100644
--- a/crates/core/examples/distributed_harmonic.rs
+++ b/crates/core/examples/distributed_harmonic.rs
@@ -1,7 +1,11 @@
use std::{net::SocketAddr, time::Duration};
use clap::Parser;
-use stract::{distributed::member::ShardId, webgraph::Node, webpage::url_ext::UrlExt};
+use stract::{
+ distributed::member::ShardId,
+ webgraph::{Edge, Node},
+ webpage::url_ext::UrlExt,
+};
use tracing_subscriber::util::SubscriberInitExt;
#[derive(Parser)]
@@ -66,11 +70,9 @@ fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Resul
let a_path = path.join("graph_a");
let b_path = path.join("graph_b");
- let mut a =
- stract::webgraph::WebgraphWriter::new(a_path, Default::default(), Default::default(), None);
+ let mut a = stract::webgraph::WebgraphBuilder::new(a_path, 0u64.into()).open()?;
- let mut b =
- stract::webgraph::WebgraphWriter::new(b_path, Default::default(), Default::default(), None);
+ let mut b = stract::webgraph::WebgraphBuilder::new(b_path, 0u64.into()).open()?;
for (i, record) in warc.records().flatten().enumerate() {
let webpage = match stract::webpage::Html::parse_without_text(
@@ -97,19 +99,31 @@ fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Resul
if dest_domain.is_some() && source_domain.is_some() && dest_domain != source_domain {
if i <= num_records / 2 {
- a.insert(source, destination, link.text, link.rel);
+ a.insert(Edge {
+ from: source,
+ to: destination,
+ rel_flags: link.rel,
+ label: link.text,
+ sort_score: 0.0,
+ })?;
} else {
- b.insert(source, destination, link.text, link.rel);
+ b.insert(Edge {
+ from: source,
+ to: destination,
+ rel_flags: link.rel,
+ label: link.text,
+ sort_score: 0.0,
+ })?;
}
}
}
}
- a.commit();
- b.commit();
+ a.commit()?;
+ a.optimize_read()?;
- a.finalize();
- b.finalize();
+ b.commit()?;
+ b.optimize_read()?;
Ok(())
}
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index 78ad892a..be38527e 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -93,7 +93,7 @@ use utoipa_swagger_ui::SwaggerUi;
explore::ExploreExportOpticParams,
crate::webgraph::Node,
- crate::webgraph::FullEdge,
+ crate::webgraph::Edge,
crate::search_prettifier::StructuredData,
crate::search_prettifier::OneOrManyString,
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 80699908..9940cbc4 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -33,7 +33,7 @@ use crate::{
ranking::models::lambdamart::LambdaMART,
searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher, SearchClient},
similar_hosts::SimilarHostsFinder,
- webgraph::remote::{Host, Page, RemoteWebgraph},
+ webgraph::remote::RemoteWebgraph,
};
use crate::ranking::models::cross_encoder::CrossEncoderModel;
@@ -72,14 +72,13 @@ pub struct Counters {
pub struct State {
pub config: ApiConfig,
- pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph<Host>>>>,
- pub page_webgraph: Arc<RemoteWebgraph<Page>>,
- pub host_webgraph: Arc<RemoteWebgraph<Host>>,
+ pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph>>>,
+ pub webgraph: Arc<RemoteWebgraph>,
pub autosuggest: Autosuggest,
pub counters: Counters,
pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,
pub _cluster: Arc<Cluster>,
- pub similar_hosts: SimilarHostsFinder<Host>,
+ pub similar_hosts: SimilarHostsFinder,
}
pub async fn favicon() -> impl IntoResponse {
@@ -172,8 +171,7 @@ pub async fn router(
None => Bangs::empty(),
};
- let host_webgraph = RemoteWebgraph::<Host>::new(cluster.clone()).await;
- let page_webgraph = RemoteWebgraph::<Page>::new(cluster.clone()).await;
+ let webgraph = RemoteWebgraph::new(cluster.clone()).await;
let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster)).await;
let live_searcher = LiveSearcher::new(Arc::clone(&cluster));
@@ -228,21 +226,19 @@ pub async fn router(
.await;
}
- let host_webgraph = Arc::new(host_webgraph);
- let page_webgraph = Arc::new(page_webgraph);
+ let webgraph = Arc::new(webgraph);
- searcher = searcher.with_webgraph(Arc::clone(&host_webgraph));
+ searcher = searcher.with_webgraph(Arc::clone(&webgraph));
let similar_hosts =
- SimilarHostsFinder::new(Arc::clone(&host_webgraph), config.max_similar_hosts);
+ SimilarHostsFinder::new(Arc::clone(&webgraph), config.max_similar_hosts);
Arc::new(State {
config: config.clone(),
searcher: Arc::new(searcher),
autosuggest,
counters,
- host_webgraph,
- page_webgraph,
+ webgraph,
improvement_queue: query_store_queue,
_cluster: cluster,
similar_hosts,
diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rs
index 648cdcaf..282c2594 100644
--- a/crates/core/src/api/webgraph.rs
+++ b/crates/core/src/api/webgraph.rs
@@ -22,7 +22,13 @@ use utoipa::{IntoParams, ToSchema};
use crate::{
config::WebgraphGranularity,
- webgraph::{EdgeLimit, FullEdge, Node},
+ webgraph::{
+ query::{
+ FullBacklinksQuery, FullForwardlinksQuery, FullHostBacklinksQuery,
+ FullHostForwardlinksQuery,
+ },
+ Edge, EdgeLimit, Node,
+ },
};
use super::State;
@@ -104,7 +110,7 @@ pub mod host {
Err(_) => return Err(StatusCode::BAD_REQUEST),
}
- match state.host_webgraph.knows(params.host).await {
+ match state.webgraph.knows(params.host).await {
Ok(Some(node)) => Ok(Json(KnowsHost::Known {
host: node.as_str().to_string(),
})),
@@ -120,7 +126,7 @@ pub mod host {
path = "/beta/api/webgraph/host/ingoing",
params(HostLinksParams),
responses(
- (status = 200, description = "Incoming links for a particular host", body = Vec<FullEdge>),
+ (status = 200, description = "Incoming links for a particular host", body = Vec<Edge>),
)
)]
pub async fn ingoing_hosts(
@@ -144,7 +150,7 @@ pub mod host {
path = "/beta/api/webgraph/host/outgoing",
params(HostLinksParams),
responses(
- (status = 200, description = "Outgoing links for a particular host", body = Vec<FullEdge>),
+ (status = 200, description = "Outgoing links for a particular host", body = Vec<Edge>),
)
)]
pub async fn outgoing_hosts(
@@ -182,7 +188,7 @@ pub mod page {
path = "/beta/api/webgraph/page/ingoing",
params(PageLinksParams),
responses(
- (status = 200, description = "Incoming links for a particular page", body = Vec<FullEdge>),
+ (status = 200, description = "Incoming links for a particular page", body = Vec<Edge>),
)
)]
pub async fn ingoing_pages(
@@ -205,7 +211,7 @@ pub mod page {
path = "/beta/api/webgraph/page/outgoing",
params(PageLinksParams),
responses(
- (status = 200, description = "Outgoing links for a particular page", body = Vec<FullEdge>),
+ (status = 200, description = "Outgoing links for a particular page", body = Vec<Edge>),
)
)]
pub async fn outgoing_pages(
@@ -229,18 +235,18 @@ async fn ingoing_links(
state: Arc<State>,
node: Node,
level: WebgraphGranularity,
-) -> anyhow::Result<Vec<FullEdge>> {
+) -> anyhow::Result<Vec<Edge>> {
match level {
WebgraphGranularity::Host => {
state
- .host_webgraph
- .ingoing_edges(node, EdgeLimit::Limit(1024))
+ .webgraph
+ .search(FullHostBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
.await
}
WebgraphGranularity::Page => {
state
- .page_webgraph
- .ingoing_edges(node, EdgeLimit::Limit(1024))
+ .webgraph
+ .search(FullBacklinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
.await
}
}
@@ -250,18 +256,18 @@ async fn outgoing_links(
state: Arc<State>,
node: Node,
level: WebgraphGranularity,
-) -> anyhow::Result<Vec<FullEdge>> {
+) -> anyhow::Result<Vec<Edge>> {
match level {
WebgraphGranularity::Host => {
state
- .host_webgraph
- .outgoing_edges(node, EdgeLimit::Limit(1024))
+ .webgraph
+ .search(FullHostForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
.await
}
WebgraphGranularity::Page => {
state
- .page_webgraph
- .outgoing_edges(node, EdgeLimit::Limit(1024))
+ .webgraph
+ .search(FullForwardlinksQuery::new(node).with_limit(EdgeLimit::Limit(1024)))
.await
}
}
diff --git a/crates/core/src/backlink_grouper.rs b/crates/core/src/backlink_grouper.rs
index 7ae68130..b7a1a792 100644
--- a/crates/core/src/backlink_grouper.rs
+++ b/crates/core/src/backlink_grouper.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{log_group::HarmonicRankGroup, webgraph::Edge};
+use crate::{log_group::HarmonicRankGroup, webgraph::SmallEdgeWithLabel};
/// Number of groups to divide the backlinks into.
/// If this is changed, also change the grouped backlink fields in the schema.
@@ -23,7 +23,7 @@ const NUM_GROUPS: u64 = 10;
#[derive(Debug)]
pub struct Group {
group: u64,
- backlinks: Vec<Edge<String>>,
+ backlinks: Vec<SmallEdgeWithLabel>,
}
impl Group {
@@ -34,7 +34,7 @@ impl Group {
}
}
- fn insert(&mut self, backlink: Edge<String>) {
+ fn insert(&mut self, backlink: SmallEdgeWithLabel) {
self.backlinks.push(backlink);
}
@@ -42,7 +42,7 @@ impl Group {
self.group
}
- pub fn backlinks(&self) -> &[Edge<String>] {
+ pub fn backlinks(&self) -> &[SmallEdgeWithLabel] {
&self.backlinks
}
}
@@ -66,7 +66,7 @@ impl GroupedBacklinks {
self.groups.get(group as usize)
}
- fn add(&mut self, group: u64, backlink: Edge<String>) {
+ fn add(&mut self, group: u64, backlink: SmallEdgeWithLabel) {
if let Some(group) = self.groups.get_mut(group as usize) {
group.insert(backlink)
}
@@ -93,10 +93,8 @@ impl BacklinkGrouper {
}
}
- pub fn add(&mut self, backlink: Edge<String>) {
- let rank = backlink.from.host_rank();
-
- let group = self.grouper.group(rank);
+ pub fn add(&mut self, backlink: SmallEdgeWithLabel, host_rank: u64) {
+ let group = self.grouper.group(host_rank);
self.groups.add(group, backlink);
}
@@ -107,73 +105,103 @@ impl BacklinkGrouper {
#[cfg(test)]
mod tests {
- use crate::webgraph::NodeDatum;
+ use crate::webgraph::NodeID;
use super::*;
#[test]
fn test_grouped_backlinks() {
let mut grouper = BacklinkGrouper::new(10);
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 0u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 1u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 2u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 3u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 4u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 5u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 6u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 7u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 8u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
- grouper.add(Edge {
- from: NodeDatum::new(0u64, 9u64),
- to: NodeDatum::new(1u64, 0u64),
- label: String::new(),
- rel: Default::default(),
- });
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(10u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 0,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(9u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 1,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(8u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 2,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(7u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 3,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(6u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 4,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(5u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 5,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(4u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 6,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(3u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 7,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(2u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 8,
+ );
+ grouper.add(
+ SmallEdgeWithLabel {
+ from: NodeID::from(1u64),
+ to: NodeID::from(1u64),
+ label: String::new(),
+ rel_flags: Default::default(),
+ },
+ 9,
+ );
{
let groups = grouper.groups();
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index ac420011..184d04d2 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -97,14 +97,14 @@ pub struct IndexerDualEncoderConfig {
#[derive(Debug, serde::Deserialize, Clone)]
pub struct WebgraphConstructConfig {
- pub host_graph_base_path: Option<String>,
- pub page_graph_base_path: Option<String>,
+ pub graph_base_path: String,
+ pub shard: ShardId,
+ pub host_centrality_rank_store_path: String,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub batch_size: Option<usize>,
pub canonical_index_path: Option<String>,
- pub host_centrality_rank_store_path: Option<String>,
#[serde(default = "defaults::Webgraph::merge_all_segments")]
pub merge_all_segments: bool,
@@ -537,7 +537,6 @@ pub struct WebgraphServerConfig {
pub host: SocketAddr,
pub shard: ShardId,
pub graph_path: String,
- pub granularity: WebgraphGranularity,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 4b6a2e93..bbe8194a 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -33,7 +33,8 @@ use crate::crawler::WeightedUrl;
use crate::distributed::cluster::Cluster;
use crate::external_sort::ExternalSorter;
use crate::log_group::HarmonicRankGroup;
-use crate::webgraph::remote::{Host, Page, RemoteWebgraph};
+use crate::webgraph::query::Id2NodeQuery;
+use crate::webgraph::remote::RemoteWebgraph;
use crate::webgraph::Node;
use crate::webpage::url_ext::UrlExt;
use crate::SortableFloat;
@@ -172,8 +173,7 @@ pub struct CrawlPlanner {
host_centrality: Arc<speedy_kv::Db<NodeID, f64>>,
host_centrality_rank: Arc<speedy_kv::Db<NodeID, u64>>,
page_centrality: Arc<speedy_kv::Db<NodeID, f64>>,
- page_graph: RemoteWebgraph<Page>,
- host_graph: RemoteWebgraph<Host>,
+ webgraph: RemoteWebgraph,
config: CrawlPlannerConfig,
harmonic_group: Arc<HarmonicRankGroup>,
@@ -193,10 +193,8 @@ impl CrawlPlanner {
) -> Result<Self> {
Self::check_config(&config)?;
- let page_graph = RemoteWebgraph::new(cluster.clone()).await;
- page_graph.await_ready().await;
- let host_graph = RemoteWebgraph::new(cluster.clone()).await;
- host_graph.await_ready().await;
+ let webgraph = RemoteWebgraph::new(cluster.clone()).await;
+ webgraph.await_ready().await;
let domain_boosts = config
.domain_boosts
@@ -209,8 +207,7 @@ impl CrawlPlanner {
Ok(Self {
host_centrality: Arc::new(host_centrality),
page_centrality: Arc::new(page_centrality),
- page_graph,
- host_graph,
+ webgraph,
domain_boosts,
excluded_domains: config
.excluded_domains
@@ -404,9 +401,14 @@ impl CrawlPlanner {
}
let nodes = chunk.collect::<Vec<_>>();
- let page_graph = self.page_graph.clone();
+ let webgraph = self.webgraph.clone();
- futures.push_back(async move { page_graph.batch_get_node(&nodes).await.unwrap() });
+ futures.push_back(async move {
+ webgraph
+ .batch_search(nodes.into_iter().map(Id2NodeQuery::Page).collect())
+ .await
+ .unwrap()
+ });
}
while let Some(nodes) = futures.next().await {
@@ -441,9 +443,14 @@ impl CrawlPlanner {
}
let nodes = chunk.collect::<Vec<_>>();
- let host_graph = self.host_graph.clone();
+ let webgraph = self.webgraph.clone();
- futures.push_back(async move { host_graph.batch_get_node(&nodes).await.unwrap() });
+ futures.push_back(async move {
+ webgraph
+ .batch_search(nodes.into_iter().map(Id2NodeQuery::Host).collect())
+ .await
+ .unwrap()
+ });
}
while let Some(nodes) = futures.next().await {
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 512bdedc..45472fa9 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -16,8 +16,6 @@
use std::net::SocketAddr;
-use crate::config::WebgraphGranularity;
-
#[derive(
serde::Serialize,
serde::Deserialize,
@@ -117,7 +115,6 @@ pub enum Service {
Webgraph {
host: SocketAddr,
shard: ShardId,
- granularity: WebgraphGranularity,
},
Dht {
host: SocketAddr,
@@ -148,12 +145,8 @@ impl std::fmt::Display for Service {
write!(f, "LiveIndex {} {} {}", host, shard, state)
}
Self::Api { host } => write!(f, "Api {}", host),
- Self::Webgraph {
- host,
- shard,
- granularity,
- } => {
- write!(f, "Webgraph {} {} {}", host, shard, granularity)
+ Self::Webgraph { host, shard } => {
+ write!(f, "Webgraph {} {}", host, shard)
}
Self::Dht { host, shard } => write!(f, "Dht {} {}", host, shard),
Self::HarmonicWorker { host, shard } => write!(f, "HarmonicWorker {} {}", host, shard),
diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rs
index 7c53b6dd..586b275e 100644
--- a/crates/core/src/distributed/sonic/replication.rs
+++ b/crates/core/src/distributed/sonic/replication.rs
@@ -347,6 +347,10 @@ where
pub fn id(&self) -> &Id {
&self.id
}
+
+ pub fn replicas(&self) -> &ReplicatedClient<S> {
+ &self.replicas
+ }
}
pub struct ShardedClient<S: sonic::service::Service, Id: ShardIdentifier> {
diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rs
index 489a0276..bb247277 100644
--- a/crates/core/src/distributed/sonic/service.rs
+++ b/crates/core/src/distributed/sonic/service.rs
@@ -210,11 +210,11 @@ macro_rules! sonic_service {
use $crate::distributed::sonic;
- #[derive(Debug, Clone, ::bincode::Encode, ::bincode::Decode)]
+ #[derive(Clone, ::bincode::Encode, ::bincode::Decode)]
pub enum Request {
$($req(Box<$req>),)*
}
- #[derive(::bincode::Encode, ::bincode::Decode, Debug)]
+ #[derive(::bincode::Encode, ::bincode::Decode)]
pub enum Response {
$($req(Box<<$req as sonic::service::Message<$service>>::Response>),)*
}
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index 60b24585..29682882 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -34,7 +34,6 @@ use tantivy::{
use crate::{
image_store::{EntityImageStore, Image, ImageStore},
- inverted_index::merge_tantivy_segments,
tokenizer::fields::DefaultTokenizer,
Result,
};
@@ -418,7 +417,7 @@ impl EntityIndex {
let base_path = Path::new(&self.path);
let segments: Vec<_> = self.tv_index.load_metas()?.segments.into_iter().collect();
- merge_tantivy_segments(
+ tantivy::merge_segments(
self.writer.as_mut().expect("writer has not been prepared"),
segments,
base_path,
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index 911c2fac..4cb420e2 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -20,8 +20,11 @@ use std::{cmp, collections::BTreeMap};
use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};
use crate::{
- ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
- kahan_sum::KahanSum, webgraph, webpage::html::links::RelFlags,
+ ampc::dht::upsert,
+ entrypoint::ampc::approximated_harmonic_centrality::DhtTable,
+ kahan_sum::KahanSum,
+ webgraph::{self, EdgeLimit},
+ webpage::html::links::RelFlags,
};
use rayon::prelude::*;
@@ -70,20 +73,24 @@ impl Workers {
for outgoing in self
.worker
.graph()
- .raw_outgoing_edges(&node, webgraph::EdgeLimit::Limit(MAX_OUTGOING_EDGES))
+ .search(
+ &webgraph::query::ForwardlinksQuery::new(node)
+ .with_limit(EdgeLimit::Limit(MAX_OUTGOING_EDGES)),
+ )
+ .unwrap_or_default()
.into_iter()
- .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
+ .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
.map(|e| e.to)
{
let d = dist + 1;
- let current_dist = distances.entry(outgoing.node()).or_insert(u8::MAX);
+ let current_dist = distances.entry(outgoing).or_insert(u8::MAX);
if d < *current_dist {
*current_dist = d;
if d < max_dist {
- queue.push(cmp::Reverse((d, outgoing.node())));
+ queue.push(cmp::Reverse((d, outgoing)));
}
}
}
@@ -111,8 +118,8 @@ impl Mapper for ApproxCentralityMapper {
pool.install(|| match self {
ApproxCentralityMapper::InitCentrality => {
- let num_nodes = worker.graph().estimate_num_nodes();
- let nodes = worker.graph().nodes().collect::<Vec<_>>();
+ let num_nodes = worker.graph().page_nodes().len();
+ let nodes: Vec<_> = worker.graph().page_nodes().into_iter().collect();
nodes
.par_chunks(BATCH_SIZE)
.progress_count(num_nodes as u64 / BATCH_SIZE as u64)
@@ -133,7 +140,7 @@ impl Mapper for ApproxCentralityMapper {
let sampled = worker
.graph()
- .random_nodes_with_outgoing(num_samples as usize);
+ .random_page_nodes_with_outgoing(num_samples as usize);
let pb = indicatif::ProgressBar::new(sampled.len() as u64);
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
index ee67f684..d6a37a35 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/worker.rs
@@ -58,7 +58,7 @@ impl Message<ApproxCentralityWorker> for NumNodes {
type Response = u64;
fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {
- worker.graph.estimate_num_nodes() as u64
+ worker.graph.page_nodes().len() as u64
}
}
@@ -71,7 +71,14 @@ impl Message<ApproxCentralityWorker> for BatchId2Node {
fn handle(self, worker: &ApproxCentralityWorker) -> Self::Response {
self.0
.iter()
- .filter_map(|id| worker.graph.id2node(id).map(|node| (*id, node)))
+ .filter_map(|id| {
+ worker
+ .graph
+ .search(&webgraph::query::Id2NodeQuery::Page(*id))
+ .ok()
+ .flatten()
+ .map(|node| (*id, node))
+ })
.collect()
}
}
@@ -123,9 +130,7 @@ impl RemoteWorker for RemoteApproxCentralityWorker {
pub fn run(config: ApproxHarmonicWorkerConfig) -> Result<()> {
let tokio_conf = config.clone();
- let graph = Webgraph::builder(config.graph_path)
- .single_threaded()
- .open();
+ let graph = Webgraph::builder(config.graph_path, config.shard).open()?;
let worker = ApproxCentralityWorker::new(graph, config.shard);
let service = Service::ApproxHarmonicWorker {
host: tokio_conf.host,
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
index a5efc4b4..b39f3934 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
@@ -48,10 +48,10 @@ pub enum CentralityMapper {
impl CentralityMapper {
/// get old values from prev dht using edge.from where edge.from in changed_nodes
fn get_old_counters(
- batch: &[webgraph::Edge<()>],
+ batch: &[webgraph::SmallEdge],
dht: &DhtConn<CentralityTables>,
) -> BTreeMap<webgraph::NodeID, HyperLogLog<64>> {
- let nodes: Vec<_> = batch.iter().map(|edge| edge.from.node()).collect();
+ let nodes: Vec<_> = batch.iter().map(|edge| edge.from).collect();
if nodes.is_empty() {
return BTreeMap::new();
@@ -87,7 +87,7 @@ impl CentralityMapper {
/// upsert old edge.from `old_counters` into edge.to in dht.next,
/// thereby updating their hyperloglog counters
fn update_counters(
- batch: &[webgraph::Edge<()>],
+ batch: &[webgraph::SmallEdge],
dht: &DhtConn<CentralityTables>,
) -> Vec<(webgraph::NodeID, UpsertAction)> {
let old_counters = Self::get_old_counters(batch, dht);
@@ -95,12 +95,9 @@ impl CentralityMapper {
let updates: Vec<_> = batch
.iter()
.map(|edge| {
- let mut counter = old_counters
- .get(&edge.from.node())
- .cloned()
- .unwrap_or_default();
- counter.add(edge.from.node().as_u64());
- (edge.to.node(), counter)
+ let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();
+ counter.add(edge.from.as_u64());
+ (edge.to, counter)
})
.collect();
@@ -129,7 +126,7 @@ impl CentralityMapper {
fn update_dht(
worker: &CentralityWorker,
- batch: &[webgraph::Edge<()>],
+ batch: &[webgraph::SmallEdge],
new_changed_nodes: &Mutex<U64BloomFilter>,
dht: &DhtConn<CentralityTables>,
) {
@@ -226,7 +223,7 @@ impl CentralityMapper {
let mut batch = Vec::with_capacity(batch_size);
let mut changed_nodes = worker.changed_nodes().lock().unwrap();
- for node in worker.graph().nodes() {
+ for node in worker.graph().host_nodes() {
changed_nodes.insert(node.as_u64());
batch.push(node);
if batch.len() >= batch_size {
@@ -271,9 +268,9 @@ impl CentralityMapper {
for edge in worker
.graph()
- .edges()
- .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
- .filter(|e| changed_nodes.contains(e.from.node().as_u64()))
+ .host_edges()
+ .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
+ .filter(|e| changed_nodes.contains(e.from.as_u64()))
{
batch.push(edge);
if batch.len() >= batch_size {
@@ -316,7 +313,8 @@ impl CentralityMapper {
let changed_nodes = worker.changed_nodes().lock().unwrap();
for node in worker
.graph()
- .nodes()
+ .host_nodes()
+ .into_iter()
.filter(|n| changed_nodes.contains(n.as_u64()))
{
batch.push(node);
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
index ed63b38b..008ddedb 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
@@ -81,11 +81,10 @@ impl Job for CentralityJob {
#[cfg(test)]
mod tests {
use tracing_test::traced_test;
+ use webgraph::{Edge, Webgraph};
use crate::{
- executor::Executor,
- free_socket_addr,
- webgraph::{centrality::harmonic::HarmonicCentrality, Compression, WebgraphWriter},
+ free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality,
webpage::html::links::RelFlags,
};
@@ -95,47 +94,41 @@ mod tests {
#[traced_test]
fn test_simple_graph() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut combined = WebgraphWriter::new(
- temp_dir.as_ref().join("combined"),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- let mut a = WebgraphWriter::new(
- temp_dir.as_ref().join("a"),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- let mut b = WebgraphWriter::new(
- temp_dir.as_ref().join("b"),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
+ let mut combined = Webgraph::builder(temp_dir.as_ref().join("combined"), 0u64.into())
+ .open()
+ .unwrap();
+ let mut a = Webgraph::builder(temp_dir.as_ref().join("a"), 0u64.into())
+ .open()
+ .unwrap();
+ let mut b = Webgraph::builder(temp_dir.as_ref().join("b"), 0u64.into())
+ .open()
+ .unwrap();
let edges = crate::webgraph::tests::test_edges();
for (i, (from, to, label)) in edges.into_iter().enumerate() {
- combined.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
+ let e = Edge {
+ from: from.clone(),
+ to: to.clone(),
+ label: label.clone(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
+ };
+ combined.insert(e.clone()).unwrap();
if i % 2 == 0 {
- a.insert(from, to, label, RelFlags::default());
+ a.insert(e).unwrap();
} else {
- b.insert(from, to, label, RelFlags::default());
+ b.insert(e).unwrap();
}
}
- combined.commit();
- a.commit();
- b.commit();
-
- let combined = combined.finalize();
- let a = a.finalize();
- let b = b.finalize();
+ combined.commit().unwrap();
+ a.commit().unwrap();
+ b.commit().unwrap();
let expected = HarmonicCentrality::calculate(&combined);
- let num_nodes = combined.nodes().count();
+ let num_nodes = combined.host_nodes().len();
let worker = CentralityWorker::new(1.into(), a);
let worker_addr = free_socket_addr();
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
index 1662c91e..6018821b 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
@@ -46,7 +46,7 @@ pub struct CentralityWorker {
impl CentralityWorker {
pub fn new(shard: ShardId, graph: Webgraph) -> Self {
- let num_nodes = graph.estimate_num_nodes() as u64;
+ let num_nodes = graph.host_nodes().len() as u64;
let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);
changed_nodes.fill();
@@ -107,7 +107,7 @@ impl Message<CentralityWorker> for NumNodes {
type Response = u64;
fn handle(self, worker: &CentralityWorker) -> Self::Response {
- worker.graph.estimate_num_nodes() as u64
+ worker.graph.host_nodes().len() as u64
}
}
@@ -120,7 +120,14 @@ impl Message<CentralityWorker> for BatchId2Node {
fn handle(self, worker: &CentralityWorker) -> Self::Response {
self.0
.iter()
- .filter_map(|id| worker.graph.id2node(id).map(|node| (*id, node)))
+ .filter_map(|id| {
+ worker
+ .graph
+ .search(&webgraph::query::Id2NodeQuery::Host(*id))
+ .ok()
+ .flatten()
+ .map(|node| (*id, node))
+ })
.collect()
}
}
@@ -168,9 +175,7 @@ impl RemoteWorker for RemoteCentralityWorker {
pub fn run(config: HarmonicWorkerConfig) -> Result<()> {
let tokio_conf = config.clone();
- let graph = Webgraph::builder(config.graph_path)
- .single_threaded()
- .open();
+ let graph = Webgraph::builder(config.graph_path, config.shard).open()?;
let worker = CentralityWorker::new(config.shard, graph);
let service = Service::HarmonicWorker {
host: tokio_conf.host,
diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rs
index a43ec60f..1f0bad9e 100644
--- a/crates/core/src/entrypoint/centrality.rs
+++ b/crates/core/src/entrypoint/centrality.rs
@@ -28,7 +28,8 @@ use crate::{
approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality, store_csv,
store_harmonic, TopNodes,
},
- remote::{Page, RemoteWebgraph},
+ query::{BacklinksQuery, Id2NodeQuery},
+ remote::RemoteWebgraph,
EdgeLimit, NodeID, WebgraphBuilder,
},
SortableFloat,
@@ -42,7 +43,9 @@ impl Centrality {
"Building harmonic centrality for {}",
webgraph_path.as_ref().to_str().unwrap()
);
- let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
+ let graph = WebgraphBuilder::new(webgraph_path, 0u64.into())
+ .open()
+ .expect("webgraph should open");
let harmonic_centrality = HarmonicCentrality::calculate(&graph);
let store = store_harmonic(
harmonic_centrality.iter().map(|(n, c)| (*n, c)),
@@ -52,7 +55,15 @@ impl Centrality {
let top_harmonics =
crate::webgraph::centrality::top_nodes(&store, TopNodes::Top(1_000_000))
.into_iter()
- .map(|(n, c)| (graph.id2node(&n).unwrap(), c))
+ .map(|(n, c)| {
+ (
+ graph
+ .search(&crate::webgraph::query::Id2NodeQuery::Host(n))
+ .unwrap()
+ .unwrap(),
+ c,
+ )
+ })
.collect();
store_csv(top_harmonics, base_output.as_ref().join("harmonic.csv"));
@@ -65,7 +76,7 @@ impl Centrality {
webgraph_path.as_ref().to_str().unwrap()
);
- let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();
+ let graph = WebgraphBuilder::new(webgraph_path, 0u64.into()).open()?;
let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("harmonic"));
let mut approx_rank: speedy_kv::Db<crate::webgraph::NodeID, u64> =
@@ -92,7 +103,13 @@ impl Centrality {
}
if top_nodes.len() < 1_000_000 {
- top_nodes.push((graph.id2node(&node).unwrap(), centrality));
+ top_nodes.push((
+ graph
+ .search(&crate::webgraph::query::Id2NodeQuery::Page(node))
+ .unwrap()
+ .unwrap(),
+ centrality,
+ ));
}
}
@@ -114,14 +131,15 @@ impl Centrality {
)
.await?,
);
- let graph: RemoteWebgraph<Page> = RemoteWebgraph::new(cluster).await;
+ let graph: RemoteWebgraph = RemoteWebgraph::new(cluster).await;
+ graph.await_ready().await;
let mut harmonic: speedy_kv::Db<NodeID, f64> =
speedy_kv::Db::open_or_create(config.output_path.join("harmonic"))?;
let original_centrality: speedy_kv::Db<NodeID, f64> =
speedy_kv::Db::open_or_create(&config.original_centrality_path)?;
- let mut node_ids = graph.stream_node_ids().await;
+ let mut node_ids = graph.stream_page_node_ids().await;
while let Some(node_id) = node_ids.next().await {
match original_centrality.get(&node_id)? {
@@ -130,10 +148,10 @@ impl Centrality {
}
None => {
if let Some(seed) = graph
- .raw_ingoing_edges(node_id, EdgeLimit::Limit(1))
+ .search(BacklinksQuery::new(node_id).with_limit(EdgeLimit::Limit(1)))
.await?
.pop()
- .map(|edge| edge.from.node())
+ .map(|edge| edge.from)
{
if let Some(seed_centrality) = original_centrality.get(&seed)? {
harmonic.insert(node_id, seed_centrality * config.discount_factor)?;
@@ -166,7 +184,9 @@ impl Centrality {
{
let (ids, centrality): (Vec<_>, Vec<_>) = chunk.into_iter().unzip();
- let nodes = graph.batch_get_node(&ids).await?;
+ let nodes = graph
+ .batch_search(ids.into_iter().map(Id2NodeQuery::Page).collect())
+ .await?;
for (node, centrality) in nodes.into_iter().zip(centrality) {
if let Some(node) = node {
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index e116ecf1..1df056a5 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -98,15 +98,10 @@ fn build_spellchecker() -> Result<()> {
fn create_webgraph() -> Result<()> {
debug!("Creating webgraph");
- let out_path_host = Path::new(DATA_PATH).join("webgraph_host");
- let out_path_page = Path::new(DATA_PATH).join("webgraph_page");
+ let out_path = Path::new(DATA_PATH).join("webgraph");
- if out_path_host.exists() {
- std::fs::remove_dir_all(&out_path_host)?;
- }
-
- if out_path_page.exists() {
- std::fs::remove_dir_all(&out_path_page)?;
+ if out_path.exists() {
+ std::fs::remove_dir_all(&out_path)?;
}
let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
@@ -120,33 +115,30 @@ fn create_webgraph() -> Result<()> {
};
let mut worker = webgraph::WebgraphWorker {
- host_graph: Some(webgraph::open_host_graph_writer(&out_path_host, None)),
- page_graph: Some(webgraph::open_page_graph_writer(&out_path_page, None)),
+ graph: crate::webgraph::Webgraph::open(&out_path, 0u64.into()).unwrap(),
+ host_centrality_store: None,
canonical_index: None,
};
worker.process_job(&job);
-
- worker.host_graph.unwrap().finalize();
- worker.page_graph.unwrap().finalize();
+ worker.graph.optimize_read().unwrap();
Ok(())
}
fn calculate_centrality() {
debug!("Calculating centrality");
- let webgraph_path = Path::new(DATA_PATH).join("webgraph_host");
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph");
let out_path = Path::new(DATA_PATH).join("centrality");
if !out_path.exists() {
Centrality::build_harmonic(&webgraph_path, &out_path);
}
- let webgraph_page = Path::new(DATA_PATH).join("webgraph_page");
let out_path_page = Path::new(DATA_PATH).join("centrality_page");
if !out_path_page.exists() {
- Centrality::build_approx_harmonic(webgraph_page, out_path_page).unwrap();
+ Centrality::build_approx_harmonic(webgraph_path, out_path_page).unwrap();
}
}
@@ -175,7 +167,7 @@ fn create_inverted_index() -> Result<()> {
},
};
- let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");
+ let webgraph_path = Path::new(DATA_PATH).join("webgraph");
let centrality_path = Path::new(DATA_PATH).join("centrality");
let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");
let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index cec45517..e03c58a9 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -27,13 +27,13 @@ use crate::backlink_grouper::BacklinkGrouper;
use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig};
use crate::distributed::cluster::Cluster;
use crate::models::dual_encoder::DualEncoder as DualEncoderModel;
-use crate::webgraph::remote::{Page, RemoteWebgraph};
+use crate::webgraph::remote::RemoteWebgraph;
use crate::Result;
use crate::index::Index;
use crate::rake::RakeModel;
use crate::ranking::SignalComputer;
-use crate::webgraph::{self, Edge, EdgeLimit, Node, NodeID};
+use crate::webgraph::{self, EdgeLimit, Node, NodeID, SmallEdgeWithLabel};
use crate::webpage::{safety_classifier, Html, Webpage};
const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
@@ -83,7 +83,7 @@ struct DualEncoder {
}
pub(super) enum Webgraph {
- Remote(RemoteWebgraph<Page>),
+ Remote(RemoteWebgraph),
Local(webgraph::Webgraph),
}
@@ -92,17 +92,30 @@ impl Webgraph {
&self,
ids: Vec<NodeID>,
limit: EdgeLimit,
- ) -> Vec<Vec<Edge<String>>> {
+ ) -> Vec<Vec<SmallEdgeWithLabel>> {
let edges = match self {
Self::Remote(webgraph) => webgraph
- .batch_raw_ingoing_edges_with_labels(&ids, limit)
+ .batch_search(
+ ids.into_iter()
+ .map(|id| {
+ webgraph::query::BacklinksWithLabelsQuery::new(id).with_limit(limit)
+ })
+ .collect(),
+ )
.await
.unwrap_or_default(),
Self::Local(webgraph) => {
let mut res = Vec::new();
for id in ids {
- res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));
+ res.push(
+ webgraph
+ .search(
+ &webgraph::query::BacklinksWithLabelsQuery::new(id)
+ .with_limit(limit),
+ )
+ .unwrap_or_default(),
+ );
}
res
@@ -148,9 +161,9 @@ impl Webgraph {
async fn new(config: &IndexerGraphConfig) -> Self {
match config {
IndexerGraphConfig::Local { path } => Self::Local(
- webgraph::WebgraphBuilder::new(path)
- .single_threaded()
- .open(),
+ webgraph::WebgraphBuilder::new(path, 0u64.into())
+ .open()
+ .expect("webgraph should open"),
),
IndexerGraphConfig::Remote { gossip } => {
let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);
@@ -421,7 +434,13 @@ impl IndexingWorker {
BacklinkGrouper::new(self.host_centrality_rank_store.len() as u64);
for backlink in backlinks {
- grouper.add(backlink);
+ let from = self
+ .host_centrality_rank_store
+ .get(&backlink.from)
+ .unwrap()
+ .unwrap_or(u64::MAX);
+
+ grouper.add(backlink, from);
}
page.set_grouped_backlinks(grouper.groups())
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 190f9aba..1cf99a6b 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -17,11 +17,10 @@ use crate::{
canon_index::CanonicalIndex,
config::{self, WarcSource, WebgraphConstructConfig},
entrypoint::download_all_warc_files,
- webgraph::{self, Node, NodeID, WebgraphWriter},
- webpage::{url_ext::UrlExt, Html},
+ webgraph::{self, Edge, Node, NodeID},
+ webpage::Html,
Result,
};
-use anyhow::bail;
use itertools::Itertools;
use url::Url;
@@ -62,30 +61,6 @@ pub struct Job {
pub warc_paths: Vec<String>,
}
-pub fn open_host_graph_writer<P: AsRef<Path>>(
- path: P,
- host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
-) -> webgraph::WebgraphWriter {
- WebgraphWriter::new(
- path,
- crate::executor::Executor::single_thread(),
- webgraph::Compression::Lz4,
- host_centrality_store,
- )
-}
-
-pub fn open_page_graph_writer<P: AsRef<Path>>(
- path: P,
- host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
-) -> webgraph::WebgraphWriter {
- WebgraphWriter::new(
- path,
- crate::executor::Executor::single_thread(),
- webgraph::Compression::Lz4,
- host_centrality_store,
- )
-}
-
fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {
if let Some(url) = index.get(&url).unwrap() {
url
@@ -95,8 +70,8 @@ fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {
}
pub struct WebgraphWorker {
- pub host_graph: Option<webgraph::WebgraphWriter>,
- pub page_graph: Option<webgraph::WebgraphWriter>,
+ pub host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, f64>>>,
+ pub graph: webgraph::Webgraph,
pub canonical_index: Option<Arc<CanonicalIndex>>,
}
@@ -122,57 +97,53 @@ impl WebgraphWorker {
}
};
+ let mut source = webpage.url().clone();
+ if let Some(index) = &self.canonical_index {
+ source = canonical_or_self(index, source);
+ }
+
+ let source = Node::from(source);
+ let source_centrality = self
+ .host_centrality_store
+ .as_ref()
+ .and_then(|store| store.get(&source.clone().into_host().id()).unwrap())
+ .unwrap_or(0.0);
+
for mut link in webpage
.anchor_links()
.into_iter()
.filter(|link| matches!(link.destination.scheme(), "http" | "https"))
{
- let mut source = link.source.clone();
let mut destination = link.destination.clone();
if let Some(index) = &self.canonical_index {
- source = canonical_or_self(index, source);
destination = canonical_or_self(index, destination);
}
link.text = link.text.chars().take(128).collect();
- let mut source = Node::from(source);
+ let destination = Node::from(destination);
- let mut destination = Node::from(destination);
+ let destination_centrality = self
+ .host_centrality_store
+ .as_ref()
+ .and_then(|store| store.get(&destination.clone().into_host().id()).unwrap())
+ .unwrap_or(0.0);
trace!("inserting link {:?}", link);
- if let Some(graph) = self.page_graph.as_mut() {
- graph.insert(
- source.clone(),
- destination.clone(),
- link.text.clone(),
- link.rel,
- )
- }
-
- let dest_domain = link.destination.root_domain();
- let source_domain = link.source.root_domain();
- if dest_domain.is_some()
- && source_domain.is_some()
- && dest_domain != source_domain
- {
- source = source.into_host();
- destination = destination.into_host();
-
- if let Some(graph) = self.host_graph.as_mut() {
- graph.insert(source, destination, link.text, link.rel)
- }
- }
+ self.graph
+ .insert(Edge {
+ from: source.clone(),
+ to: destination,
+ rel_flags: link.rel,
+ label: link.text,
+ sort_score: source_centrality + destination_centrality,
+ })
+ .unwrap();
}
}
- if let Some(graph) = self.host_graph.as_mut() {
- graph.commit()
- }
- if let Some(graph) = self.page_graph.as_mut() {
- graph.commit()
- }
+ self.graph.commit().unwrap();
}
info!("{} done", name);
@@ -183,10 +154,6 @@ pub struct Webgraph {}
impl Webgraph {
pub fn run(config: &WebgraphConstructConfig) -> Result<()> {
- if config.page_graph_base_path.is_none() && config.host_graph_base_path.is_none() {
- bail!("either page_graph_base_path or host_graph_base_path must be set");
- }
-
let warc_paths = config.warc_source.paths()?;
let job_config = JobConfig::from(config.warc_source.clone());
@@ -209,18 +176,14 @@ impl Webgraph {
None
};
- let host_centrality_rank_store = if let Some(path) = &config.host_centrality_rank_store_path
- {
- Some(Arc::new(speedy_kv::Db::open_or_create(path)?))
- } else {
- None
- };
+ let host_centrality_rank_store = Arc::new(speedy_kv::Db::open_or_create(
+ &config.host_centrality_rank_store_path,
+ )?);
let num_workers = usize::from(std::thread::available_parallelism()?);
let mut handlers = Vec::new();
- let host_path = &config.host_graph_base_path;
- let page_path = &config.page_graph_base_path;
+ let graph_path = &config.graph_base_path;
const MAX_FINALIZE_CONCURRENT: usize = 8;
let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);
@@ -230,23 +193,12 @@ impl Webgraph {
}
for i in 0..num_workers {
- let host_path = host_path.clone();
- let host_path = host_path
- .as_ref()
- .map(|p| Path::new(p).join(format!("worker_{i}")));
-
- let page_path = page_path.clone();
- let page_path = page_path
- .as_ref()
- .map(|p| Path::new(p).join(format!("worker_{i}")));
+ let graph_path = graph_path.clone();
+ let graph_path = Path::new(&graph_path).join(format!("worker_{i}"));
let mut worker = WebgraphWorker {
- host_graph: host_path
- .as_ref()
- .map(|p| open_host_graph_writer(p, host_centrality_rank_store.clone())),
- page_graph: page_path
- .as_ref()
- .map(|p| open_page_graph_writer(p, host_centrality_rank_store.clone())),
+ graph: webgraph::Webgraph::open(graph_path, config.shard)?,
+ host_centrality_store: Some(host_centrality_rank_store.clone()),
canonical_index: canonical_index.clone(),
};
@@ -259,11 +211,11 @@ impl Webgraph {
r.recv().unwrap();
- let host = worker.host_graph.map(|graph| graph.finalize());
- let page = worker.page_graph.map(|graph| graph.finalize());
+ worker.graph.commit().unwrap();
+ worker.graph.optimize_read().unwrap();
s.send(()).unwrap();
- (host, page)
+ worker.graph
}));
}
@@ -272,39 +224,16 @@ impl Webgraph {
graphs.push(handler.join().unwrap());
}
- let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
-
- for (other_host, other_page) in graphs {
- if let (Some(graph), Some(other)) = (host_graph.as_mut(), other_host) {
- graph.merge(other)?;
- }
+ let mut graph = graphs.pop().unwrap();
- if let (Some(graph), Some(other)) = (page_graph.as_mut(), other_page) {
- graph.merge(other)?;
- }
+ for other in graphs {
+ graph.merge(other)?;
}
if config.merge_all_segments {
- if let Some(host) = host_graph.as_mut() {
- host.optimize_read(); // save space in id2node db
- host.merge_all_segments(Default::default())?;
- }
-
- if let Some(page) = page_graph.as_mut() {
- page.optimize_read(); // save space in id2node db
- page.merge_all_segments(Default::default())?;
- }
- }
-
- if let Some(host) = host_graph.as_mut() {
- host.optimize_read();
- crate::mv(host.path(), config.host_graph_base_path.as_ref().unwrap())?;
- }
-
- if let Some(page) = page_graph.as_mut() {
- page.optimize_read();
- crate::mv(page.path(), config.page_graph_base_path.as_ref().unwrap())?;
+ graph.optimize_read().unwrap();
}
+ crate::mv(graph.path(), &config.graph_base_path)?;
Ok(())
}
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index dbe60278..74671b5c 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -24,12 +24,20 @@ use crate::config;
use crate::distributed::cluster::Cluster;
use crate::distributed::member::Member;
use crate::distributed::member::Service;
+use crate::distributed::sonic;
use crate::distributed::sonic::service::sonic_service;
use crate::distributed::sonic::service::Message;
-use crate::webgraph::Edge;
-use crate::webgraph::EdgeLimit;
-use crate::webgraph::FullEdge;
-use crate::webgraph::Node;
+use crate::webgraph;
+use crate::webgraph::query::BacklinksQuery;
+use crate::webgraph::query::BacklinksWithLabelsQuery;
+use crate::webgraph::query::ForwardlinksQuery;
+use crate::webgraph::query::FullBacklinksQuery;
+use crate::webgraph::query::FullForwardlinksQuery;
+use crate::webgraph::query::FullHostBacklinksQuery;
+use crate::webgraph::query::FullHostForwardlinksQuery;
+use crate::webgraph::query::HostBacklinksQuery;
+use crate::webgraph::query::HostForwardlinksQuery;
+use crate::webgraph::query::Id2NodeQuery;
use crate::webgraph::NodeID;
use crate::webgraph::Webgraph;
use crate::webgraph::WebgraphBuilder;
@@ -47,170 +55,117 @@ pub struct WebGraphService {
graph: Arc<Webgraph>,
}
-sonic_service!(
- WebGraphService,
- [
- GetNode,
- IngoingEdges,
- OutgoingEdges,
- RawIngoingEdges,
- RawOutgoingEdges,
- RawIngoingEdgesWithLabels,
- RawOutgoingEdgesWithLabels,
- GetNodeIDs
- ]
-);
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetNode {
- pub node: NodeID,
-}
-
-impl Message<WebGraphService> for GetNode {
- type Response = Option<Node>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.id2node(&self.node)
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct IngoingEdges {
- pub node: Node,
- pub limit: EdgeLimit,
-}
-
-impl Message<WebGraphService> for IngoingEdges {
- type Response = Vec<FullEdge>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.ingoing_edges(self.node, self.limit)
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct OutgoingEdges {
- pub node: Node,
- pub limit: EdgeLimit,
-}
-
-impl Message<WebGraphService> for OutgoingEdges {
- type Response = Vec<FullEdge>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.outgoing_edges(self.node, self.limit)
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct RawIngoingEdges {
- pub node: NodeID,
- pub limit: EdgeLimit,
-}
-
-impl Message<WebGraphService> for RawIngoingEdges {
- type Response = Vec<Edge<()>>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_ingoing_edges(&self.node, self.limit)
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct RawOutgoingEdges {
- pub node: NodeID,
- pub limit: EdgeLimit,
-}
-
-impl Message<WebGraphService> for RawOutgoingEdges {
- type Response = Vec<Edge<()>>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.raw_outgoing_edges(&self.node, self.limit)
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct RawIngoingEdgesWithLabels {
- pub node: NodeID,
- pub limit: EdgeLimit,
+pub trait RetrieveReq: bincode::Encode + bincode::Decode + Clone {
+ type Query: webgraph::Query + bincode::Encode + bincode::Decode;
+ fn new(
+ query: Self::Query,
+ fruit: <<Self::Query as webgraph::Query>::Collector as webgraph::Collector>::Fruit,
+ ) -> Self;
}
-impl Message<WebGraphService> for RawIngoingEdgesWithLabels {
- type Response = Vec<Edge<String>>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server
- .graph
- .raw_ingoing_edges_with_labels(&self.node, self.limit)
- }
+pub trait Query
+where
+ Self: webgraph::Query
+ + bincode::Encode
+ + bincode::Decode
+ + sonic::service::Wrapper<WebGraphService>,
+{
+ type RetrieveReq: RetrieveReq<Query = Self>;
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct RawOutgoingEdgesWithLabels {
- pub node: NodeID,
- pub limit: EdgeLimit,
+#[derive(bincode::Encode, bincode::Decode, Clone)]
+pub struct EncodedError {
+ pub msg: String,
}
-impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {
- type Response = Vec<Edge<String>>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server
- .graph
- .raw_outgoing_edges_with_labels(&self.node, self.limit)
+impl std::fmt::Display for EncodedError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}", self.msg)
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct InDegreeUpperBound {
- pub node: NodeID,
-}
-
-impl Message<WebGraphService> for InDegreeUpperBound {
- type Response = u64;
+macro_rules! impl_search {
+ ([$($q:ident),*$(,)?]) => {
+ $(
+ impl Message<WebGraphService> for $q {
+ type Response = Result<<<$q as webgraph::Query>::Collector as webgraph::Collector>::Fruit, EncodedError>;
+
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ server.graph.search_initial(&self).map_err(|e| EncodedError { msg: e.to_string() })
+ }
+ }
+
+ paste::item! {
+ #[derive(bincode::Encode, bincode::Decode, Clone)]
+ pub struct [<$q Retrieve>] {
+ pub query: $q,
+ #[bincode(with_serde)]
+ pub fruit: <<$q as webgraph::Query>::Collector as webgraph::Collector>::Fruit,
+ }
+
+ impl Message<WebGraphService> for [<$q Retrieve>] {
+ type Response = Result<<$q as webgraph::Query>::IntermediateOutput, EncodedError>;
+ async fn handle(self, server: &WebGraphService) -> Self::Response {
+ server.graph.retrieve(&self.query, self.fruit).map_err(|e| EncodedError { msg: e.to_string() })
+ }
+ }
+
+ impl Query for $q {
+ type RetrieveReq = [<$q Retrieve>];
+ }
+
+ impl RetrieveReq for [<$q Retrieve>] {
+ type Query = $q;
+ fn new(query: Self::Query, fruit: <<Self::Query as webgraph::Query>::Collector as webgraph::Collector>::Fruit) -> Self {
+ Self { query, fruit }
+ }
+ }
+ }
+ )*
+
+ paste::item! {
+ sonic_service!(WebGraphService, [GetPageNodeIDs, $(
+ $q,
+ [<$q Retrieve>],
+ )*]);
+ }
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.in_degree_upper_bound(&self.node)
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct OutDegreeUpperBound {
- pub node: NodeID,
-}
-
-impl Message<WebGraphService> for OutDegreeUpperBound {
- type Response = u64;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.out_degree_upper_bound(&self.node)
- }
-}
+impl_search!([
+ BacklinksQuery,
+ HostBacklinksQuery,
+ FullBacklinksQuery,
+ FullHostBacklinksQuery,
+ BacklinksWithLabelsQuery,
+ Id2NodeQuery,
+ ForwardlinksQuery,
+ HostForwardlinksQuery,
+ FullForwardlinksQuery,
+ FullHostForwardlinksQuery,
+]);
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetNodeIDs {
+pub struct GetPageNodeIDs {
pub offset: u64,
pub limit: u64,
}
-impl Message<WebGraphService> for GetNodeIDs {
+impl Message<WebGraphService> for GetPageNodeIDs {
type Response = Vec<NodeID>;
async fn handle(self, server: &WebGraphService) -> Self::Response {
server
.graph
- .iter_nodes_with_offset(self.offset)
- .take(self.limit as usize)
- .map(|(id, _)| id)
- .collect()
+ .page_node_ids_with_offset(self.offset, self.limit)
}
}
pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
- let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());
+ let graph = Arc::new(WebgraphBuilder::new(config.graph_path, config.shard).open()?);
let server = WebGraphService { graph }.bind(addr).await.unwrap();
@@ -219,7 +174,6 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
Cluster::join(
Member::new(Service::Webgraph {
host: addr,
- granularity: config.granularity,
shard: config.shard,
}),
config.gossip_addr,
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index 03b8f906..a902883a 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -19,8 +19,6 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
-use tantivy::tokenizer::TokenizerManager;
-
use crate::collector::MainCollector;
use crate::inverted_index::{self, InvertedIndex};
use crate::query::Query;
@@ -65,10 +63,6 @@ impl Index {
self.inverted_index.set_auto_merge_policy();
}
- pub fn tokenizers(&self) -> &TokenizerManager {
- self.inverted_index.tokenizers()
- }
-
#[cfg(test)]
pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
let dir = crate::gen_temp_dir()?;
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index c077df92..9c3c9f0f 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -18,8 +18,6 @@ use tantivy::index::SegmentId;
use tantivy::indexer::{MergeOperation, SegmentEntry};
use tantivy::merge_policy::NoMergePolicy;
-use tantivy::{IndexWriter, SegmentMeta};
-
use crate::numericalfield_reader::NumericalFieldReader;
use crate::webpage::Webpage;
@@ -30,63 +28,6 @@ use std::path::Path;
use super::InvertedIndex;
-struct SegmentMergeCandidate {
- num_docs: u32,
- segments: Vec<SegmentMeta>,
-}
-
-pub fn merge_tantivy_segments<P: AsRef<Path>>(
- writer: &mut IndexWriter,
- mut segments: Vec<SegmentMeta>,
- base_path: P,
- max_num_segments: u64,
-) -> Result<()> {
- assert!(max_num_segments > 0);
-
- if segments.len() <= max_num_segments as usize {
- return Ok(());
- }
-
- let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)
-
- let mut merge_segments = Vec::new();
-
- for _ in 0..num_segments {
- merge_segments.push(SegmentMergeCandidate {
- num_docs: 0,
- segments: Vec::new(),
- });
- }
-
- segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
-
- for segment in segments {
- let best_candidate = merge_segments
- .iter_mut()
- .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
- .unwrap();
-
- best_candidate.num_docs += segment.num_docs();
- best_candidate.segments.push(segment);
- }
-
- for merge in merge_segments
- .into_iter()
- .filter(|merge| !merge.segments.is_empty())
- {
- let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();
- writer.merge(&segment_ids[..]).wait()?;
-
- for segment in merge.segments {
- for file in segment.list_files() {
- std::fs::remove_file(base_path.as_ref().join(file)).ok();
- }
- }
- }
-
- Ok(())
-}
-
impl InvertedIndex {
pub fn prepare_writer(&mut self) -> Result<()> {
if self.writer.is_some() {
@@ -144,8 +85,10 @@ impl InvertedIndex {
.into_iter()
.collect();
- merge_tantivy_segments(
- self.writer.as_mut().expect("writer has not been prepared"),
+ tantivy::merge_segments(
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared"),
segments,
base_path,
max_num_segments,
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index f455f3af..7347bd13 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -32,7 +32,6 @@ mod key_phrase;
mod retrieved_webpage;
mod search;
-pub use indexing::merge_tantivy_segments;
pub use key_phrase::KeyPhrase;
pub use retrieved_webpage::RetrievedWebpage;
@@ -197,10 +196,6 @@ impl InvertedIndex {
self.snippet_config = config;
}
- pub fn tokenizers(&self) -> &TokenizerManager {
- self.tantivy_index.tokenizers()
- }
-
pub fn schema(&self) -> Arc<Schema> {
Arc::clone(&self.schema)
}
@@ -251,7 +246,7 @@ mod tests {
ranking::{LocalRanker, SignalComputer},
search_ctx::Ctx,
searcher::SearchQuery,
- webgraph::{Edge, NodeDatum},
+ webgraph::{NodeID, SmallEdgeWithLabel},
webpage::{schema_org, Html, Webpage},
OneOrMany,
};
@@ -526,11 +521,11 @@ mod tests {
..Default::default()
};
- webpage.set_backlinks(vec![Edge {
- from: NodeDatum::new(0u64, 0),
- to: NodeDatum::new(1u64, 0),
+ webpage.set_backlinks(vec![SmallEdgeWithLabel {
+ from: NodeID::from(0u64),
+ to: NodeID::from(1u64),
label: "B site is great".to_string(),
- rel: Default::default(),
+ rel_flags: Default::default(),
}]);
index.insert(&webpage).expect("failed to insert webpage");
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 8aea95ba..f0bdc896 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -29,7 +29,6 @@ use crate::collector::{approx_count, MainCollector};
use crate::highlighted::HighlightedFragment;
use crate::numericalfield_reader::NumericalFieldReader;
-use crate::query::shortcircuit::ShortCircuitQuery;
use crate::query::Query;
use crate::ranking::pipeline::LocalRecallRankingWebpage;
use crate::ranking::SignalComputer;
@@ -38,6 +37,7 @@ use crate::search_ctx::Ctx;
use crate::snippet;
use crate::snippet::TextSnippet;
use crate::webgraph::NodeID;
+use tantivy::query::ShortCircuitQuery;
use crate::schema::text_field::TextField;
use crate::webpage::url_ext::UrlExt;
@@ -61,7 +61,7 @@ impl InvertedIndex {
}
let simple_terms = query.simple_terms().to_vec();
- let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());
+ let query: Box<dyn tantivy::query::Query> = Box::new(query.clone());
if let Some(limit) = collector.top_docs().max_docs().cloned() {
if limit.segments == 0 {
@@ -72,7 +72,8 @@ impl InvertedIndex {
}
let docs_per_segment = (limit.total_docs / limit.segments) as u64;
- query = Box::new(ShortCircuitQuery::new(query, docs_per_segment));
+ let query: Box<dyn tantivy::query::Query> =
+ Box::new(ShortCircuitQuery::new(query, docs_per_segment));
let (count, pointers) = ctx.tv_searcher.search(
&query,
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 8bc2edb3..006c9c38 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -352,21 +352,16 @@ fn main() -> Result<()> {
mut paths,
merge_all_segments,
} => {
- let mut webgraph = WebgraphBuilder::new(paths.remove(0))
- .single_threaded()
- .open();
+ let mut webgraph = WebgraphBuilder::new(paths.remove(0), 0u64.into()).open()?;
for other_path in paths {
- let other = WebgraphBuilder::new(&other_path).single_threaded().open();
+ let other = WebgraphBuilder::new(&other_path, 0u64.into()).open()?;
webgraph.merge(other)?;
}
if merge_all_segments {
- webgraph.optimize_read(); // save space in id2node db
- webgraph.merge_all_segments(Default::default())?;
+ webgraph.optimize_read()?;
}
-
- webgraph.optimize_read();
}
WebgraphOptions::Server { config_path } => {
let config: config::WebgraphServerConfig = load_toml_config(config_path);
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index fbcb479a..f413692e 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -35,7 +35,6 @@ pub mod optic;
pub mod parser;
mod pattern_query;
mod plan;
-pub mod shortcircuit;
pub mod union;
use self::{optic::AsMultipleTantivyQuery, parser::SimpleOrPhrase};
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index ab5dbecc..6f0ce316 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -259,7 +259,7 @@ mod tests {
live::LiveSearcher,
LocalSearchClient, LocalSearcher, SearchQuery,
},
- webgraph::{Node, WebgraphWriter},
+ webgraph::{Edge, Node, Webgraph},
webpage::{html::links::RelFlags, Html, Webpage},
};
@@ -605,47 +605,55 @@ mod tests {
let dir = crate::gen_temp_dir().unwrap();
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
- let mut writer = WebgraphWriter::new(
- &dir,
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
-
- writer.insert(
- Node::from("https://www.e.com").into_host(),
- Node::from("https://www.a.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("https://www.a.com").into_host(),
- Node::from("https://www.e.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
-
- writer.insert(
- Node::from("https://www.c.com").into_host(),
- Node::from("https://www.c.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
-
- writer.insert(
- Node::from("https://www.b.com").into_host(),
- Node::from("https://www.e.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("https://www.e.com").into_host(),
- Node::from("https://www.b.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
-
- let graph = writer.finalize();
+ let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.e.com").into_host(),
+ to: Node::from("https://www.a.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.a.com").into_host(),
+ to: Node::from("https://www.e.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.c.com").into_host(),
+ to: Node::from("https://www.c.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.b.com").into_host(),
+ to: Node::from("https://www.e.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.e.com").into_host(),
+ to: Node::from("https://www.b.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
index
.insert(&Webpage {
diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rs
index c4462a75..7bec574c 100644
--- a/crates/core/src/ranking/bitvec_similarity.rs
+++ b/crates/core/src/ranking/bitvec_similarity.rs
@@ -187,8 +187,7 @@ impl BitVec {
#[cfg(test)]
mod tests {
use crate::{
- executor::Executor,
- webgraph::{Compression, Node, WebgraphWriter},
+ webgraph::{Edge, Node, Webgraph},
webpage::html::links::RelFlags,
};
@@ -295,22 +294,33 @@ mod tests {
async fn test_ignores_no_follow() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- &temp_dir,
- Executor::single_thread(),
- Compression::default(),
- None,
- );
+ let mut graph = Webgraph::open(&temp_dir, 0u64.into()).unwrap();
let a = Node::from("A");
let b = Node::from("B");
let c = Node::from("C");
- writer.insert(a.clone(), b.clone(), String::new(), RelFlags::NOFOLLOW);
-
- writer.insert(a.clone(), c.clone(), String::new(), RelFlags::default());
-
- let graph = writer.finalize();
+ graph
+ .insert(Edge {
+ from: a.clone(),
+ to: b.clone(),
+ rel_flags: RelFlags::NOFOLLOW,
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: a.clone(),
+ to: c.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
let bitvecs = BitVec::batch_new_for(&[b.id(), c.id()], &graph).await;
diff --git a/crates/core/src/ranking/bm25f.rs b/crates/core/src/ranking/bm25f.rs
index f1b0cad5..6e7b9fe2 100644
--- a/crates/core/src/ranking/bm25f.rs
+++ b/crates/core/src/ranking/bm25f.rs
@@ -14,11 +14,53 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::collections::HashMap;
+
use itertools::Itertools;
use tantivy::{Score, Searcher, Term};
+use crate::schema::text_field::{AllBody, TextField};
+
use super::bm25::{compute_tf_cache, idf, Bm25Constants};
+#[derive(Default)]
+pub struct WeightCache {
+ cache: HashMap<String, u64>,
+}
+
+impl WeightCache {
+ pub fn new() -> Self {
+ Self {
+ cache: HashMap::new(),
+ }
+ }
+
+ fn compute(term: &str, searcher: &Searcher) -> u64 {
+ // use 'AllBody' field as an approximation of the term doc freq across all fields
+ let field = searcher.schema().get_field(AllBody.name()).unwrap();
+ let term = Term::from_field_text(field, term);
+ searcher.doc_freq(&term).unwrap_or_default()
+ }
+
+ pub fn get(&mut self, term: &str, searcher: &Searcher) -> u64 {
+ *self
+ .cache
+ .entry(term.to_string())
+ .or_insert_with(|| Self::compute(term, searcher))
+ }
+}
+
+pub struct PreparedTerm {
+ term: Term,
+ doc_freq: u64,
+}
+
+impl PreparedTerm {
+ pub fn new(term: Term, doc_freq: u64) -> Self {
+ Self { term, doc_freq }
+ }
+}
+
/// A BM25F weight that uses the same IDF weight for all fields.
/// The idea is that the term 'the' might not appear very frequently e.g. in the title field,
/// but it should still be considered a common term.
@@ -39,17 +81,21 @@ pub struct MultiBm25FWeight {
}
impl MultiBm25FWeight {
- pub fn for_terms(searcher: &Searcher, terms: &[Term], constants: Bm25Constants) -> Self {
+ pub fn for_terms(
+ searcher: &Searcher,
+ terms: &[PreparedTerm],
+ constants: Bm25Constants,
+ ) -> Self {
if terms.is_empty() {
return Self {
weights: Vec::new(),
};
}
- let field = terms[0].field();
+ let field = terms[0].term.field();
for term in terms.iter().skip(1) {
assert_eq!(
- term.field(),
+ term.term.field(),
field,
"All terms must belong to the same field."
);
@@ -69,19 +115,8 @@ impl MultiBm25FWeight {
let mut weights = Vec::new();
for term in terms {
- // use highest freq as an approximation of the term doc freq across all fields
- let term_doc_freq = searcher
- .schema()
- .fields()
- .filter_map(|(field, _)| {
- let term = Term::from_field_text(field, term.value().as_str().unwrap());
- searcher.doc_freq(&term).ok()
- })
- .max()
- .unwrap_or_default();
-
weights.push(Bm25FWeight::for_one_term(
- term_doc_freq,
+ term.doc_freq,
total_num_docs,
average_fieldnorm,
constants,
diff --git a/crates/core/src/ranking/computer/mod.rs b/crates/core/src/ranking/computer/mod.rs
index 00dbeae1..b660be5f 100644
--- a/crates/core/src/ranking/computer/mod.rs
+++ b/crates/core/src/ranking/computer/mod.rs
@@ -290,6 +290,8 @@ impl SignalComputer {
if let Some(query) = &self.query_data {
if !query.simple_terms.is_empty() {
+ let mut bm25f_weight_cache = super::bm25f::WeightCache::new();
+
for signal in CoreSignalEnum::all() {
if let Some((text_field, tv_field)) = signal
.as_textfield()
@@ -308,13 +310,18 @@ impl SignalComputer {
.collect::<String>();
let mut terms = Vec::new();
+ let mut bm25f_terms = Vec::new();
let mut tokenizer = text_field.query_tokenizer(query.lang.as_ref());
let mut stream = tokenizer.token_stream(&simple_query);
let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);
while let Some(token) = it.next() {
let term = tantivy::Term::from_field_text(tv_field, &token.text);
- terms.push(term);
+ terms.push(term.clone());
+ bm25f_terms.push(super::bm25f::PreparedTerm::new(
+ term,
+ bm25f_weight_cache.get(&token.text, tv_searcher),
+ ));
}
if terms.is_empty() {
@@ -341,7 +348,7 @@ impl SignalComputer {
)?;
let bm25f = MultiBm25FWeight::for_terms(
tv_searcher,
- &terms,
+ &bm25f_terms,
text_field.bm25_constants(),
);
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index b374d367..2db05dd0 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -148,7 +148,7 @@ mod tests {
searcher::{
api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,
},
- webgraph::{EdgeLimit, Node, Webgraph, WebgraphWriter},
+ webgraph::{Edge, EdgeLimit, Node, Webgraph},
webpage::{html::links::RelFlags, Html, Webpage},
};
@@ -159,7 +159,7 @@ mod tests {
graph
.raw_ingoing_edges(node, EdgeLimit::Unlimited)
.into_iter()
- .map(|e| e.from.node().as_u64())
+ .map(|e| e.from.as_u64())
.collect(),
)
}
@@ -167,64 +167,92 @@ mod tests {
#[tokio::test]
async fn it_favors_liked_hosts() {
let dir = crate::gen_temp_dir().unwrap();
- let mut wrt = WebgraphWriter::new(
- &dir,
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
-
- wrt.insert(
- Node::from("a.com"),
- Node::from("b.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("c.com"),
- Node::from("d.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("a.com"),
- Node::from("e.com"),
- String::new(),
- RelFlags::default(),
- );
-
- wrt.insert(
- Node::from("z.com"),
- Node::from("a.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("z.com"),
- Node::from("b.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("z.com"),
- Node::from("c.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("z.com"),
- Node::from("d.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("z.com"),
- Node::from("e.com"),
- String::new(),
- RelFlags::default(),
- );
-
- let graph = wrt.finalize();
+ let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("a.com").into_host(),
+ to: Node::from("b.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("c.com").into_host(),
+ to: Node::from("d.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("a.com").into_host(),
+ to: Node::from("e.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("a.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("b.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("c.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("d.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("d.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("z.com").into_host(),
+ to: Node::from("e.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
let mut scorer = Scorer::new(&graph, &[Node::from("b.com").id()], &[], false).await;
let e = Node::from("e.com").id();
@@ -237,39 +265,46 @@ mod tests {
#[allow(clippy::too_many_lines)]
async fn it_ranks_search_results() {
let dir = crate::gen_temp_dir().unwrap();
- let mut wrt = WebgraphWriter::new(
- &dir,
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
-
- wrt.insert(
- Node::from("b.com"),
- Node::from("a.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("c.com"),
- Node::from("d.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("b.com"),
- Node::from("e.com"),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("c.com"),
- Node::from("b.com"),
- String::new(),
- RelFlags::default(),
- );
-
- let graph = wrt.finalize();
+ let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("b.com").into_host(),
+ to: Node::from("a.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("c.com").into_host(),
+ to: Node::from("d.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("b.com").into_host(),
+ to: Node::from("e.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("c.com").into_host(),
+ to: Node::from("b.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index e5133904..3f12e758 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -135,7 +135,7 @@ mod tests {
index::Index,
models::dual_encoder::DualEncoder,
searcher::{LocalSearcher, SearchQuery},
- webgraph::{Edge, NodeDatum},
+ webgraph::{NodeID, SmallEdgeWithLabel},
webpage::{Html, Webpage},
};
@@ -438,11 +438,11 @@ mod tests {
..Default::default()
};
- webpage.set_backlinks(vec![Edge {
- from: NodeDatum::new(0u64, 0),
- to: NodeDatum::new(1u64, 1),
+ webpage.set_backlinks(vec![SmallEdgeWithLabel {
+ from: NodeID::from(0u64),
+ to: NodeID::from(1u64),
label: "test this is the best test site".to_string(),
- rel: Default::default(),
+ rel_flags: Default::default(),
}]);
index.insert(&webpage).expect("failed to insert webpage");
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index cc59912d..65cf33a3 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -24,7 +24,7 @@ mod tests {
searcher::{
api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,
},
- webgraph::{Node, WebgraphWriter},
+ webgraph::{Edge, Node, Webgraph},
webpage::{html::links::RelFlags, Html, Webpage},
};
const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
@@ -35,63 +35,81 @@ mod tests {
let dir = crate::gen_temp_dir().unwrap();
let (mut index, _dir) = Index::temporary().expect("Unable to open index");
- let mut wrt = WebgraphWriter::new(
- &dir,
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
-
- wrt.insert(
- Node::from("https://www.first.com").into_host(),
- Node::from("https://www.nan.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.nan.com").into_host(),
- Node::from("https://www.first.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.third.com").into_host(),
- Node::from("https://www.third.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.nan.com").into_host(),
- Node::from("https://www.second.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.second.com").into_host(),
- Node::from("https://www.nan.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.second.com").into_host(),
- Node::from("https://www.third.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.extra.com").into_host(),
- Node::from("https://www.first.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- wrt.insert(
- Node::from("https://www.second.com").into_host(),
- Node::from("https://www.extra.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
-
- let graph = wrt.finalize();
+ let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.first.com").into_host(),
+ to: Node::from("https://www.nan.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.nan.com").into_host(),
+ to: Node::from("https://www.first.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.third.com").into_host(),
+ to: Node::from("https://www.third.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.nan.com").into_host(),
+ to: Node::from("https://www.second.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.second.com").into_host(),
+ to: Node::from("https://www.nan.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.second.com").into_host(),
+ to: Node::from("https://www.third.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.extra.com").into_host(),
+ to: Node::from("https://www.first.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://www.second.com").into_host(),
+ to: Node::from("https://www.extra.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
index
.insert(&Webpage {
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 042d4ddb..8444129c 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -41,7 +41,7 @@ use crate::ranking::{
};
use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
-use crate::webgraph::remote::{RemoteWebgraph, WebgraphGranularity};
+use crate::webgraph::remote::RemoteWebgraph;
use crate::webgraph::EdgeLimit;
use crate::webpage::html::links::RelFlags;
use crate::widgets::{Widget, Widgets};
@@ -171,34 +171,42 @@ impl From<ApiConfig> for Config {
}
pub trait Graph {
- fn batch_raw_ingoing(
+ fn batch_raw_ingoing_hosts(
&self,
nodes: &[webgraph::NodeID],
limit: EdgeLimit,
- ) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;
+ ) -> impl Future<Output = Vec<Vec<webgraph::SmallEdge>>>;
}
-impl<G: WebgraphGranularity> Graph for RemoteWebgraph<G> {
- async fn batch_raw_ingoing(
+impl Graph for RemoteWebgraph {
+ async fn batch_raw_ingoing_hosts(
&self,
nodes: &[webgraph::NodeID],
limit: EdgeLimit,
- ) -> Vec<Vec<webgraph::Edge<()>>> {
- self.batch_raw_ingoing_edges(nodes, limit)
- .await
- .unwrap_or_default()
+ ) -> Vec<Vec<webgraph::SmallEdge>> {
+ self.batch_search(
+ nodes
+ .iter()
+ .map(|n| webgraph::query::HostBacklinksQuery::new(*n).with_limit(limit))
+ .collect(),
+ )
+ .await
+ .unwrap_or_default()
}
}
impl Graph for webgraph::Webgraph {
- async fn batch_raw_ingoing(
+ async fn batch_raw_ingoing_hosts(
&self,
nodes: &[webgraph::NodeID],
limit: EdgeLimit,
- ) -> Vec<Vec<webgraph::Edge<()>>> {
+ ) -> Vec<Vec<webgraph::SmallEdge>> {
nodes
.iter()
- .map(|n| self.raw_ingoing_edges(n, limit))
+ .map(|n| {
+ self.search(&webgraph::query::HostBacklinksQuery::new(*n).with_limit(limit))
+ .unwrap_or_default()
+ })
.collect()
}
}
@@ -207,12 +215,12 @@ impl<T> Graph for Arc<T>
where
T: Graph,
{
- fn batch_raw_ingoing(
+ fn batch_raw_ingoing_hosts(
&self,
nodes: &[webgraph::NodeID],
limit: EdgeLimit,
- ) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>> {
- self.as_ref().batch_raw_ingoing(nodes, limit)
+ ) -> impl Future<Output = Vec<Vec<webgraph::SmallEdge>>> {
+ self.as_ref().batch_raw_ingoing_hosts(nodes, limit)
}
}
@@ -221,14 +229,14 @@ where
T: Graph,
{
async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {
- self.batch_raw_ingoing(nodes, EdgeLimit::Limit(1024))
+ self.batch_raw_ingoing_hosts(nodes, EdgeLimit::Limit(1024))
.await
.into_iter()
.map(|edges| {
edges
.into_iter()
- .filter(|edge| !edge.rel.contains(RelFlags::NOFOLLOW))
- .map(|edge| edge.from.node())
+ .filter(|edge| !edge.rel_flags.contains(RelFlags::NOFOLLOW))
+ .map(|edge| edge.from)
.collect()
})
.collect()
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 82f42c28..3d0d954c 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -23,8 +23,9 @@ use url::Url;
use crate::{
ranking::{bitvec_similarity, inbound_similarity},
webgraph::{
- remote::{RemoteWebgraph, WebgraphGranularity},
- EdgeLimit, Node, NodeID,
+ query::{HostBacklinksQuery, HostForwardlinksQuery, Id2NodeQuery},
+ remote::RemoteWebgraph,
+ EdgeLimit, Node, NodeID, SmallEdge,
},
webpage::{html::links::RelFlags, url_ext::UrlExt},
SortableFloat,
@@ -36,13 +37,13 @@ pub struct ScoredNode {
pub score: f64,
}
-pub struct SimilarHostsFinder<G: WebgraphGranularity> {
- webgraph: Arc<RemoteWebgraph<G>>,
+pub struct SimilarHostsFinder {
+ webgraph: Arc<RemoteWebgraph>,
max_similar_hosts: usize,
}
-impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
- pub fn new(webgraph: Arc<RemoteWebgraph<G>>, max_similar_hosts: usize) -> Self {
+impl SimilarHostsFinder {
+ pub fn new(webgraph: Arc<RemoteWebgraph>, max_similar_hosts: usize) -> Self {
Self {
webgraph,
max_similar_hosts,
@@ -53,58 +54,62 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
inbound_similarity::Scorer::new(&self.webgraph, liked, &[], true).await
}
- pub async fn find_similar_hosts(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
- const DEAD_LINKS_BUFFER: usize = 30;
- let orig_limit = limit.min(self.max_similar_hosts);
- let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;
-
- let nodes: Vec<_> = nodes
- .iter()
- .filter_map(|url| Url::robust_parse(url).ok())
- .map(|url| Node::from(url).into_host())
- .collect();
-
- let domains = nodes
- .iter()
- .filter_map(|node| {
- Url::parse(&format!("http://{}", &node.as_str()))
- .ok()
- .and_then(|url| url.root_domain().map(|d| d.to_string()))
- })
- .collect::<HashSet<_>>();
-
- let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();
-
- let mut scorer = self.scorer(&nodes).await;
+ async fn batch_ingoing_edges(&self, nodes: &[NodeID], limit: EdgeLimit) -> Vec<Vec<SmallEdge>> {
+ self.webgraph
+ .batch_search(
+ nodes
+ .iter()
+ .map(|n| HostBacklinksQuery::new(*n).with_limit(limit))
+ .collect(),
+ )
+ .await
+ .unwrap_or_default()
+ }
- let in_edges = self
- .webgraph
- .batch_raw_ingoing_edges(&nodes, EdgeLimit::Limit(128))
+ async fn batch_outgoing_edges(
+ &self,
+ nodes: &[NodeID],
+ limit: EdgeLimit,
+ ) -> Vec<Vec<SmallEdge>> {
+ self.webgraph
+ .batch_search(
+ nodes
+ .iter()
+ .map(|n| HostForwardlinksQuery::new(*n).with_limit(limit))
+ .collect(),
+ )
.await
- .unwrap_or_default();
+ .unwrap_or_default()
+ }
+
+ async fn potential_nodes(&self, nodes: &[NodeID]) -> Vec<NodeID> {
+ let in_edges = self.batch_ingoing_edges(nodes, EdgeLimit::Limit(128)).await;
let backlink_nodes = in_edges
.iter()
.flatten()
- .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW))
- .map(|e| e.from.node())
+ .filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))
+ .map(|e| e.from)
.unique()
.collect::<Vec<_>>();
let outgoing_edges = self
- .webgraph
- .batch_raw_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512))
- .await
- .unwrap_or_default();
+ .batch_outgoing_edges(&backlink_nodes, EdgeLimit::Limit(512))
+ .await;
- let potential_nodes: Vec<_> = outgoing_edges
+ outgoing_edges
.iter()
.flatten()
- .filter(|e| !e.rel.contains(RelFlags::NOFOLLOW))
- .map(|e| e.to.node())
+ .filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))
+ .map(|e| e.to)
.unique()
.filter(|n| !nodes.contains(n))
- .collect();
+ .collect()
+ }
+
+ async fn scored_nodes(&self, nodes: &[NodeID], limit: usize) -> Vec<(NodeID, SortableFloat)> {
+ let mut scorer = self.scorer(nodes).await;
+ let potential_nodes = self.potential_nodes(nodes).await;
let inbounds = bitvec_similarity::BitVec::batch_new_for(&potential_nodes, &self.webgraph)
.await
@@ -112,7 +117,7 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
.zip_eq(potential_nodes.into_iter())
.map(|(b, n)| (n, b));
- let scored_nodes: Vec<_> = crate::sorted_k(
+ crate::sorted_k(
inbounds.map(|(n, b)| {
let score = scorer.score(&n, &b);
Reverse((SortableFloat(score), n))
@@ -121,7 +126,32 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
)
.into_iter()
.map(|Reverse((score, n))| (n, score))
- .collect();
+ .collect()
+ }
+
+ pub async fn find_similar_hosts(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
+ const DEAD_LINKS_BUFFER: usize = 30;
+ let orig_limit = limit.min(self.max_similar_hosts);
+ let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;
+
+ let nodes: Vec<_> = nodes
+ .iter()
+ .filter_map(|url| Url::robust_parse(url).ok())
+ .map(|url| Node::from(url).into_host())
+ .collect();
+
+ let domains = nodes
+ .iter()
+ .filter_map(|node| {
+ Url::parse(&format!("http://{}", &node.as_str()))
+ .ok()
+ .and_then(|url| url.root_domain().map(|d| d.to_string()))
+ })
+ .collect::<HashSet<_>>();
+
+ let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();
+
+ let scored_nodes = self.scored_nodes(&nodes, limit).await;
let potential_nodes = scored_nodes
.iter()
@@ -130,10 +160,8 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
// remove dead links (nodes without outgoing edges might be dead links)
let known_nodes = self
- .webgraph
- .batch_raw_ingoing_edges(&potential_nodes, EdgeLimit::Limit(1))
- .await
- .unwrap_or_default();
+ .batch_ingoing_edges(&potential_nodes, EdgeLimit::Limit(1))
+ .await;
let (potential_nodes, scores): (Vec<_>, Vec<_>) = scored_nodes
.into_iter()
@@ -143,7 +171,12 @@ impl<G: WebgraphGranularity> SimilarHostsFinder<G> {
let nodes = self
.webgraph
- .batch_get_node(&potential_nodes)
+ .batch_search(
+ potential_nodes
+ .into_iter()
+ .map(Id2NodeQuery::Host)
+ .collect(),
+ )
.await
.unwrap_or_default();
diff --git a/crates/core/src/webgraph/builder.rs b/crates/core/src/webgraph/builder.rs
index 4b3643b9..bc80ba7c 100644
--- a/crates/core/src/webgraph/builder.rs
+++ b/crates/core/src/webgraph/builder.rs
@@ -16,29 +16,23 @@
use std::path::Path;
-use crate::executor::Executor;
-
use super::Webgraph;
+use crate::{ampc::dht::ShardId, Result};
pub struct WebgraphBuilder {
path: Box<Path>,
- executor: Executor,
+ shard_id: ShardId,
}
impl WebgraphBuilder {
- pub fn new<P: AsRef<Path>>(path: P) -> Self {
+ pub fn new<P: AsRef<Path>>(path: P, shard_id: ShardId) -> Self {
Self {
path: path.as_ref().into(),
- executor: Executor::multi_thread("webgraph").unwrap(),
+ shard_id,
}
}
- pub fn single_threaded(mut self) -> Self {
- self.executor = Executor::single_thread();
- self
- }
-
- pub fn open(self) -> Webgraph {
- Webgraph::open(self.path, self.executor)
+ pub fn open(self) -> Result<Webgraph> {
+ Webgraph::open(self.path, self.shard_id)
}
}
diff --git a/crates/core/src/webgraph/centrality/approx_harmonic.rs b/crates/core/src/webgraph/centrality/approx_harmonic.rs
index b22a3245..c0d50a5f 100644
--- a/crates/core/src/webgraph/centrality/approx_harmonic.rs
+++ b/crates/core/src/webgraph/centrality/approx_harmonic.rs
@@ -20,7 +20,10 @@ use dashmap::DashMap;
use indicatif::ParallelProgressIterator;
use rayon::prelude::*;
-use crate::webgraph::{NodeID, ShortestPaths, Webgraph};
+use crate::{
+ config::WebgraphGranularity,
+ webgraph::{NodeID, ShortestPaths, Webgraph},
+};
const EPSILON: f64 = 0.3;
@@ -34,7 +37,7 @@ pub struct ApproxHarmonic {
impl ApproxHarmonic {
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output: P) -> Self {
- let num_nodes = graph.estimate_num_nodes();
+ let num_nodes = graph.page_nodes().len();
tracing::info!("found approximately {} nodes in graph", num_nodes);
@@ -42,14 +45,14 @@ impl ApproxHarmonic {
tracing::info!("sampling {} nodes", num_samples);
- let sampled = graph.random_nodes_with_outgoing(num_samples);
+ let sampled = graph.random_page_nodes_with_outgoing(num_samples);
let centralities: DashMap<NodeID, f32> = DashMap::new();
let norm = num_nodes as f32 / (num_samples as f32 * (num_nodes as f32 - 1.0));
sampled.into_par_iter().progress().for_each(|source| {
- let dists = graph.raw_distances_with_max(source, 7);
+ let dists = graph.raw_distances_with_max(source, 7, WebgraphGranularity::Page);
for (target, dist) in dists {
if dist == 0 {
diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rs
index 8a840c3f..cc4fe910 100644
--- a/crates/core/src/webgraph/centrality/betweenness.rs
+++ b/crates/core/src/webgraph/centrality/betweenness.rs
@@ -23,7 +23,7 @@ use indicatif::{ProgressBar, ProgressStyle};
use crate::{
intmap::IntMap,
- webgraph::{EdgeLimit, Node, NodeID, Webgraph},
+ webgraph::{query, EdgeLimit, Node, NodeID, Webgraph},
};
fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32) {
@@ -31,7 +31,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
let mut n = 0;
let mut max_dist = 0;
- let nodes: Vec<_> = graph.nodes().take(100_000).collect();
+ let nodes: Vec<_> = graph.host_nodes().into_iter().take(100_000).collect();
let pb =
if with_progress {
@@ -70,8 +70,11 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
while let Some(v) = q.pop_front() {
stack.push(v);
- for edge in graph.raw_outgoing_edges(&v, EdgeLimit::Unlimited) {
- let w = edge.to.node();
+ for edge in graph
+ .search(&query::ForwardlinksQuery::new(v).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default()
+ {
+ let w = edge.to;
if !distances.contains_key(&w) {
let dist_v = distances.get(&v).unwrap();
@@ -128,7 +131,15 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
(
centrality
.into_iter()
- .map(|(id, centrality)| (graph.id2node(&id).unwrap(), centrality / norm))
+ .map(|(id, centrality)| {
+ (
+ graph
+ .search(&query::Id2NodeQuery::Host(id))
+ .unwrap()
+ .unwrap(),
+ centrality / norm,
+ )
+ })
.collect(),
max_dist,
)
@@ -166,29 +177,32 @@ mod tests {
use file_store::temp::TempDir;
use maplit::hashmap;
- use crate::{webgraph::WebgraphWriter, webpage::html::links::RelFlags};
+ use crate::{
+ webgraph::{Edge, Webgraph},
+ webpage::html::links::RelFlags,
+ };
use super::*;
fn create_path_graph(n: usize) -> (Webgraph, TempDir) {
let temp_dir = file_store::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- temp_dir.as_ref().join("test-webgraph"),
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test-webgraph"), 0u64.into())
+ .open()
+ .unwrap();
for i in 0..n - 1 {
- writer.insert(
- Node::from(i.to_string()),
- Node::from((i + 1).to_string()),
- String::new(),
- RelFlags::default(),
- );
+ graph
+ .insert(Edge {
+ from: Node::from(i.to_string()),
+ to: Node::from((i + 1).to_string()),
+ label: String::new(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
+ })
+ .unwrap();
}
+ graph.commit().unwrap();
- let graph = writer.finalize();
(graph, temp_dir)
}
diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rs
deleted file mode 100644
index b87e5f46..00000000
--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs
+++ /dev/null
@@ -1,165 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-//! This is a centrality measure that is based on the harmonic centrality.
-//! The idea is to use the harmonic centrality from the domain graph to
-//! derive a centrality measure for the page graph.
-
-use anyhow::Result;
-use bloom::U64BloomFilter;
-use rayon::prelude::*;
-use std::{collections::BTreeMap, path::Path, sync::Mutex};
-
-use crate::webgraph::{EdgeLimit, NodeID, Webgraph};
-
-struct BloomMap {
- map: Vec<Mutex<U64BloomFilter>>,
-}
-
-impl BloomMap {
- fn new(num_blooms: usize, estimated_items: u64, fp: f64) -> Self {
- let mut map = Vec::new();
-
- for _ in 0..num_blooms {
- map.push(Mutex::new(U64BloomFilter::new(estimated_items, fp)));
- }
-
- Self { map }
- }
-
- fn insert(&self, item: &NodeID) {
- let h = item.as_u64();
- self.map[(h as usize) % self.map.len()]
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .insert(h);
- }
-
- fn finalize(mut self) -> U64BloomFilter {
- let mut bf = self.map.pop().unwrap().into_inner().unwrap();
-
- for m in self.map {
- bf.union(m.into_inner().unwrap());
- }
-
- bf
- }
-}
-
-pub struct DerivedCentrality {
- inner: speedy_kv::Db<NodeID, f64>,
-}
-
-impl DerivedCentrality {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let inner = speedy_kv::Db::open_or_create(path).unwrap();
- Self { inner }
- }
-
- pub fn build<P: AsRef<Path>>(
- host_harmonic: &speedy_kv::Db<NodeID, f64>,
- page_graph: &Webgraph,
- output: P,
- ) -> Result<Self> {
- if output.as_ref().exists() {
- return Err(anyhow::anyhow!("output path already exists"));
- }
-
- let num_nodes = page_graph.nodes().count();
-
- let has_outgoing = BloomMap::new(8, num_nodes as u64, 0.01);
-
- page_graph.par_edges().for_each(|edge| {
- has_outgoing.insert(&edge.from.node());
- });
-
- let has_outgoing = has_outgoing.finalize();
-
- let mut non_normalized =
- speedy_kv::Db::open_or_create(output.as_ref().join("non_normalized")).unwrap();
-
- let norms: Mutex<BTreeMap<NodeID, f64>> = Mutex::new(BTreeMap::new());
- let pb = indicatif::ProgressBar::new(num_nodes as u64);
-
- page_graph.node_ids().for_each(|(node, id)| {
- pb.inc(1);
- if has_outgoing.contains(id.as_u64()) {
- let host_node = node.clone().into_host().id();
-
- if let Some(harmonic) = host_harmonic.get(&host_node).unwrap() {
- let mut ingoing: Vec<_> = page_graph
- .raw_ingoing_edges(&id, EdgeLimit::Limit(128))
- .into_iter()
- .filter_map(|e| page_graph.id2node(&e.from.node()))
- .map(|n| n.into_host())
- .collect();
- ingoing.sort();
- ingoing.dedup();
-
- let votes = ingoing
- .into_iter()
- .filter_map(|n| host_harmonic.get(&n.id()).unwrap())
- .sum::<f64>();
- let page_score = harmonic * votes;
-
- non_normalized.insert(id, page_score).unwrap();
-
- if non_normalized.uncommitted_inserts() > 1_000_000 {
- non_normalized.commit().unwrap();
- }
-
- let mut l = norms.lock().unwrap_or_else(|e| e.into_inner());
- let norm = l.entry(host_node).or_insert(0.0);
- *norm = (*norm).max(votes);
- }
- }
- });
-
- non_normalized.commit().unwrap();
- non_normalized.merge_all_segments().unwrap();
-
- pb.finish_and_clear();
-
- let norms = norms.into_inner().unwrap();
-
- let mut db = speedy_kv::Db::open_or_create(output.as_ref()).unwrap();
- for (id, score) in non_normalized.iter() {
- let node = page_graph.id2node(&id).unwrap().into_host().id();
- let norm = norms.get(&node).unwrap();
- let normalized = score / *norm;
- db.insert(id, normalized).unwrap();
-
- if db.uncommitted_inserts() > 1_000_000 {
- db.commit().unwrap();
- }
- }
- db.commit().unwrap();
- db.merge_all_segments().unwrap();
-
- drop(non_normalized);
- std::fs::remove_dir_all(output.as_ref().join("non_normalized"))?;
-
- Ok(Self { inner: db })
- }
-
- pub fn get(&self, node: &NodeID) -> Option<f64> {
- self.inner.get(node).unwrap()
- }
-
- pub fn iter(&self) -> impl Iterator<Item = (NodeID, f64)> + '_ {
- self.inner.iter()
- }
-}
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index 192ba90d..fdcab2ae 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -27,7 +27,7 @@ use tracing::info;
use crate::{
hyperloglog::HyperLogLog,
kahan_sum::KahanSum,
- webgraph::{EdgeLimit, NodeID, Webgraph},
+ webgraph::{query, EdgeLimit, NodeID, Webgraph},
webpage::html::links::RelFlags,
};
@@ -56,7 +56,7 @@ fn initialize(
) -> u64 {
let mut num_nodes = 0;
- for node in graph.nodes() {
+ for node in graph.host_nodes() {
let mut counter = HyperLogLog::default();
counter.add(node.as_u64());
@@ -82,14 +82,14 @@ fn update_changed_counters(
exact_changed_nodes.iter().for_each(|changed_node| {
for edge in graph
- .raw_outgoing_edges(changed_node, EdgeLimit::Unlimited)
+ .search(&query::ForwardlinksQuery::new(*changed_node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default()
.into_iter()
- .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
+ .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
{
- if let (Some(counter_to), Some(counter_from)) = (
- counters.new.get_mut(&edge.to.node()),
- counters.old.get(&edge.from.node()),
- ) {
+ if let (Some(counter_to), Some(counter_from)) =
+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
+ {
if counter_to
.registers()
.iter()
@@ -97,9 +97,9 @@ fn update_changed_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.node().as_u64());
+ new_changed_nodes.insert(edge.to.as_u64());
- new_exact_changed_nodes.insert(edge.to.node());
+ new_exact_changed_nodes.insert(edge.to);
has_changes.store(true, Ordering::Relaxed);
}
@@ -126,14 +126,13 @@ fn update_all_counters(
}
graph
- .edges()
- .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
+ .host_edges()
+ .filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
.for_each(|edge| {
- if changed_nodes.contains(edge.from.node().as_u64()) {
- if let (Some(counter_to), Some(counter_from)) = (
- counters.new.get_mut(&edge.to.node()),
- counters.old.get(&edge.from.node()),
- ) {
+ if changed_nodes.contains(edge.from.as_u64()) {
+ if let (Some(counter_to), Some(counter_from)) =
+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
+ {
if counter_to
.registers()
.iter()
@@ -141,10 +140,10 @@ fn update_all_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.node().as_u64());
+ new_changed_nodes.insert(edge.to.as_u64());
if let Some(exact_changed_nodes) = &mut exact_changed_nodes {
- exact_changed_nodes.insert(edge.to.node());
+ exact_changed_nodes.insert(edge.to);
}
has_changes.store(true, Ordering::Relaxed);
@@ -220,7 +219,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);
- for node in graph.nodes() {
+ for node in graph.host_nodes() {
changed_nodes.insert(node.as_u64());
}
@@ -316,7 +315,7 @@ mod tests {
use super::*;
use crate::{
- webgraph::{Node, WebgraphWriter},
+ webgraph::{Edge, Node, Webgraph},
webpage::html::links::RelFlags,
};
@@ -342,18 +341,23 @@ mod tests {
fn test_graph() -> (Webgraph, TempDir) {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- temp_dir.as_ref().join("test"),
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test"), 0u64.into())
+ .open()
+ .unwrap();
for (from, to, label) in test_edges() {
- writer.insert(from, to, label, RelFlags::default());
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
}
- let graph = writer.finalize();
+ graph.commit().unwrap();
(graph, temp_dir)
}
@@ -361,99 +365,138 @@ mod tests {
#[test]
fn host_harmonic_centrality() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- temp_dir.as_ref().join("test"),
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
-
- writer.insert(
- Node::from("A.com/1").into_host(),
- Node::from("A.com/2").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/1").into_host(),
- Node::from("A.com/3").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/1").into_host(),
- Node::from("A.com/4").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/2").into_host(),
- Node::from("A.com/1").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/2").into_host(),
- Node::from("A.com/3").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/2").into_host(),
- Node::from("A.com/4").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/3").into_host(),
- Node::from("A.com/1").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/3").into_host(),
- Node::from("A.com/2").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/3").into_host(),
- Node::from("A.com/4").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/4").into_host(),
- Node::from("A.com/1").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/4").into_host(),
- Node::from("A.com/2").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("A.com/4").into_host(),
- Node::from("A.com/3").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("C.com").into_host(),
- Node::from("B.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
- writer.insert(
- Node::from("D.com").into_host(),
- Node::from("B.com").into_host(),
- String::new(),
- RelFlags::default(),
- );
-
- let graph = writer.finalize();
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test"), 0u64.into())
+ .open()
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/1").into_host(),
+ to: Node::from("A.com/2").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/1").into_host(),
+ to: Node::from("A.com/3").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/1").into_host(),
+ to: Node::from("A.com/4").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/2").into_host(),
+ to: Node::from("A.com/1").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/2").into_host(),
+ to: Node::from("A.com/3").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/2").into_host(),
+ to: Node::from("A.com/4").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/3").into_host(),
+ to: Node::from("A.com/1").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/3").into_host(),
+ to: Node::from("A.com/2").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/3").into_host(),
+ to: Node::from("A.com/4").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/4").into_host(),
+ to: Node::from("A.com/1").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/4").into_host(),
+ to: Node::from("A.com/2").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A.com/4").into_host(),
+ to: Node::from("A.com/3").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("C.com").into_host(),
+ to: Node::from("B.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("D.com").into_host(),
+ to: Node::from("B.com").into_host(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
let centrality = HarmonicCentrality::calculate(&graph);
@@ -485,69 +528,94 @@ mod tests {
let centrality = HarmonicCentrality::calculate(&graph);
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut other = WebgraphWriter::new(
- temp_dir.as_ref().join("test"),
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test"), 0u64.into())
+ .open()
+ .unwrap();
for (from, to, label) in test_edges() {
- other.insert(from, to, label, RelFlags::default());
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
}
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "1".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "2".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "3".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "4".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "5".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "6".to_string(),
- RelFlags::default(),
- );
- other.commit();
- other.insert(
- Node::from("A"),
- Node::from("B"),
- "7".to_string(),
- RelFlags::default(),
- );
- other.commit();
-
- let mut graph = other.finalize();
- graph.optimize_read();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "1".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "2".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "3".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "4".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "5".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "6".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "7".to_string(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+
+ graph.optimize_read().unwrap();
let centrality_extra = HarmonicCentrality::calculate(&graph);
@@ -557,18 +625,23 @@ mod tests {
#[test]
fn test_rel_flag_ignored() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = WebgraphWriter::new(
- temp_dir.as_ref().join("test"),
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test"), 0u64.into())
+ .open()
+ .unwrap();
for (from, to, label) in test_edges() {
- graph.insert(from, to, label, RelFlags::TAG);
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::TAG,
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
}
- let graph = graph.finalize();
+ graph.commit().unwrap();
let centrality = HarmonicCentrality::calculate(&graph);
@@ -578,18 +651,23 @@ mod tests {
#[test]
fn test_same_icann_domain_ignored() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = WebgraphWriter::new(
- &temp_dir,
- crate::executor::Executor::single_thread(),
- crate::webgraph::Compression::default(),
- None,
- );
+ let mut graph = Webgraph::builder(temp_dir.as_ref().join("test"), 0u64.into())
+ .open()
+ .unwrap();
for (from, to, label) in test_edges() {
- graph.insert(from, to, label, RelFlags::SAME_ICANN_DOMAIN);
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::SAME_ICANN_DOMAIN,
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
}
- let graph = graph.finalize();
+ graph.commit().unwrap();
let centrality = HarmonicCentrality::calculate(&graph);
diff --git a/crates/core/src/webgraph/centrality/mod.rs b/crates/core/src/webgraph/centrality/mod.rs
index b3392fcb..422b0ff4 100644
--- a/crates/core/src/webgraph/centrality/mod.rs
+++ b/crates/core/src/webgraph/centrality/mod.rs
@@ -22,7 +22,6 @@ use super::{Node, NodeID};
pub mod approx_harmonic;
pub mod betweenness;
-pub mod derived_harmonic;
pub mod harmonic;
#[derive(Debug, Clone, Copy)]
diff --git a/crates/core/src/webgraph/doc_address.rs b/crates/core/src/webgraph/doc_address.rs
new file mode 100644
index 00000000..da68d481
--- /dev/null
+++ b/crates/core/src/webgraph/doc_address.rs
@@ -0,0 +1,55 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{DocId, SegmentOrdinal};
+
+use crate::ampc::dht::ShardId;
+
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+ bincode::Encode,
+ bincode::Decode,
+ serde::Serialize,
+ serde::Deserialize,
+)]
+pub struct DocAddress {
+ pub shard_id: ShardId,
+ pub segment_ord: SegmentOrdinal,
+ pub doc_id: DocId,
+}
+
+impl DocAddress {
+ pub fn new(shard_id: ShardId, segment_ord: SegmentOrdinal, doc_id: DocId) -> Self {
+ Self {
+ shard_id,
+ segment_ord,
+ doc_id,
+ }
+ }
+}
+
+impl From<DocAddress> for tantivy::DocAddress {
+ fn from(doc_address: DocAddress) -> Self {
+ Self::new(doc_address.segment_ord, doc_address.doc_id)
+ }
+}
diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rs
new file mode 100644
index 00000000..f1f8c9d1
--- /dev/null
+++ b/crates/core/src/webgraph/document.rs
@@ -0,0 +1,160 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use strum::VariantArray;
+use tantivy::{
+ schema::{document::DocumentDeserialize, OwnedValue},
+ Document,
+};
+
+use crate::webpage::html::links::RelFlags;
+
+use super::{
+ schema::{Field, FieldEnum, FieldEnumDiscriminants},
+ Node, NodeID,
+};
+
+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
+pub struct SmallEdge {
+ pub from: NodeID,
+ pub to: NodeID,
+ pub rel_flags: RelFlags,
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct SmallEdgeWithLabel {
+ pub from: NodeID,
+ pub to: NodeID,
+ pub rel_flags: RelFlags,
+ pub label: String,
+}
+
+#[derive(
+ Debug,
+ Clone,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ utoipa::ToSchema,
+)]
+pub struct Edge {
+ pub from: Node,
+ pub to: Node,
+ pub rel_flags: RelFlags,
+ pub label: String,
+ pub sort_score: f64,
+}
+
+impl Edge {
+ pub fn empty() -> Self {
+ Self {
+ from: Node::empty(),
+ to: Node::empty(),
+ rel_flags: RelFlags::default(),
+ label: String::default(),
+ sort_score: 0.0,
+ }
+ }
+}
+
+impl Document for Edge {
+ type Value<'a> = ReferenceValue<'a>;
+
+ type FieldsValuesIter<'a> = FieldsIter<'a>;
+
+ fn iter_fields_and_values(&self) -> Self::FieldsValuesIter<'_> {
+ FieldsIter::new(self)
+ }
+}
+
+impl DocumentDeserialize for Edge {
+ fn deserialize<'de, D>(
+ deserializer: D,
+ ) -> Result<Self, tantivy::schema::document::DeserializeError>
+ where
+ D: tantivy::schema::document::DocumentDeserializer<'de>,
+ {
+ let mut deserializer = deserializer;
+ let mut edge = Edge::empty();
+
+ while let Some((field, value)) = deserializer.next_field::<OwnedValue>()? {
+ let field =
+ FieldEnum::from(FieldEnumDiscriminants::VARIANTS[field.field_id() as usize]);
+ field
+ .set_value(&mut edge, value)
+ .map_err(|e| tantivy::schema::document::DeserializeError::custom(e.to_string()))?;
+ }
+
+ Ok(edge)
+ }
+}
+
+pub struct FieldsIter<'a> {
+ edge: &'a Edge,
+ index: usize,
+}
+
+impl<'a> FieldsIter<'a> {
+ pub fn new(edge: &'a Edge) -> Self {
+ Self { edge, index: 0 }
+ }
+}
+
+impl<'a> Iterator for FieldsIter<'a> {
+ type Item = (tantivy::schema::Field, ReferenceValue<'a>);
+
+ fn next(&mut self) -> Option<Self::Item> {
+ FieldEnumDiscriminants::VARIANTS
+ .get(self.index)
+ .map(|field| {
+ let field = FieldEnum::from(*field);
+ let tv_field = tantivy::schema::Field::from_field_id(self.index as u32);
+
+ let value = field.document_value(self.edge);
+ self.index += 1;
+
+ (tv_field, value)
+ })
+ }
+}
+
+/// A enum representing a value for tantivy to index.
+#[derive(Clone, Debug, PartialEq)]
+pub enum ReferenceValue<'a> {
+ Str(&'a str),
+ U64(u64),
+ F64(f64),
+}
+
+impl<'a> tantivy::schema::Value<'a> for ReferenceValue<'a> {
+ type ArrayIter = std::iter::Empty<Self>;
+ type ObjectIter = std::iter::Empty<(&'a str, Self)>;
+
+ fn as_value(&self) -> tantivy::schema::document::ReferenceValue<'a, Self> {
+ match self {
+ ReferenceValue::Str(s) => tantivy::schema::document::ReferenceValue::Leaf(
+ tantivy::schema::document::ReferenceValueLeaf::Str(s),
+ ),
+ ReferenceValue::U64(u) => tantivy::schema::document::ReferenceValue::Leaf(
+ tantivy::schema::document::ReferenceValueLeaf::U64(*u),
+ ),
+ ReferenceValue::F64(f) => tantivy::schema::document::ReferenceValue::Leaf(
+ tantivy::schema::document::ReferenceValueLeaf::F64(*f),
+ ),
+ }
+ }
+}
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
deleted file mode 100644
index 8e1f8f0b..00000000
--- a/crates/core/src/webgraph/edge.rs
+++ /dev/null
@@ -1,218 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/
-
-use utoipa::ToSchema;
-
-use crate::webpage::html::links::RelFlags;
-
-use super::{FullNodeID, Node, NodeDatum, NodeID};
-
-pub const MAX_LABEL_LENGTH: usize = 1024;
-
-pub trait EdgeLabel
-where
- Self: Send + Sync + Sized,
-{
- fn to_bytes(&self) -> anyhow::Result<Vec<u8>>;
- fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self>;
-}
-
-impl EdgeLabel for String {
- fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
- Ok(self.as_bytes().to_vec())
- }
-
- fn from_bytes(bytes: &[u8]) -> anyhow::Result<Self> {
- Ok(String::from_utf8(bytes.to_vec())?)
- }
-}
-
-impl EdgeLabel for () {
- fn to_bytes(&self) -> anyhow::Result<Vec<u8>> {
- Ok(Vec::new())
- }
-
- fn from_bytes(_bytes: &[u8]) -> anyhow::Result<Self> {
- Ok(())
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]
-pub struct Edge<L>
-where
- L: EdgeLabel,
-{
- pub from: NodeDatum,
- pub to: NodeDatum,
- pub rel: RelFlags,
- pub label: L,
-}
-
-impl<L> Edge<L>
-where
- L: EdgeLabel,
-{
- pub fn rel_flags(&self) -> RelFlags {
- self.rel
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]
-pub struct InsertableEdge<L>
-where
- L: EdgeLabel,
-{
- pub from: FullNodeID,
- pub to: FullNodeID,
- pub rel: RelFlags,
- pub label: L,
-}
-
-#[cfg(test)]
-impl<L> From<InsertableEdge<L>> for Edge<L>
-where
- L: EdgeLabel,
-{
- fn from(edge: InsertableEdge<L>) -> Self {
- Edge {
- from: NodeDatum::new(edge.from.id, u64::MAX),
- to: NodeDatum::new(edge.to.id, u64::MAX),
- rel: edge.rel,
- label: edge.label,
- }
- }
-}
-
-#[derive(
- Debug,
- Clone,
- serde::Serialize,
- serde::Deserialize,
- bincode::Encode,
- bincode::Decode,
- PartialEq,
- Eq,
- Hash,
- ToSchema,
-)]
-#[serde(rename_all = "camelCase")]
-pub struct FullEdge {
- pub from: Node,
- pub to: Node,
- pub label: String,
-}
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct SegmentEdge<L>
-where
- L: EdgeLabel,
-{
- pub from: NodeDatum,
- pub to: NodeDatum,
- pub rel: RelFlags,
- pub label: L,
-}
-
-impl<L> From<SegmentEdge<L>> for Edge<L>
-where
- L: EdgeLabel,
-{
- fn from(edge: SegmentEdge<L>) -> Self {
- Edge {
- from: edge.from,
- to: edge.to,
- rel: edge.rel,
- label: edge.label,
- }
- }
-}
-
-#[cfg(test)]
-impl<L> From<Edge<L>> for SegmentEdge<L>
-where
- L: EdgeLabel,
-{
- fn from(edge: Edge<L>) -> Self {
- SegmentEdge {
- from: edge.from,
- to: edge.to,
- rel: edge.rel,
- label: edge.label,
- }
- }
-}
-
-#[cfg(test)]
-impl<L> From<InsertableEdge<L>> for SegmentEdge<L>
-where
- L: EdgeLabel,
-{
- fn from(edge: InsertableEdge<L>) -> Self {
- SegmentEdge {
- from: NodeDatum::new(edge.from.id, u64::MAX),
- to: NodeDatum::new(edge.to.id, u64::MAX),
- rel: edge.rel,
- label: edge.label,
- }
- }
-}
-
-#[derive(Debug, Clone)]
-pub struct StoredEdge<L = ()> {
- pub other: NodeDatum,
- pub rel: RelFlags,
- pub label: L,
-}
-
-impl StoredEdge<()> {
- pub fn new(other: NodeDatum, rel: RelFlags) -> Self {
- StoredEdge {
- other,
- rel,
- label: (),
- }
- }
-}
-
-impl<L> StoredEdge<L> {
- pub fn with_label<L2>(self, label: L2) -> StoredEdge<L2> {
- StoredEdge {
- other: self.other,
- rel: self.rel,
- label,
- }
- }
-
- #[inline]
- pub fn label(&self) -> &L {
- &self.label
- }
-
- #[inline]
- pub fn other(&self) -> NodeID {
- self.other.node()
- }
-
- #[inline]
- pub fn rel(&self) -> RelFlags {
- self.rel
- }
-
- #[inline]
- pub fn other_host_rank(&self) -> u64 {
- self.other.host_rank()
- }
-}
diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rs
deleted file mode 100644
index 555ae6bb..00000000
--- a/crates/core/src/webgraph/id_node_db.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/license
-
-use std::path::Path;
-
-use super::{Node, NodeID};
-
-pub struct Id2NodeDb {
- db: speedy_kv::Db<NodeID, Node>,
-}
-
-impl Id2NodeDb {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- Self {
- db: speedy_kv::Db::open_or_create(path).unwrap(),
- }
- }
-
- pub fn put(&mut self, id: &NodeID, node: &Node) {
- self.db.insert(*id, node.clone()).unwrap();
- }
-
- pub fn get(&self, id: &NodeID) -> Option<Node> {
- self.db.get(id).unwrap()
- }
-
- pub fn keys(&self) -> impl Iterator<Item = NodeID> + '_ {
- self.db.iter().map(|(id, _)| id)
- }
-
- pub fn iter_with_offset(&self, offset: u64) -> impl Iterator<Item = (NodeID, Node)> + '_ {
- self.db.iter_with_offset(offset)
- }
-
- pub fn estimate_num_keys(&self) -> usize {
- self.db.len()
- }
-
- pub fn iter(&self) -> impl Iterator<Item = (NodeID, Node)> + '_ {
- self.db.iter()
- }
-
- pub fn merge(&mut self, other: Self) {
- self.db.merge(other.db).unwrap();
- }
-
- pub fn flush(&mut self) {
- self.db.commit().unwrap();
- }
-
- pub fn optimize_read(&mut self) {
- self.db.merge_all_segments().unwrap();
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_id2node_db() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut db = Id2NodeDb::open(&temp_dir);
-
- let a_node = Node::from("a".to_string());
- let a_id = NodeID::from(0_u64);
-
- db.put(&a_id, &a_node);
- db.flush();
-
- assert_eq!(db.get(&a_id), Some(a_node.clone()));
-
- let b_node = Node::from("b".to_string());
- let b_id = NodeID::from(1_u64);
-
- assert_eq!(db.get(&b_id), None);
-
- db.put(&b_id, &b_node);
- db.flush();
-
- assert_eq!(db.get(&b_id), Some(b_node));
- assert_eq!(db.get(&a_id), Some(a_node));
- }
-}
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
deleted file mode 100644
index f862182c..00000000
--- a/crates/core/src/webgraph/merge.rs
+++ /dev/null
@@ -1,440 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/license
-
-use std::{cmp::Reverse, collections::BinaryHeap};
-
-use super::{store::EdgeRange, NodeID, StoredEdge};
-use std::hash::Hash;
-
-#[derive(Debug)]
-pub struct MergeNode<O = ()> {
- node: NodeID,
- range: EdgeRange,
- labels: std::ops::Range<u64>,
- ord: O,
-}
-
-impl MergeNode {
- pub fn new(node: NodeID, range: EdgeRange, labels: std::ops::Range<u64>) -> Self {
- Self {
- node,
- range,
- labels,
- ord: (),
- }
- }
-}
-
-impl<O> MergeNode<O> {
- pub fn with_ord<O2>(self, ord: O2) -> MergeNode<O2> {
- MergeNode {
- node: self.node,
- range: self.range,
- labels: self.labels,
- ord,
- }
- }
-
- pub fn id(&self) -> NodeID {
- self.node
- }
-
- pub fn range(&self) -> &EdgeRange {
- &self.range
- }
-
- pub fn labels(&self) -> std::ops::Range<u64> {
- self.labels.clone()
- }
-
- pub fn ord(&self) -> &O {
- &self.ord
- }
-}
-
-impl<O> Ord for MergeNode<O> {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- // nodes in the fst are sorted by their id,
- // not by their sort key
- self.node.cmp(&other.node)
- }
-}
-impl<O> PartialOrd for MergeNode<O> {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-impl<O> PartialEq for MergeNode<O> {
- fn eq(&self, other: &Self) -> bool {
- self.node == other.node
- }
-}
-impl<O> Eq for MergeNode<O> {}
-
-#[derive(Debug, Clone, Copy)]
-pub struct MergeSegmentOrd(usize);
-impl MergeSegmentOrd {
- pub fn new(ord: usize) -> Self {
- Self(ord)
- }
-
- pub fn as_usize(&self) -> usize {
- self.0
- }
-}
-
-type MinHeap<T> = BinaryHeap<Reverse<T>>;
-
-/// Merge multiple iterators of MergeNodes into a single iterator based on the node id
-pub struct MergeIter<'a> {
- iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = MergeNode<MergeSegmentOrd>> + 'a>>>,
-}
-impl<'a> MergeIter<'a> {
- pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {
- let iters = iters
- .into_iter()
- .enumerate()
- .map(|(ord, iter)| {
- let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
- as Box<dyn Iterator<Item = _>>;
-
- Reverse(file_store::Peekable::new(it))
- })
- .collect();
-
- Self { iters }
- }
-
- pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {
- buf.clear();
-
- let next = {
- let item = self.iters.peek_mut();
-
- if item.is_none() {
- return false;
- }
-
- let mut item = item.unwrap();
-
- if item.0.peek().is_none() {
- return false;
- }
-
- item.0.next().unwrap()
- };
-
- let node = next.node;
- buf.push(next);
-
- // advance all iterators that have the same node
- while let Some(mut peek) = self.iters.peek_mut() {
- if peek.0.peek().map(|x| x.node) == Some(node) {
- buf.push(peek.0.next().unwrap());
- } else {
- break;
- }
- }
-
- true
- }
-}
-
-/// Merge multiple iterators of NodeDatum into a single iterator based on the sort key
-pub struct EdgeMerger<'a, L = String> {
- iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = StoredEdge<L>> + 'a>>>,
-}
-
-impl<'a, L> EdgeMerger<'a, L> {
- pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {
- let iters = iters
- .into_iter()
- .map(|iter| {
- let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
- Reverse(file_store::Peekable::new(it))
- })
- .collect();
-
- Self { iters }
- }
-}
-
-impl<'a, L> Iterator for EdgeMerger<'a, L> {
- type Item = StoredEdge<L>;
-
- fn next(&mut self) -> Option<Self::Item> {
- let res = self.iters.peek_mut().and_then(|mut item| item.0.next());
-
- if let Some(edge) = &res {
- while let Some(mut peek) = self.iters.peek_mut() {
- if peek.0.peek().map(|x| x.other.node()) == Some(edge.other.node()) {
- peek.0.next().unwrap();
- } else {
- break;
- }
- }
- } else {
- debug_assert!(self.iters.iter().all(|p| p.0.peek().is_none()));
- }
-
- res
- }
-}
-
-impl<L> Ord for StoredEdge<L> {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.other.cmp(&other.other)
- }
-}
-
-impl<L> PartialOrd for StoredEdge<L> {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl<L> PartialEq for StoredEdge<L> {
- fn eq(&self, other: &Self) -> bool {
- self.other == other.other
- }
-}
-
-impl<L> Eq for StoredEdge<L> {}
-
-impl<L> Hash for StoredEdge<L> {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.other.hash(state);
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::{webgraph::NodeDatum, webpage::html::links::RelFlags};
- use proptest::prelude::*;
-
- use super::*;
-
- #[test]
- fn test_merge_nodes() {
- let a = vec![
- MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
- MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
- MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
- ];
-
- let b = vec![
- MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10),
- MergeNode::new(3u64.into(), EdgeRange::new(0..10, 5), 0..10),
- MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
- ];
-
- let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
- let mut buf = Vec::new();
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 1u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 2u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 3u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 4u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 2);
- assert_eq!(buf[0].id(), 5u64.into());
- assert_eq!(buf[1].id(), 5u64.into());
-
- assert!(!merger.advance(&mut buf));
- }
-
- #[test]
- fn test_merge_nodes_unequal_len1() {
- let a = vec![
- MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
- MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
- MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
- ];
-
- let b = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];
-
- let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
- let mut buf = Vec::new();
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 1u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 2u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 4u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 5u64.into());
-
- assert!(!merger.advance(&mut buf));
- }
-
- #[test]
- fn test_merge_nodes_unequal_len2() {
- let a = vec![MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10)];
- let b = vec![
- MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),
- MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),
- MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),
- ];
-
- let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);
- let mut buf = Vec::new();
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 1u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 2u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 4u64.into());
-
- assert!(merger.advance(&mut buf));
- assert_eq!(buf.len(), 1);
- assert_eq!(buf[0].id(), 5u64.into());
-
- assert!(!merger.advance(&mut buf));
- }
-
- #[test]
- fn test_datum_merge() {
- let a = vec![
- StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
- ];
-
- let b = vec![
- StoredEdge::new(NodeDatum::new(4u64, 2), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(5u64, 3), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
- ];
-
- let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);
-
- assert_eq!(merger.next().unwrap().other.host_rank(), 1);
- assert_eq!(merger.next().unwrap().other.host_rank(), 2);
- assert_eq!(merger.next().unwrap().other.host_rank(), 3);
- assert_eq!(merger.next().unwrap().other.host_rank(), 4);
- assert_eq!(merger.next().unwrap().other.host_rank(), 5);
- assert!(merger.next().is_none());
- }
-
- #[test]
- fn test_datum_merge_unequal_len() {
- let a = vec![
- StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
- ];
-
- let b = vec![StoredEdge::new(
- NodeDatum::new(4u64, 2),
- RelFlags::default(),
- )];
-
- let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);
-
- assert_eq!(merger.next().unwrap().other.host_rank(), 1);
- assert_eq!(merger.next().unwrap().other.host_rank(), 2);
- assert_eq!(merger.next().unwrap().other.host_rank(), 4);
- assert_eq!(merger.next().unwrap().other.host_rank(), 5);
- assert!(merger.next().is_none());
- }
-
- #[test]
- fn test_mergenode_heap_order() {
- let mut heap = MinHeap::new();
-
- heap.push(Reverse(MergeNode::new(
- 15414505588350686497u64.into(),
- EdgeRange::new(0..10, 1),
- 0..10,
- )));
- heap.push(Reverse(MergeNode::new(
- 470624832116206178u64.into(),
- EdgeRange::new(0..10, 1),
- 0..10,
- )));
- heap.push(Reverse(MergeNode::new(
- 15414505588350686497u64.into(),
- EdgeRange::new(0..10, 1),
- 0..10,
- )));
-
- assert_eq!(
- heap.into_iter()
- .map(|x| x.0.id().as_u64())
- .collect::<Vec<_>>(),
- vec![
- 470624832116206178,
- 15414505588350686497,
- 15414505588350686497
- ]
- );
- }
-
- proptest! {
- #[test]
- fn merge(mut ids: Vec<u64>) {
- ids.sort();
-
- let mut a: Vec<MergeNode> = Vec::new();
- let mut b: Vec<MergeNode> = Vec::new();
-
- for id in ids.clone() {
- if rand::random() {
- a.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));
- } else {
- b.push(MergeNode::new(NodeID::from(id), EdgeRange::new(0..10, 4), 0..10));
- }
- }
-
- let mut merge = MergeIter::new(vec![a.into_iter() , b.into_iter()]);
-
- let mut merged = Vec::new();
- let mut buf = Vec::new();
-
- while merge.advance(&mut buf) {
- merged.extend(buf.iter().map(|n| n.id().as_u64()));
- }
-
- prop_assert_eq!(merged, ids);
- }
- }
-}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 8be8563a..38963c93 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -13,82 +13,42 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::fs::File;
-use std::io::{BufReader, BufWriter, Read, Write};
+
+use std::fs;
use std::path::{Path, PathBuf};
-use std::sync::Arc;
-use std::{fs, io};
+
+use crate::ampc::dht::ShardId;
+use crate::Result;
use itertools::Itertools;
+pub use query::{Collector, Query};
use rand::seq::{IteratorRandom, SliceRandom};
-use rayon::prelude::*;
-use uuid::Uuid;
-
-use self::id_node_db::Id2NodeDb;
-use self::segment::Segment;
-use crate::executor::Executor;
+use rustc_hash::FxHashSet;
+use store::EdgeStore;
-use crate::iter_ext::VectorExt;
-use crate::Result;
pub use builder::WebgraphBuilder;
-pub use compression::Compression;
-pub use edge::*;
+pub use document::*;
pub use node::*;
pub use shortest_path::ShortestPaths;
-pub use writer::WebgraphWriter;
+
+use searcher::Searcher;
mod builder;
pub mod centrality;
-mod compression;
-mod edge;
-mod id_node_db;
-mod merge;
+mod doc_address;
+mod document;
mod node;
+pub mod query;
pub mod remote;
-mod segment;
+mod schema;
+mod searcher;
mod shortest_path;
mod store;
-mod store_writer;
-mod writer;
-type SegmentID = String;
-
-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]
-struct Meta {
- comitted_segments: Vec<SegmentID>,
-}
-
-impl Meta {
- fn open<P: AsRef<Path>>(path: P) -> Self {
- let mut reader = BufReader::new(
- File::options()
- .create(true)
- .truncate(false)
- .write(true)
- .read(true)
- .open(path)
- .unwrap(),
- );
- let mut buf = String::new();
- reader.read_to_string(&mut buf).unwrap();
- serde_json::from_str(&buf).unwrap_or_default()
- }
+#[cfg(test)]
+pub mod tests;
- fn save<P: AsRef<Path>>(&self, path: P) {
- let mut writer = BufWriter::new(
- File::options()
- .create(true)
- .write(true)
- .read(true)
- .truncate(true)
- .open(path)
- .unwrap(),
- );
-
- let json = serde_json::to_string_pretty(&self).unwrap();
- writer.write_all(json.as_bytes()).unwrap();
- }
-}
+pub const MAX_LABEL_LENGTH: usize = 256;
#[derive(
Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,
@@ -100,986 +60,134 @@ pub enum EdgeLimit {
}
impl EdgeLimit {
- fn apply<'a, T>(&self, it: impl Iterator<Item = T> + 'a) -> Box<dyn Iterator<Item = T> + 'a> {
+ pub fn limit(&self) -> Option<usize> {
+ match self {
+ EdgeLimit::Unlimited => None,
+ EdgeLimit::Limit(limit) => Some(*limit),
+ EdgeLimit::LimitAndOffset { limit, .. } => Some(*limit),
+ }
+ }
+
+ pub fn offset(&self) -> Option<usize> {
match self {
- EdgeLimit::Unlimited => Box::new(it),
- EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),
- EdgeLimit::LimitAndOffset { limit, offset } => Box::new(it.skip(*offset).take(*limit)),
+ EdgeLimit::LimitAndOffset { offset, .. } => Some(*offset),
+ _ => None,
}
}
}
pub struct Webgraph {
path: String,
- segments: Vec<Segment>,
- executor: Arc<Executor>,
- id2node: Id2NodeDb,
- meta: Meta,
+ store: EdgeStore,
}
impl Webgraph {
- pub fn builder<P: AsRef<Path>>(path: P) -> WebgraphBuilder {
- WebgraphBuilder::new(path)
+ pub fn builder<P: AsRef<Path>>(path: P, shard_id: ShardId) -> WebgraphBuilder {
+ WebgraphBuilder::new(path, shard_id)
}
pub fn path(&self) -> PathBuf {
PathBuf::from(&self.path)
}
- fn meta<P: AsRef<Path>>(path: P) -> Meta {
- let meta_path = path.as_ref().join("metadata.json");
- Meta::open(meta_path)
- }
-
- fn save_metadata(&mut self) {
- let path = Path::new(&self.path).join("metadata.json");
- self.meta.save(path);
- }
-
- fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
- fs::create_dir_all(&path).unwrap();
- let meta = Self::meta(&path);
+ pub fn open<P: AsRef<Path>>(path: P, shard_id: ShardId) -> Result<Self> {
+ fs::create_dir_all(&path)?;
- fs::create_dir_all(path.as_ref().join("segments")).unwrap();
+ let store = EdgeStore::open(path.as_ref().join("edges"), shard_id)?;
- let mut segments = Vec::new();
- for segment in &meta.comitted_segments {
- segments.push(Segment::open(
- path.as_ref().join("segments"),
- segment.clone(),
- ));
- }
-
- Self {
+ Ok(Self {
+ store,
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- segments,
- executor: Arc::new(executor),
- id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
- meta,
- }
+ })
}
- pub fn merge(&mut self, other: Webgraph) -> io::Result<()> {
- let other_folder = other.path.clone();
- self.id2node.merge(other.id2node);
- self.id2node.flush();
-
- for segment in other.segments {
- let id = segment.id();
- let new_path = Path::new(&self.path).join("segments");
- std::fs::rename(segment.path(), new_path.join(segment.id())).unwrap();
-
- self.meta.comitted_segments.push(segment.id());
- drop(segment);
- self.segments.push(Segment::open(new_path, id));
- }
-
- fs::remove_dir_all(other_folder)?;
-
- self.save_metadata();
+ pub fn insert(&mut self, edge: Edge) -> Result<()> {
+ let mut edge = edge;
+ edge.label = edge.label.chars().take(MAX_LABEL_LENGTH).collect();
+ self.store.insert(edge)?;
Ok(())
}
- pub fn merge_all_segments(&mut self, compression: Compression) -> Result<()> {
- let segments = std::mem::take(&mut self.segments);
-
- let id = Uuid::new_v4().to_string();
- let path = Path::new(&self.path).join("segments");
-
- Segment::merge(segments, compression, &path, id.clone())?;
- let new_segment = Segment::open(path, id.clone());
-
- self.segments.push(new_segment);
- self.meta.comitted_segments = vec![id];
-
- self.save_metadata();
-
+ pub fn commit(&mut self) -> Result<()> {
+ self.store.commit()?;
Ok(())
}
- pub fn optimize_read(&mut self) {
- self.executor
- .map(|s| s.optimize_read(), self.segments.iter_mut())
- .unwrap();
-
- self.id2node.optimize_read();
- }
-
- pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
- self.ingoing_edges_by_id(&node.id(), limit)
- }
+ pub fn merge(&mut self, other: Webgraph) -> Result<()> {
+ let other_folder = other.path.clone();
+ self.store.merge(other.store)?;
- pub fn ingoing_edges_by_id(&self, node_id: &NodeID, limit: EdgeLimit) -> Vec<FullEdge> {
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.ingoing_edges_with_label(node_id))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
- .unique_by(|edge| edge.from.node());
+ fs::remove_dir_all(other_folder)?;
- limit
- .apply(edges)
- .map(|e| FullEdge {
- from: self.id2node(&e.from.node()).unwrap(),
- to: self.id2node(&e.to.node()).unwrap(),
- label: e.label,
- })
- .collect()
+ Ok(())
}
- pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.ingoing_edges(node))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
- .unique_by(|edge| edge.from.node());
-
- limit
- .apply(edges)
- .map(|e| Edge {
- from: e.from,
- to: e.to,
- label: e.label,
- rel: e.rel,
- })
- .collect()
+ pub fn optimize_read(&mut self) -> Result<()> {
+ self.store.optimize_read()?;
+ Ok(())
}
- pub fn raw_ingoing_edges_with_labels(
+ pub fn search_initial<Q: Query>(
&self,
- node: &NodeID,
- limit: EdgeLimit,
- ) -> Vec<Edge<String>> {
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.ingoing_edges_with_label(node))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
- .unique_by(|edge| edge.from.node());
-
- limit
- .apply(edges)
- .map(|e| Edge {
- from: e.from,
- to: e.to,
- label: e.label,
- rel: e.rel,
- })
- .collect()
+ query: &Q,
+ ) -> Result<<Q::Collector as Collector>::Fruit> {
+ self.store.search_initial(query)
}
- pub fn raw_outgoing_edges_with_labels(
+ pub fn retrieve<Q: Query>(
&self,
- node: &NodeID,
- limit: EdgeLimit,
- ) -> Vec<Edge<String>> {
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.outgoing_edges_with_label(node))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
- .unique_by(|edge| edge.to.node());
-
- limit
- .apply(edges)
- .map(|e| Edge {
- from: e.from,
- to: e.to,
- label: e.label,
- rel: e.rel,
- })
- .collect()
+ query: &Q,
+ fruit: <Q::Collector as Collector>::Fruit,
+ ) -> Result<Q::IntermediateOutput> {
+ self.store.retrieve(query, fruit)
}
- pub fn out_degree_upper_bound(&self, node: &NodeID) -> u64 {
- self.segments
- .iter()
- .map(|segment| segment.out_degree(node))
- .sum()
+ pub fn search<Q>(&self, query: &Q) -> Result<Q::Output>
+ where
+ Q: Query,
+ <<Q::Collector as Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as Collector>::Fruit>,
+ {
+ self.store.search(query)
}
- pub fn in_degree_upper_bound(&self, node: &NodeID) -> u64 {
- self.segments
- .iter()
- .map(|segment| segment.in_degree(node))
- .sum()
+ pub fn host_nodes(&self) -> FxHashSet<NodeID> {
+ self.store.iter_host_node_ids(0, u32::MAX).collect()
}
- pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {
- let id = node.id();
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.outgoing_edges_with_label(&id))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
- .unique_by(|edge| edge.to.node());
-
- limit
- .apply(edges)
- .map(|e| FullEdge {
- from: self.id2node(&e.from.node()).unwrap(),
- to: self.id2node(&e.to.node()).unwrap(),
- label: e.label,
- })
- .collect()
+ pub fn page_nodes(&self) -> FxHashSet<NodeID> {
+ self.store.iter_page_node_ids(0, u32::MAX).collect()
}
- pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {
- let edges = self
- .segments
- .iter()
- .map(|segment| segment.outgoing_edges(node))
- .collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
- .unique_by(|edge| edge.to.node());
-
- limit
- .apply(edges)
- .map(|e| Edge {
- from: e.from,
- to: e.to,
- label: e.label,
- rel: e.rel,
- })
- .collect()
- }
-
- pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- self.id2node.get(id)
- }
-
- pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {
- self.id2node.keys()
- }
-
- pub fn random_nodes_with_outgoing(&self, num: usize) -> Vec<NodeID> {
+ pub fn random_page_nodes_with_outgoing(&self, num: usize) -> Vec<NodeID> {
let mut rng = rand::thread_rng();
let mut nodes = self
- .edges()
- .map(|e| e.from.node())
+ .page_edges()
+ .map(|e| e.from)
.unique()
.choose_multiple(&mut rng, num);
nodes.shuffle(&mut rng);
nodes
}
- pub fn par_nodes(&self) -> impl ParallelIterator<Item = NodeID> + '_ {
- self.id2node.keys().par_bridge()
- }
-
- pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {
- self.id2node.iter().map(|(id, node)| (node, id))
- }
-
- pub fn estimate_num_nodes(&self) -> usize {
- self.id2node.estimate_num_keys()
- }
-
- pub fn iter_nodes_with_offset(&self, offset: u64) -> impl Iterator<Item = (NodeID, Node)> + '_ {
- self.id2node.iter_with_offset(offset)
+ pub fn page_node_ids_with_offset(&self, offset: u64, limit: u64) -> Vec<NodeID> {
+ self.store
+ .iter_page_node_ids(offset as u32, limit as u32)
+ .collect()
}
/// Iterate all edges in the graph at least once.
/// Some edges may be returned multiple times.
/// This happens if they are present in more than one segment.
- pub fn edges(&self) -> impl Iterator<Item = Edge<()>> + '_ {
- self.segments
- .iter()
- .flat_map(|segment| segment.edges().map(|e| e.into()))
- }
-
- pub fn par_edges(&self) -> impl ParallelIterator<Item = Edge<()>> + '_ {
- self.segments
- .par_iter()
- .flat_map(|segment| segment.edges().par_bridge().map(|e| e.into()))
- }
-}
-
-#[cfg(test)]
-pub mod tests {
- use crate::webpage::html::links::RelFlags;
-
- use super::*;
- use file_store::temp::TempDir;
- use proptest::prelude::*;
-
- pub fn test_edges() -> Vec<(Node, Node, String)> {
- vec![
- (Node::from("A"), Node::from("B"), String::new()),
- (Node::from("B"), Node::from("C"), String::new()),
- (Node::from("A"), Node::from("C"), String::new()),
- (Node::from("C"), Node::from("A"), String::new()),
- (Node::from("D"), Node::from("C"), String::new()),
- ]
- }
-
- pub fn test_graph() -> (Webgraph, TempDir) {
- // ┌-----┐
- // │ │
- // ┌───A◄─┐ │
- // │ │ │
- // ▼ │ │
- // B─────►C◄-┘
- // ▲
- // │
- // │
- // D
-
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = WebgraphWriter::new(
- &temp_dir,
- Executor::single_thread(),
- Compression::default(),
- None,
- );
-
- for (from, to, label) in test_edges() {
- graph.insert(from, to, label, RelFlags::default());
- }
-
- graph.commit();
-
- (graph.finalize(), temp_dir)
- }
-
- #[test]
- fn distance_calculation() {
- let (graph, _temp_dir) = test_graph();
-
- let distances = graph.distances(Node::from("D"));
-
- assert_eq!(distances.get(&Node::from("C")), Some(&1));
- assert_eq!(distances.get(&Node::from("A")), Some(&2));
- assert_eq!(distances.get(&Node::from("B")), Some(&3));
+ pub fn page_edges(&self) -> impl Iterator<Item = SmallEdge> + '_ {
+ self.store.iter_pages_small()
}
- #[test]
- fn nonexisting_node() {
- let (graph, _temp_dir) = test_graph();
- assert_eq!(graph.distances(Node::from("E")).len(), 0);
- assert_eq!(graph.reversed_distances(Node::from("E")).len(), 0);
- }
-
- #[test]
- fn reversed_distance_calculation() {
- let (graph, _temp_dir) = test_graph();
-
- let distances = graph.reversed_distances(Node::from("D"));
-
- assert_eq!(distances.get(&Node::from("C")), None);
- assert_eq!(distances.get(&Node::from("A")), None);
- assert_eq!(distances.get(&Node::from("B")), None);
-
- let distances = graph.reversed_distances(Node::from("A"));
-
- assert_eq!(distances.get(&Node::from("C")), Some(&1));
- assert_eq!(distances.get(&Node::from("D")), Some(&2));
- assert_eq!(distances.get(&Node::from("B")), Some(&2));
- }
-
- #[test]
- fn merge_path() {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- for (i, (from, to, label)) in (0..).zip(&[
- (Node::from("A"), Node::from("B"), String::new()),
- (Node::from("B"), Node::from("C"), String::new()),
- (Node::from("C"), Node::from("D"), String::new()),
- (Node::from("D"), Node::from("E"), String::new()),
- (Node::from("E"), Node::from("F"), String::new()),
- (Node::from("F"), Node::from("G"), String::new()),
- (Node::from("G"), Node::from("H"), String::new()),
- ]) {
- let mut wrt = WebgraphWriter::new(
- &temp_dir.as_ref().join(format!("test_{}", i)),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
- graph.optimize_read();
-
- assert_eq!(
- graph.distances(Node::from("A")).get(&Node::from("H")),
- Some(&7)
- );
-
- assert_eq!(
- graph
- .reversed_distances(Node::from("H"))
- .get(&Node::from("A")),
- Some(&7)
- );
- }
-
- #[test]
- fn merge_simple() {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- for (i, (from, to, label)) in (0..).zip(&test_edges()) {
- let mut wrt = WebgraphWriter::new(
- &temp_dir.as_ref().join(format!("test_{}", i)),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
- graph.optimize_read();
-
- let mut res = graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.to.cmp(&b.to));
- assert_eq!(res.len(), 2);
-
- assert_eq!(res[0].to, Node::from("B"));
- assert_eq!(res[1].to, Node::from("C"));
-
- let mut res = graph.outgoing_edges(Node::from("B"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.to.cmp(&b.to));
-
- assert_eq!(res.len(), 1);
- assert_eq!(res[0].to, Node::from("C"));
-
- let mut res = graph.outgoing_edges(Node::from("C"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.to.cmp(&b.to));
-
- assert_eq!(res.len(), 1);
- assert_eq!(res[0].to, Node::from("A"));
-
- let mut res = graph.outgoing_edges(Node::from("D"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.to.cmp(&b.to));
-
- assert_eq!(res.len(), 1);
- assert_eq!(res[0].to, Node::from("C"));
-
- let mut res = graph.ingoing_edges(Node::from("A"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.from.cmp(&b.from));
-
- assert_eq!(res.len(), 1);
- assert_eq!(res[0].from, Node::from("C"));
-
- let mut res = graph.ingoing_edges(Node::from("B"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.from.cmp(&b.from));
-
- assert_eq!(res.len(), 1);
- assert_eq!(res[0].from, Node::from("A"));
-
- let mut res = graph.ingoing_edges(Node::from("C"), EdgeLimit::Unlimited);
- res.sort_by(|a, b| a.from.cmp(&b.from));
-
- assert_eq!(res.len(), 3);
- assert_eq!(res[0].from, Node::from("A"));
- assert_eq!(res[1].from, Node::from("B"));
- assert_eq!(res[2].from, Node::from("D"));
- }
-
- #[test]
- fn merge_cycle() {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- for (i, (from, to, label)) in (0..).zip(&[
- (Node::from("A"), Node::from("B"), String::new()),
- (Node::from("B"), Node::from("B"), String::new()),
- (Node::from("B"), Node::from("C"), String::new()),
- (Node::from("C"), Node::from("A"), String::new()),
- ]) {
- let mut wrt = WebgraphWriter::new(
- &temp_dir.as_ref().join(format!("test_{}", i)),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- assert_eq!(
- graph.distances(Node::from("A")).get(&Node::from("C")),
- Some(&2)
- );
-
- graph.merge_all_segments(Compression::default()).unwrap();
-
- assert_eq!(
- graph.distances(Node::from("A")).get(&Node::from("C")),
- Some(&2)
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
- assert_eq!(
- graph
- .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
- assert_eq!(
- graph
- .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
- }
-
- #[test]
- fn merge_star() {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- for (i, (from, to, label)) in (0..).zip(&[
- (Node::from("A"), Node::from("B"), String::new()),
- (Node::from("A"), Node::from("C"), String::new()),
- (Node::from("A"), Node::from("D"), String::new()),
- (Node::from("A"), Node::from("E"), String::new()),
- ]) {
- let mut wrt = WebgraphWriter::new(
- &temp_dir.as_ref().join(format!("test_{}", i)),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
- graph.optimize_read();
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 4
- );
-
- assert_eq!(
- graph
- .ingoing_edges(Node::from("B"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
-
- assert_eq!(
- graph
- .ingoing_edges(Node::from("C"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
-
- assert_eq!(
- graph
- .ingoing_edges(Node::from("D"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
- }
-
- #[test]
- fn merge_reverse_star() {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- for (i, (from, to, label)) in (0..).zip(&[
- (Node::from("B"), Node::from("A"), String::new()),
- (Node::from("C"), Node::from("A"), String::new()),
- (Node::from("D"), Node::from("A"), String::new()),
- (Node::from("E"), Node::from("A"), String::new()),
- ]) {
- let mut wrt = WebgraphWriter::new(
- &temp_dir.as_ref().join(format!("test_{}", i)),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
- graph.optimize_read();
-
- assert_eq!(
- graph
- .ingoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 4
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("D"), EdgeLimit::Unlimited)
- .len(),
- 1
- );
- }
-
- proptest! {
- #[test]
- fn prop_merge(
- nodes in
- proptest::collection::vec(
- ("[a-z]", "[a-z]"), 0..100
- )
- ) {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut wrt = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- for (from, to) in nodes.clone() {
- wrt.insert(Node::new_for_test(from.as_str()), Node::new_for_test(to.as_str()), String::new(), RelFlags::default());
-
- if rand::random::<usize>() % 10 == 0 {
- graphs.push(wrt.finalize());
- wrt = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- }
- }
-
- if graphs.is_empty() {
- return Ok(());
- }
-
- graphs.push(wrt.finalize());
-
- let mut graph = graphs.pop().unwrap();
-
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
-
- for (from, to) in nodes {
- if from == to {
- continue;
- }
-
- let from = Node::new_for_test(from.as_str());
- let to = Node::new_for_test(to.as_str());
-
- let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
- let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
-
- prop_assert!(outgoing.iter().any(|e| e.to == to));
- prop_assert!(ingoing.iter().any(|e| e.from == from));
- }
- }
- }
-
- fn proptest_case(nodes: &[(&str, &str)]) {
- let mut graphs = Vec::new();
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut wrt = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
-
- for (i, (from, to)) in nodes.iter().enumerate() {
- wrt.insert(
- Node::new_for_test(from),
- Node::new_for_test(to),
- String::new(),
- RelFlags::default(),
- );
-
- if i % 2 == 0 {
- graphs.push(wrt.finalize());
- wrt = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- }
- }
-
- graphs.push(wrt.finalize());
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- graph.merge_all_segments(Compression::default()).unwrap();
-
- for (from, to) in nodes {
- if from == to {
- continue;
- }
-
- let from = Node::new_for_test(from);
- let to = Node::new_for_test(to);
-
- let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
- let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
-
- assert!(outgoing.iter().any(|e| e.to == to));
- assert!(ingoing.iter().any(|e| e.from == from));
- }
- }
-
- #[test]
- fn merge_proptest_case1() {
- let nodes = [("k", "d"), ("k", "t"), ("y", "m")];
- proptest_case(&nodes);
- }
-
- #[test]
- fn merge_proptest_case2() {
- let nodes = [("i", "k"), ("k", "g"), ("y", "m"), ("q", "r"), ("e", "g")];
-
- proptest_case(&nodes);
- }
-
- #[test]
- fn merge_proptest_case3() {
- let nodes = [("h", "c"), ("r", "r")];
-
- proptest_case(&nodes);
- }
-
- #[test]
- fn node_lowercase_name() {
- let n = Node::from("TEST".to_string());
- assert_eq!(n.as_str(), "test");
- }
-
- #[test]
- fn host_node_cleanup() {
- let n = Node::from("https://www.example.com?test").into_host();
- assert_eq!(n.as_str(), "example.com");
- }
-
- #[test]
- fn remove_protocol() {
- let n = Node::from("https://www.example.com/?test");
-
- assert_eq!(n.as_str(), "example.com/?test=");
- }
-
- #[test]
- fn cap_label_length() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
-
- writer.insert(
- Node::from("A"),
- Node::from("B"),
- "a".repeat(MAX_LABEL_LENGTH + 1),
- RelFlags::default(),
- );
-
- let graph = writer.finalize();
-
- assert_eq!(graph.segments.len(), 1);
- assert_eq!(
- graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)[0].label,
- "a".repeat(MAX_LABEL_LENGTH)
- );
- }
-
- #[test]
- fn test_edge_limits() {
- let (graph, temp_dir) = test_graph();
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 2
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
- .len(),
- 1
- );
-
- let mut graphs = Vec::new();
- for (from, to, label) in &[
- (Node::from("A"), Node::from("B"), String::new()),
- (Node::from("A"), Node::from("C"), String::new()),
- ] {
- let mut wrt = WebgraphWriter::new(
- temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
- Executor::single_thread(),
- Compression::default(),
- None,
- );
- wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());
- graphs.push(wrt.finalize());
- }
-
- let mut graph = graphs.pop().unwrap();
-
- for other in graphs {
- graph.merge(other).unwrap();
- }
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 2
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
- .len(),
- 1
- );
-
- graph.optimize_read();
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
- .len(),
- 2
- );
-
- assert_eq!(
- graph
- .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
- .len(),
- 1
- );
- }
-
- #[test]
- fn test_node_normalized() {
- let n = Node::from("http://www.example.com/abc");
- assert_eq!(n.as_str(), "example.com/abc");
-
- let n = Node::from("http://www.example.com/abc#123");
- assert_eq!(n.as_str(), "example.com/abc");
- }
-
- #[test]
- fn test_rel_flags() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut writer = WebgraphWriter::new(
- &temp_dir,
- Executor::single_thread(),
- Compression::default(),
- None,
- );
-
- writer.insert(
- Node::from("A"),
- Node::from("B"),
- String::new(),
- RelFlags::IS_IN_FOOTER | RelFlags::TAG,
- );
-
- let graph = writer.finalize();
-
- assert_eq!(
- graph.raw_outgoing_edges(&Node::from("A").id(), EdgeLimit::Unlimited)[0].rel,
- RelFlags::IS_IN_FOOTER | RelFlags::TAG,
- );
- }
-
- #[test]
- fn test_limit_and_offset() {
- let (graph, _temp_dir) = test_graph();
-
- let no_offset = graph.raw_outgoing_edges(
- &Node::from("A").id(),
- EdgeLimit::LimitAndOffset {
- limit: 2,
- offset: 0,
- },
- );
- assert_eq!(no_offset.len(), 2);
-
- let edges = graph.raw_outgoing_edges(
- &Node::from("A").id(),
- EdgeLimit::LimitAndOffset {
- limit: 2,
- offset: 1,
- },
- );
- assert_eq!(edges.len(), 1);
- assert_eq!(edges[0].to, no_offset[1].to);
+ /// Iterate all host edges in the graph at least once.
+ /// Some edges may be returned multiple times.
+ /// This happens if they are present in more than one segment.
+ pub fn host_edges(&self) -> impl Iterator<Item = SmallEdge> + '_ {
+ self.store.iter_hosts_small()
}
}
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index d37ebee9..ac24d151 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -126,6 +126,19 @@ impl Node {
}
}
+ pub fn empty() -> Self {
+ Node {
+ name: String::new(),
+ }
+ }
+
+ /// Dangerous! No validation is done on the input string.
+ pub fn from_str_not_validated(name: &str) -> Self {
+ Self {
+ name: name.to_string(),
+ }
+ }
+
pub fn into_host(self) -> Node {
let url = if self.name.contains("://") {
Url::parse(&self.name)
@@ -207,64 +220,5 @@ pub fn normalize_url(url: &Url) -> String {
normalized = prefix.to_string();
}
- normalized
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct NodeDatum {
- id: NodeID,
- host_rank: u64,
-}
-
-impl PartialOrd for NodeDatum {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl Ord for NodeDatum {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.host_rank
- .cmp(&other.host_rank)
- .then_with(|| self.id.cmp(&other.id))
- }
-}
-
-impl PartialEq for NodeDatum {
- fn eq(&self, other: &Self) -> bool {
- self.id == other.id && self.host_rank == other.host_rank
- }
-}
-
-impl Eq for NodeDatum {}
-
-impl std::hash::Hash for NodeDatum {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.id.hash(state);
- self.host_rank.hash(state);
- }
-}
-
-impl NodeDatum {
- pub fn new<N>(node: N, host_rank: u64) -> Self
- where
- N: Into<NodeID>,
- {
- Self {
- id: node.into(),
- host_rank,
- }
- }
-}
-
-impl NodeDatum {
- #[inline]
- pub fn node(&self) -> NodeID {
- self.id
- }
-
- #[inline]
- pub fn host_rank(&self) -> u64 {
- self.host_rank
- }
+ normalized.to_lowercase()
}
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
new file mode 100644
index 00000000..f7007cf6
--- /dev/null
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -0,0 +1,500 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use itertools::Itertools;
+use tantivy::query::ShortCircuitQuery;
+
+use super::{
+ collector::TopDocsCollector,
+ raw::{HostLinksQuery, LinksQuery},
+ Query,
+};
+use crate::{
+ ampc::dht::ShardId,
+ webgraph::{
+ doc_address::DocAddress,
+ document::Edge,
+ schema::{Field, FromHostId, FromId, RelFlags, ToHostId, ToId},
+ searcher::Searcher,
+ EdgeLimit, Node, NodeID, SmallEdge, SmallEdgeWithLabel,
+ },
+ Result,
+};
+
+pub fn fetch_small_edges<F: Field>(
+ searcher: &Searcher,
+ mut doc_ids: Vec<DocAddress>,
+ node_id_field: F,
+) -> Result<Vec<(NodeID, crate::webpage::RelFlags)>> {
+ doc_ids.sort_unstable_by_key(|doc| doc.segment_ord);
+ let mut prev_segment_id = None;
+ let mut field_column = None;
+ let mut rel_flags_column = None;
+
+ let mut edges = Vec::with_capacity(doc_ids.len());
+
+ for doc in doc_ids {
+ if Some(doc.segment_ord) != prev_segment_id {
+ prev_segment_id = Some(doc.segment_ord);
+ let segment_reader = searcher.tantivy_searcher().segment_reader(doc.segment_ord);
+ field_column = Some(
+ segment_reader
+ .column_fields()
+ .u64(node_id_field.name())
+ .unwrap(),
+ );
+ rel_flags_column = Some(segment_reader.column_fields().u64(RelFlags.name()).unwrap());
+ }
+
+ let Some(id) = field_column.as_ref().unwrap().first(doc.doc_id) else {
+ continue;
+ };
+ let Some(rel_flags) = rel_flags_column.as_ref().unwrap().first(doc.doc_id) else {
+ continue;
+ };
+
+ edges.push((NodeID::from(id), crate::webpage::RelFlags::from(rel_flags)));
+ }
+
+ Ok(edges)
+}
+
+pub fn fetch_edges(searcher: &Searcher, mut doc_ids: Vec<DocAddress>) -> Result<Vec<Edge>> {
+ doc_ids.sort_unstable_by_key(|doc| doc.segment_ord);
+
+ let mut edges = Vec::with_capacity(doc_ids.len());
+
+ for doc in doc_ids {
+ edges.push(searcher.tantivy_searcher().doc(doc.into())?);
+ }
+
+ Ok(edges)
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct BacklinksQuery {
+ node: NodeID,
+ limit: EdgeLimit,
+}
+
+impl BacklinksQuery {
+ pub fn new(node: NodeID) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for BacklinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type Output = Vec<SmallEdge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, ToId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(
+ ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, ToId)), limit as u64),
+ ),
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let nodes = fetch_small_edges(searcher, docs, FromId)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
+ from: node,
+ to: self.node,
+ rel_flags,
+ }))
+ .collect())
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct HostBacklinksQuery {
+ node: NodeID,
+ limit: EdgeLimit,
+}
+
+impl HostBacklinksQuery {
+ pub fn new(node: NodeID) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(mut self, limit: EdgeLimit) -> Self {
+ self.limit = limit;
+ self
+ }
+}
+
+impl Query for HostBacklinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type Output = Vec<SmallEdge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(self.node, ToHostId, FromHostId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(self.node, ToHostId, FromHostId)),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let nodes = fetch_small_edges(searcher, docs, FromHostId)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
+ from: node,
+ to: self.node,
+ rel_flags,
+ }))
+ .collect())
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct FullBacklinksQuery {
+ node: Node,
+ limit: EdgeLimit,
+}
+
+impl FullBacklinksQuery {
+ pub fn new(node: Node) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for FullBacklinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, Edge)>;
+ type Output = Vec<Edge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), ToId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), ToId)),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(scores.into_iter().zip_eq(edges).collect())
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct FullHostBacklinksQuery {
+ node: Node,
+ limit: EdgeLimit,
+}
+
+impl FullHostBacklinksQuery {
+ pub fn new(node: Node) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for FullHostBacklinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, Edge)>;
+ type Output = Vec<Edge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ ToHostId,
+ FromHostId,
+ )),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ ToHostId,
+ FromHostId,
+ )),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(edges.into_iter().map(|e| Edge {
+ from: e.from.into_host(),
+ to: e.to.into_host(),
+ rel_flags: e.rel_flags,
+ label: e.label,
+ sort_score: e.sort_score,
+ }))
+ .collect())
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct BacklinksWithLabelsQuery {
+ node: NodeID,
+ limit: EdgeLimit,
+}
+
+impl BacklinksWithLabelsQuery {
+ pub fn new(node: NodeID) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for BacklinksWithLabelsQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, SmallEdgeWithLabel)>;
+ type Output = Vec<SmallEdgeWithLabel>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ Box::new(LinksQuery::new(self.node, ToId))
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(edges.into_iter().map(|e| SmallEdgeWithLabel {
+ from: e.from.id(),
+ to: e.to.id(),
+ rel_flags: e.rel_flags,
+ label: e.label,
+ }))
+ .collect())
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/fast_count.rs b/crates/core/src/webgraph/query/collector/fast_count.rs
new file mode 100644
index 00000000..3a6ec69d
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/fast_count.rs
@@ -0,0 +1,108 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rustc_hash::FxHashMap;
+
+use crate::ampc::dht::ShardId;
+
+use super::Collector;
+
+pub enum FastCountValue {
+ U64(u64),
+}
+
+impl FastCountValue {
+ fn as_term(&self, field: tantivy::schema::Field) -> tantivy::schema::Term {
+ match self {
+ FastCountValue::U64(value) => tantivy::schema::Term::from_field_u64(field, *value),
+ }
+ }
+}
+
+pub struct FastCountCollector {
+ field_name: String,
+ value: FastCountValue,
+ shard_id: Option<ShardId>,
+}
+
+impl FastCountCollector {
+ pub fn new(field_name: String, value: FastCountValue) -> Self {
+ Self {
+ field_name,
+ value,
+ shard_id: None,
+ }
+ }
+
+ pub fn with_shard_id(mut self, shard_id: ShardId) -> Self {
+ self.shard_id = Some(shard_id);
+ self
+ }
+}
+
+impl Collector for FastCountCollector {
+ type Fruit = FxHashMap<ShardId, u64>;
+
+ type Child = FastSegmentCountCollector;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ let field = segment.schema().get_field(&self.field_name)?;
+ let term = self.value.as_term(field);
+
+ let num_docs = segment
+ .inverted_index(field)?
+ .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
+ .map(|postings| postings.doc_freq() as u64)
+ .unwrap_or(0);
+
+ Ok(FastSegmentCountCollector {
+ shard_id: self.shard_id.unwrap(),
+ count: num_docs,
+ })
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut map = FxHashMap::default();
+ for result in segment_fruits {
+ for (shard_id, count) in result {
+ *map.entry(shard_id).or_insert(0) += count;
+ }
+ }
+ Ok(map)
+ }
+}
+
+pub struct FastSegmentCountCollector {
+ shard_id: ShardId,
+ count: u64,
+}
+
+impl tantivy::collector::SegmentCollector for FastSegmentCountCollector {
+ type Fruit = FxHashMap<ShardId, u64>;
+
+ fn collect(&mut self, _: tantivy::DocId, _: tantivy::Score) {}
+
+ fn harvest(self) -> Self::Fruit {
+ FxHashMap::from_iter(vec![(self.shard_id, self.count)])
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/first_doc.rs b/crates/core/src/webgraph/query/collector/first_doc.rs
new file mode 100644
index 00000000..0f4a0a08
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/first_doc.rs
@@ -0,0 +1,89 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};
+
+use super::Collector;
+use crate::{ampc::dht::ShardId, webgraph::doc_address::DocAddress};
+
+pub struct FirstDocCollector {
+ shard_id: Option<ShardId>,
+}
+
+impl FirstDocCollector {
+ pub fn with_shard_id(shard_id: ShardId) -> Self {
+ Self {
+ shard_id: Some(shard_id),
+ }
+ }
+
+ pub fn without_shard_id() -> Self {
+ Self { shard_id: None }
+ }
+}
+
+impl Collector for FirstDocCollector {
+ type Fruit = Option<DocAddress>;
+ type Child = FirstDocSegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_id: SegmentOrdinal,
+ _: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ Ok(FirstDocSegmentCollector::new(
+ segment_id,
+ self.shard_id.unwrap(),
+ ))
+ }
+
+ fn merge_fruits(
+ &self,
+ fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ Ok(fruits.into_iter().flatten().next())
+ }
+}
+
+pub struct FirstDocSegmentCollector {
+ first_doc: Option<DocAddress>,
+ segment_id: SegmentOrdinal,
+ shard_id: ShardId,
+}
+
+impl FirstDocSegmentCollector {
+ pub fn new(segment_id: SegmentOrdinal, shard_id: ShardId) -> Self {
+ Self {
+ first_doc: None,
+ segment_id,
+ shard_id,
+ }
+ }
+}
+
+impl SegmentCollector for FirstDocSegmentCollector {
+ type Fruit = Option<DocAddress>;
+
+ fn collect(&mut self, doc: DocId, _: tantivy::Score) {
+ if self.first_doc.is_none() {
+ self.first_doc = Some(DocAddress::new(self.shard_id, self.segment_id, doc));
+ }
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.first_doc
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rs
new file mode 100644
index 00000000..b1478590
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/mod.rs
@@ -0,0 +1,111 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod fast_count;
+mod first_doc;
+mod top_docs;
+
+pub use fast_count::{FastCountCollector, FastCountValue};
+pub use first_doc::FirstDocCollector;
+pub use top_docs::TopDocsCollector;
+
+use tantivy::{
+ collector::{Fruit, SegmentCollector},
+ query::Weight,
+ SegmentOrdinal, SegmentReader,
+};
+
+pub trait Collector: Sync + Send {
+ type Fruit: Fruit;
+ type Child: SegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_local_id: SegmentOrdinal,
+ segment: &SegmentReader,
+ ) -> crate::Result<Self::Child>;
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit>;
+
+ fn collect_segment(
+ &self,
+ weight: &dyn Weight,
+ segment_ord: u32,
+ reader: &SegmentReader,
+ ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
+ let mut segment_collector = self.for_segment(segment_ord, reader)?;
+
+ weight.for_each_no_score(reader, &mut |docs| {
+ segment_collector.collect_block(docs);
+ })?;
+
+ Ok(segment_collector.harvest())
+ }
+}
+
+pub struct TantivyCollector<'a, T>(&'a T);
+
+impl<'a, T> From<&'a T> for TantivyCollector<'a, T>
+where
+ T: Collector,
+{
+ fn from(collector: &'a T) -> Self {
+ TantivyCollector(collector)
+ }
+}
+
+impl<'a, T> tantivy::collector::Collector for TantivyCollector<'a, T>
+where
+ T: Collector,
+{
+ type Fruit = T::Fruit;
+
+ type Child = T::Child;
+
+ fn for_segment(
+ &self,
+ segment_local_id: SegmentOrdinal,
+ segment: &SegmentReader,
+ ) -> tantivy::Result<Self::Child> {
+ T::for_segment(self.0, segment_local_id, segment)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+
+ fn requires_scoring(&self) -> bool {
+ false
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> tantivy::Result<Self::Fruit> {
+ T::merge_fruits(self.0, segment_fruits)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+
+ fn collect_segment(
+ &self,
+ weight: &dyn tantivy::query::Weight,
+ segment_ord: u32,
+ reader: &SegmentReader,
+ ) -> tantivy::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {
+ T::collect_segment(self.0, weight, segment_ord, reader)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
new file mode 100644
index 00000000..16789523
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -0,0 +1,226 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::marker::PhantomData;
+
+use tantivy::{
+ collector::{SegmentCollector, TopNComputer},
+ DocId, SegmentOrdinal,
+};
+
+use crate::distributed::member::ShardId;
+use crate::webgraph::{
+ doc_address::DocAddress,
+ query::document_scorer::{DefaultDocumentScorer, DocumentScorer},
+ EdgeLimit,
+};
+
+use super::Collector;
+
+pub struct TopDocsCollector<S: DocumentScorer = DefaultDocumentScorer> {
+ shard_id: Option<ShardId>,
+ limit: Option<usize>,
+ offset: Option<usize>,
+ perform_offset: bool,
+ _phantom: PhantomData<S>,
+}
+
+impl<S: DocumentScorer> From<EdgeLimit> for TopDocsCollector<S> {
+ fn from(limit: EdgeLimit) -> Self {
+ let mut collector = TopDocsCollector::new().disable_offset();
+
+ match limit {
+ EdgeLimit::Unlimited => {}
+ EdgeLimit::Limit(limit) => collector = collector.with_limit(limit),
+ EdgeLimit::LimitAndOffset { limit, offset } => {
+ collector = collector.with_limit(limit);
+ collector = collector.with_offset(offset);
+ }
+ }
+
+ collector
+ }
+}
+
+impl<S: DocumentScorer> Default for TopDocsCollector<S> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<S: DocumentScorer> TopDocsCollector<S> {
+ pub fn new() -> Self {
+ Self {
+ shard_id: None,
+ limit: None,
+ offset: None,
+ perform_offset: true,
+ _phantom: PhantomData,
+ }
+ }
+
+ pub fn with_shard_id(self, shard_id: ShardId) -> Self {
+ Self {
+ shard_id: Some(shard_id),
+ ..self
+ }
+ }
+
+ pub fn with_offset(self, offset: usize) -> Self {
+ Self {
+ offset: Some(offset),
+ ..self
+ }
+ }
+
+ pub fn with_limit(self, limit: usize) -> Self {
+ Self {
+ limit: Some(limit),
+ ..self
+ }
+ }
+
+ pub fn enable_offset(self) -> Self {
+ Self {
+ perform_offset: true,
+ ..self
+ }
+ }
+
+ pub fn disable_offset(self) -> Self {
+ Self {
+ perform_offset: false,
+ ..self
+ }
+ }
+
+ fn computer(&self) -> Computer {
+ match (self.offset, self.limit) {
+ (Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),
+ (Some(_), None) => Computer::All(AllComputer::new()),
+ (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit)),
+ (None, None) => Computer::All(AllComputer::new()),
+ }
+ }
+}
+
+impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {
+ type Fruit = Vec<(tantivy::Score, DocAddress)>;
+
+ type Child = TopDocsSegmentCollector<S>;
+
+ fn for_segment(
+ &self,
+ segment_ord: SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ let scorer = S::for_segment(segment)?;
+
+ Ok(TopDocsSegmentCollector {
+ shard_id: self.shard_id.unwrap(),
+ computer: self.computer(),
+ segment_ord,
+ scorer,
+ })
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut computer = self.computer();
+
+ for fruit in segment_fruits {
+ for (score, doc) in fruit {
+ computer.push(score, doc);
+ }
+ }
+
+ let mut result = computer.harvest();
+
+ if self.perform_offset {
+ result = result.into_iter().skip(self.offset.unwrap_or(0)).collect();
+ }
+
+ Ok(result)
+ }
+}
+
+enum Computer {
+ TopN(TopNComputer<tantivy::Score, DocAddress>),
+ All(AllComputer),
+}
+
+impl Computer {
+ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+ match self {
+ Computer::TopN(computer) => computer.push(score, doc),
+ Computer::All(computer) => computer.push(score, doc),
+ }
+ }
+
+ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ match self {
+ Computer::TopN(computer) => computer
+ .into_sorted_vec()
+ .into_iter()
+ .map(|comparable_doc| (comparable_doc.feature, comparable_doc.doc))
+ .collect(),
+ Computer::All(computer) => computer.harvest(),
+ }
+ }
+}
+
+struct AllComputer {
+ docs: Vec<(tantivy::Score, DocAddress)>,
+}
+
+impl AllComputer {
+ fn new() -> Self {
+ Self { docs: Vec::new() }
+ }
+
+ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+ self.docs.push((score, doc));
+ }
+
+ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ let mut docs = self.docs;
+ docs.sort_by(|(score1, _), (score2, _)| score2.total_cmp(score1));
+ docs
+ }
+}
+
+pub struct TopDocsSegmentCollector<S: DocumentScorer> {
+ shard_id: ShardId,
+ computer: Computer,
+ segment_ord: SegmentOrdinal,
+ scorer: S,
+}
+
+impl<S: DocumentScorer + 'static> SegmentCollector for TopDocsSegmentCollector<S> {
+ type Fruit = Vec<(tantivy::Score, DocAddress)>;
+
+ fn collect(&mut self, doc: DocId, _: tantivy::Score) {
+ let score = self.scorer.score(doc);
+ self.computer
+ .push(score, DocAddress::new(self.shard_id, self.segment_ord, doc));
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.computer.harvest()
+ }
+}
diff --git a/crates/core/src/webgraph/query/degree.rs b/crates/core/src/webgraph/query/degree.rs
new file mode 100644
index 00000000..7950bdda
--- /dev/null
+++ b/crates/core/src/webgraph/query/degree.rs
@@ -0,0 +1,91 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rustc_hash::FxHashMap;
+
+use crate::{
+ ampc::dht::ShardId,
+ webgraph::{
+ schema::{Field, ToId},
+ searcher::Searcher,
+ NodeID,
+ },
+};
+
+use super::{
+ collector::{FastCountCollector, FastCountValue},
+ raw, Query,
+};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct InDegreeQuery {
+ node: NodeID,
+}
+
+impl Query for InDegreeQuery {
+ type Collector = FastCountCollector;
+ type TantivyQuery = raw::DummyQuery;
+ type IntermediateOutput = FxHashMap<ShardId, u64>;
+ type Output = u64;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ raw::DummyQuery
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ FastCountCollector::new(
+ ToId.name().to_string(),
+ FastCountValue::U64(self.node.as_u64()),
+ )
+ .with_shard_id(shard_id)
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ FastCountCollector::new(
+ ToId.name().to_string(),
+ FastCountValue::U64(self.node.as_u64()),
+ )
+ }
+
+ fn retrieve(
+ &self,
+ _: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ Ok(fruit)
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(fruit_shard_id, _)| shard_id == *fruit_shard_id)
+ .collect()
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut map = FxHashMap::default();
+ for result in results {
+ for (shard_id, count) in result {
+ *map.entry(shard_id).or_insert(0) += count;
+ }
+ }
+ map.into_values().sum()
+ }
+}
diff --git a/crates/core/src/webgraph/query/document_scorer/mod.rs b/crates/core/src/webgraph/query/document_scorer/mod.rs
new file mode 100644
index 00000000..661093e3
--- /dev/null
+++ b/crates/core/src/webgraph/query/document_scorer/mod.rs
@@ -0,0 +1,39 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{columnar::Column, DocId, Score, SegmentReader};
+
+use crate::webgraph::schema::{Field, SortScore};
+
+pub trait DocumentScorer: Send + Sync + Sized {
+ fn for_segment(segment: &SegmentReader) -> tantivy::Result<Self>;
+ fn score(&self, doc: DocId) -> Score;
+}
+
+pub struct DefaultDocumentScorer {
+ column: Column<f64>,
+}
+
+impl DocumentScorer for DefaultDocumentScorer {
+ fn for_segment(segment: &SegmentReader) -> tantivy::Result<Self> {
+ let column = segment.column_fields().f64(SortScore.name())?;
+ Ok(Self { column })
+ }
+
+ fn score(&self, doc: DocId) -> Score {
+ self.column.first(doc).unwrap_or(0.0) as Score
+ }
+}
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
new file mode 100644
index 00000000..accf96f2
--- /dev/null
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -0,0 +1,374 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::{
+ backlink::{fetch_edges, fetch_small_edges},
+ collector::TopDocsCollector,
+ raw::{host_links::HostLinksQuery, links::LinksQuery},
+ Query,
+};
+use crate::{
+ ampc::dht::ShardId,
+ webgraph::{
+ document::Edge,
+ schema::{FromHostId, FromId, ToHostId, ToId},
+ EdgeLimit, Node, NodeID, Searcher, SmallEdge,
+ },
+ Result,
+};
+
+use itertools::Itertools;
+use tantivy::query::ShortCircuitQuery;
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct ForwardlinksQuery {
+ node: NodeID,
+ limit: EdgeLimit,
+}
+
+impl ForwardlinksQuery {
+ pub fn new(node: NodeID) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for ForwardlinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type Output = Vec<SmallEdge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, FromId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => Box::new(
+ ShortCircuitQuery::new(Box::new(LinksQuery::new(self.node, FromId)), limit as u64),
+ ),
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let nodes = fetch_small_edges(searcher, docs, ToId)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
+ from: self.node,
+ to: node,
+ rel_flags,
+ }))
+ .collect())
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct HostForwardlinksQuery {
+ node: NodeID,
+ limit: EdgeLimit,
+}
+
+impl HostForwardlinksQuery {
+ pub fn new(node: NodeID) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for HostForwardlinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, SmallEdge)>;
+ type Output = Vec<SmallEdge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(self.node, FromHostId, ToHostId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(self.node, FromHostId, ToHostId)),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let nodes = fetch_small_edges(searcher, docs, ToHostId)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(nodes.into_iter().map(|(node, rel_flags)| SmallEdge {
+ from: self.node,
+ to: node,
+ rel_flags,
+ }))
+ .collect())
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct FullForwardlinksQuery {
+ node: Node,
+ limit: EdgeLimit,
+}
+
+impl FullForwardlinksQuery {
+ pub fn new(node: Node) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for FullForwardlinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, Edge)>;
+ type Output = Vec<Edge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), FromId)),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(LinksQuery::new(self.node.id(), FromId)),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(scores.into_iter().zip_eq(edges).collect())
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct FullHostForwardlinksQuery {
+ node: Node,
+ limit: EdgeLimit,
+}
+
+impl FullHostForwardlinksQuery {
+ pub fn new(node: Node) -> Self {
+ Self {
+ node,
+ limit: EdgeLimit::Unlimited,
+ }
+ }
+
+ pub fn with_limit(self, limit: EdgeLimit) -> Self {
+ Self {
+ node: self.node,
+ limit,
+ }
+ }
+}
+
+impl Query for FullHostForwardlinksQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = Vec<(f32, Edge)>;
+ type Output = Vec<Edge>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self.limit {
+ EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ ToHostId,
+ )),
+ EdgeLimit::Limit(limit) | EdgeLimit::LimitAndOffset { limit, .. } => {
+ Box::new(ShortCircuitQuery::new(
+ Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ ToHostId,
+ )),
+ limit as u64,
+ ))
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ TopDocsCollector::from(self.limit)
+ .with_shard_id(shard_id)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopDocsCollector::from(self.limit).enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, doc)| doc.shard_id == shard)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput> {
+ let (scores, docs): (Vec<_>, Vec<_>) = fruit.into_iter().unzip();
+ let edges = fetch_edges(searcher, docs)?;
+ Ok(scores
+ .into_iter()
+ .zip_eq(edges.into_iter().map(|e| Edge {
+ from: e.from.into_host(),
+ to: e.to.into_host(),
+ rel_flags: e.rel_flags,
+ label: e.label,
+ sort_score: e.sort_score,
+ }))
+ .collect())
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut edges: Vec<_> = results.into_iter().flatten().collect();
+ edges.sort_by(|(a, _), (b, _)| b.total_cmp(a));
+ edges.into_iter().map(|(_, e)| e).collect()
+ }
+}
diff --git a/crates/core/src/webgraph/query/id2node.rs b/crates/core/src/webgraph/query/id2node.rs
new file mode 100644
index 00000000..ffac2876
--- /dev/null
+++ b/crates/core/src/webgraph/query/id2node.rs
@@ -0,0 +1,104 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::collector::FirstDocCollector;
+use super::Query;
+use crate::ampc::dht::ShardId;
+use crate::webgraph::query::raw;
+use crate::webgraph::schema::{FromHostId, FromId, ToHostId, ToId};
+use crate::webgraph::searcher::Searcher;
+use crate::webgraph::{Edge, Node, NodeID};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub enum Id2NodeQuery {
+ Page(NodeID),
+ Host(NodeID),
+}
+
+impl Query for Id2NodeQuery {
+ type Collector = FirstDocCollector;
+ type TantivyQuery = raw::Id2NodeQuery;
+ type IntermediateOutput = Option<Node>;
+ type Output = Option<Node>;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery {
+ match self {
+ Self::Page(node) => raw::Id2NodeQuery::new(*node, vec![ToId.into(), FromId.into()]),
+ Self::Host(node) => {
+ raw::Id2NodeQuery::new(*node, vec![ToHostId.into(), FromHostId.into()])
+ }
+ }
+ }
+
+ fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ FirstDocCollector::with_shard_id(shard_id)
+ }
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as super::collector::Collector>::Fruit,
+ ) -> crate::Result<Self::Output> {
+ Ok(fruit.and_then(|doc| {
+ searcher
+ .tantivy_searcher()
+ .doc::<Edge>(doc.into())
+ .ok()
+ .and_then(|e| match self {
+ Self::Page(node) => {
+ if e.to.id() == *node {
+ Some(e.to)
+ } else if e.from.id() == *node {
+ Some(e.from)
+ } else {
+ None
+ }
+ }
+ Self::Host(node) => {
+ let to = e.to.into_host();
+ let from = e.from.into_host();
+
+ if to.id() == *node {
+ Some(to)
+ } else if from.id() == *node {
+ Some(from)
+ } else {
+ None
+ }
+ }
+ })
+ }))
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ FirstDocCollector::without_shard_id()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ match fruit {
+ Some(doc) if doc.shard_id == shard_id => fruit,
+ _ => None,
+ }
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ results.into_iter().flatten().next()
+ }
+}
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
new file mode 100644
index 00000000..6b70cc47
--- /dev/null
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -0,0 +1,58 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::searcher::Searcher;
+use crate::{ampc::dht::ShardId, Result};
+pub use collector::Collector;
+
+pub mod backlink;
+pub use backlink::*;
+
+pub mod forwardlink;
+pub use forwardlink::*;
+
+pub mod id2node;
+pub use id2node::*;
+
+pub mod collector;
+mod degree;
+mod document_scorer;
+mod raw;
+
+pub trait Query: Send + Sync + bincode::Encode + bincode::Decode + Clone {
+ type Collector: Collector;
+ type TantivyQuery: tantivy::query::Query;
+ type IntermediateOutput;
+ type Output;
+
+ fn tantivy_query(&self) -> Self::TantivyQuery;
+ fn collector(&self, shard_id: ShardId) -> Self::Collector;
+ fn remote_collector(&self) -> Self::Collector;
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> <Self::Collector as Collector>::Fruit;
+
+ fn retrieve(
+ &self,
+ searcher: &Searcher,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput>;
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output;
+}
diff --git a/crates/core/src/webgraph/query/raw/dummy.rs b/crates/core/src/webgraph/query/raw/dummy.rs
new file mode 100644
index 00000000..0a6b4594
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/dummy.rs
@@ -0,0 +1,72 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct DummyQuery;
+
+impl tantivy::query::Query for DummyQuery {
+ fn weight(
+ &self,
+ _: tantivy::query::EnableScoring<'_>,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ Ok(Box::new(DummyWeight))
+ }
+}
+
+struct DummyWeight;
+
+impl tantivy::query::Weight for DummyWeight {
+ fn scorer(
+ &self,
+ _: &tantivy::SegmentReader,
+ _: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ Ok(Box::new(DummyScorer))
+ }
+
+ fn explain(
+ &self,
+ _: &tantivy::SegmentReader,
+ _: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ Ok(tantivy::query::Explanation::new(
+ "dummy query that doesn't match any documents",
+ 0.0,
+ ))
+ }
+}
+
+struct DummyScorer;
+
+impl tantivy::query::Scorer for DummyScorer {
+ fn score(&mut self) -> tantivy::Score {
+ unreachable!()
+ }
+}
+
+impl tantivy::DocSet for DummyScorer {
+ fn advance(&mut self) -> tantivy::DocId {
+ tantivy::TERMINATED
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ tantivy::TERMINATED
+ }
+
+ fn size_hint(&self) -> u32 {
+ 0
+ }
+}
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rs
new file mode 100644
index 00000000..c50af494
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/host_links.rs
@@ -0,0 +1,162 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};
+
+use crate::webgraph::{
+ schema::{Field, FieldEnum},
+ NodeID,
+};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct HostLinksQuery {
+ node: NodeID,
+ field: FieldEnum,
+ deduplication_field: FieldEnum,
+}
+
+impl HostLinksQuery {
+ pub fn new<F: Field, FDedup: Field>(
+ node: NodeID,
+ lookup_field: F,
+ deduplication_field: FDedup,
+ ) -> Self {
+ Self {
+ node,
+ field: lookup_field.into(),
+ deduplication_field: deduplication_field.into(),
+ }
+ }
+}
+
+impl tantivy::query::Query for HostLinksQuery {
+ fn weight(
+ &self,
+ _: tantivy::query::EnableScoring<'_>,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ Ok(Box::new(HostLinksWeight {
+ node: self.node,
+ field: self.field,
+ deduplication_field: self.deduplication_field,
+ }))
+ }
+}
+
+struct HostLinksWeight {
+ node: NodeID,
+ field: FieldEnum,
+ deduplication_field: FieldEnum,
+}
+
+impl tantivy::query::Weight for HostLinksWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ _: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ let schema = reader.schema();
+ let field = schema.get_field(self.field.name())?;
+ let term = tantivy::Term::from_field_u64(field, self.node.as_u64());
+
+ match HostLinksScorer::new(reader, term, self.deduplication_field) {
+ Ok(Some(scorer)) => Ok(Box::new(scorer)),
+ _ => Ok(Box::new(tantivy::query::EmptyScorer)),
+ }
+ }
+
+ fn explain(
+ &self,
+ _: &tantivy::SegmentReader,
+ _: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ Ok(tantivy::query::Explanation::new("HostLinksWeight", 0.0))
+ }
+}
+
+struct HostLinksScorer {
+ postings: SegmentPostings,
+ host_id_column: Column<u64>,
+ last_host_id: Option<u64>,
+}
+
+impl HostLinksScorer {
+ fn new(
+ reader: &tantivy::SegmentReader,
+ term: tantivy::Term,
+ deduplication_field: FieldEnum,
+ ) -> tantivy::Result<Option<Self>> {
+ let host_id_column = reader.column_fields().u64(deduplication_field.name())?;
+
+ Ok(reader
+ .inverted_index(term.field())?
+ .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
+ .map(|postings| Self {
+ last_host_id: host_id_column.first(postings.doc()),
+ host_id_column,
+ postings,
+ }))
+ }
+}
+
+impl HostLinksScorer {
+ fn host_id(&self, doc: tantivy::DocId) -> Option<u64> {
+ if doc == tantivy::TERMINATED {
+ return None;
+ }
+ self.host_id_column.first(doc)
+ }
+}
+
+impl tantivy::query::Scorer for HostLinksScorer {
+ fn score(&mut self) -> tantivy::Score {
+ unimplemented!()
+ }
+}
+impl tantivy::DocSet for HostLinksScorer {
+ fn advance(&mut self) -> tantivy::DocId {
+ self.postings.advance();
+
+ while self.last_host_id
+ == self.host_id(
+ self.postings
+ .block_cursor()
+ .skip_reader()
+ .last_doc_in_block(),
+ )
+ && self.doc() != tantivy::TERMINATED
+ {
+ self.postings.mut_block_cursor().advance();
+ self.postings.reset_cursor_start_block();
+ }
+
+ while self.host_id(self.postings.doc()) == self.last_host_id
+ && self.doc() != tantivy::TERMINATED
+ {
+ self.postings.advance();
+ }
+
+ self.last_host_id = self.host_id(self.postings.doc());
+ self.postings.doc()
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ self.postings.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.postings.size_hint()
+ }
+}
diff --git a/crates/core/src/webgraph/query/raw/id2node.rs b/crates/core/src/webgraph/query/raw/id2node.rs
new file mode 100644
index 00000000..a21b0885
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/id2node.rs
@@ -0,0 +1,110 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use itertools::Itertools;
+use tantivy::{
+ postings::SegmentPostings,
+ query::{EmptyScorer, EnableScoring, Explanation, Query, Scorer, Weight},
+ DocId, DocSet, HasLen, Score, SegmentReader, Term,
+};
+
+use crate::webgraph::{
+ schema::{Field, FieldEnum},
+ NodeID,
+};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct Id2NodeQuery {
+ node: NodeID,
+ fields: Vec<FieldEnum>,
+}
+
+impl Id2NodeQuery {
+ pub fn new(node: NodeID, fields: Vec<FieldEnum>) -> Self {
+ Self { node, fields }
+ }
+}
+
+impl Query for Id2NodeQuery {
+ fn weight(&self, _: EnableScoring<'_>) -> tantivy::Result<Box<dyn Weight>> {
+ Ok(Box::new(Id2NodeWeight {
+ node: self.node,
+ fields: self.fields.clone(),
+ }))
+ }
+}
+
+struct Id2NodeWeight {
+ node: NodeID,
+ fields: Vec<FieldEnum>,
+}
+
+impl Weight for Id2NodeWeight {
+ fn scorer(&self, reader: &SegmentReader, _: Score) -> tantivy::Result<Box<dyn Scorer>> {
+ for field in self.fields.iter() {
+ let field = reader.schema().get_field(field.name())?;
+ let inverted_index = reader.inverted_index(field)?;
+
+ let term = Term::from_field_u64(field, self.node.as_u64());
+
+ if let Some(postings) =
+ inverted_index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
+ {
+ return Ok(Box::new(Id2NodeScorer { postings }));
+ }
+ }
+
+ Ok(Box::new(EmptyScorer))
+ }
+
+ fn explain(&self, _: &SegmentReader, _: DocId) -> tantivy::Result<Explanation> {
+ Ok(Explanation::new_with_string(
+ format!(
+ "Id2Node on [{}]",
+ self.fields.iter().map(|f| f.name()).join(", ")
+ ),
+ 0.0,
+ ))
+ }
+}
+
+struct Id2NodeScorer {
+ postings: SegmentPostings,
+}
+
+impl Scorer for Id2NodeScorer {
+ fn score(&mut self) -> tantivy::Score {
+ unimplemented!()
+ }
+}
+
+impl DocSet for Id2NodeScorer {
+ fn advance(&mut self) -> tantivy::DocId {
+ tantivy::TERMINATED
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ self.postings.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ if self.postings.is_empty() {
+ 0
+ } else {
+ 1
+ }
+ }
+}
diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rs
new file mode 100644
index 00000000..01288f4c
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/links.rs
@@ -0,0 +1,104 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{postings::SegmentPostings, query::EmptyScorer, Term};
+
+use crate::webgraph::{
+ schema::{Field, FieldEnum},
+ NodeID,
+};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct LinksQuery {
+ node: NodeID,
+ field: FieldEnum,
+}
+
+impl LinksQuery {
+ pub fn new<F: Field>(node: NodeID, field: F) -> Self {
+ Self {
+ node,
+ field: field.into(),
+ }
+ }
+}
+
+impl tantivy::query::Query for LinksQuery {
+ fn weight(
+ &self,
+ _: tantivy::query::EnableScoring<'_>,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ Ok(Box::new(LinksWeight {
+ node: self.node,
+ field: self.field,
+ }))
+ }
+}
+
+struct LinksWeight {
+ node: NodeID,
+ field: FieldEnum,
+}
+
+impl tantivy::query::Weight for LinksWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ _: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ let field = reader.schema().get_field(self.field.name())?;
+ let term = Term::from_field_u64(field, self.node.as_u64());
+
+ let index = reader.inverted_index(field)?;
+ match index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)? {
+ Some(postings) => Ok(Box::new(LinksScorer { postings })),
+ None => Ok(Box::new(EmptyScorer)),
+ }
+ }
+
+ fn explain(
+ &self,
+ _: &tantivy::SegmentReader,
+ _: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ let explanation = tantivy::query::Explanation::new("LinksQuery", 0.0);
+ Ok(explanation)
+ }
+}
+
+struct LinksScorer {
+ postings: SegmentPostings,
+}
+
+impl tantivy::query::Scorer for LinksScorer {
+ fn score(&mut self) -> tantivy::Score {
+ unimplemented!()
+ }
+}
+
+impl tantivy::DocSet for LinksScorer {
+ fn advance(&mut self) -> tantivy::DocId {
+ self.postings.advance()
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ self.postings.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.postings.size_hint()
+ }
+}
diff --git a/crates/core/src/webgraph/query/raw/mod.rs b/crates/core/src/webgraph/query/raw/mod.rs
new file mode 100644
index 00000000..5087efa2
--- /dev/null
+++ b/crates/core/src/webgraph/query/raw/mod.rs
@@ -0,0 +1,26 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod dummy;
+pub use dummy::DummyQuery;
+
+pub mod host_links;
+pub mod id2node;
+pub mod links;
+
+pub use host_links::HostLinksQuery;
+pub use id2node::Id2NodeQuery;
+pub use links::LinksQuery;
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index 0d62f865..0eefcef9 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -16,12 +16,12 @@
use std::sync::Arc;
+use futures::{stream::FuturesUnordered, StreamExt};
use itertools::Itertools;
use tokio::sync::Mutex;
use url::Url;
use crate::{
- config,
distributed::{
cluster::Cluster,
member::{Service, ShardId},
@@ -33,43 +33,17 @@ use crate::{
},
streaming_response::StreamingResponse,
},
- entrypoint::webgraph_server::{
- GetNode, GetNodeIDs, IngoingEdges, OutgoingEdges, RawIngoingEdges,
- RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,
- },
+ entrypoint::webgraph_server::{self, GetPageNodeIDs, Query, RetrieveReq, WebGraphService},
+ webgraph::Collector,
Result,
};
-use super::{Edge, EdgeLimit, FullEdge, Node, NodeID};
-
-struct WebgraphClientManager<G: WebgraphGranularity>(std::marker::PhantomData<G>);
-
-pub trait WebgraphGranularity: Clone {
- fn granularity() -> config::WebgraphGranularity;
-}
-
-#[derive(Clone)]
-pub struct Page;
-
-impl WebgraphGranularity for Page {
- fn granularity() -> config::WebgraphGranularity {
- config::WebgraphGranularity::Page
- }
-}
-
-#[derive(Clone)]
-pub struct Host;
+use super::{query::BacklinksQuery, EdgeLimit, Node, NodeID};
+use crate::webgraph;
-impl WebgraphGranularity for Host {
- fn granularity() -> config::WebgraphGranularity {
- config::WebgraphGranularity::Host
- }
-}
+struct WebgraphClientManager;
-impl<G> sonic::replication::ReusableClientManager for WebgraphClientManager<G>
-where
- G: WebgraphGranularity,
-{
+impl sonic::replication::ReusableClientManager for WebgraphClientManager {
const CLIENT_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60);
type Service = WebGraphService;
@@ -83,17 +57,8 @@ where
.await
.into_iter()
.filter_map(|member| {
- if let Service::Webgraph {
- host,
- shard,
- granularity,
- } = member.service
- {
- if granularity == G::granularity() {
- Some((shard, RemoteClient::<WebGraphService>::new(host)))
- } else {
- None
- }
+ if let Service::Webgraph { host, shard } = member.service {
+ Some((shard, RemoteClient::<WebGraphService>::new(host)))
} else {
None
}
@@ -113,12 +78,12 @@ where
}
#[derive(Clone)]
-pub struct RemoteWebgraph<G: WebgraphGranularity> {
- client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager<G>>>>,
+pub struct RemoteWebgraph {
+ client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager>>>,
cluster: Arc<Cluster>,
}
-impl<G: WebgraphGranularity> RemoteWebgraph<G> {
+impl RemoteWebgraph {
pub async fn new(cluster: Arc<Cluster>) -> Self {
Self {
client: Arc::new(Mutex::new(
@@ -129,21 +94,9 @@ impl<G: WebgraphGranularity> RemoteWebgraph<G> {
}
pub async fn await_ready(&self) {
- let granularity = G::granularity();
- tracing::info!("waiting for {granularity} webgraph to come online...");
+ tracing::info!("waiting for webgraph to come online...");
self.cluster
- .await_member(|member| {
- if let Service::Webgraph {
- host: _,
- shard: _,
- granularity: remote_granularity,
- } = member.service
- {
- granularity == remote_granularity
- } else {
- false
- }
- })
+ .await_member(|member| matches!(member.service, Service::Webgraph { .. }))
.await;
}
@@ -151,300 +104,255 @@ impl<G: WebgraphGranularity> RemoteWebgraph<G> {
self.client.lock().await.conn().await
}
- pub async fn knows(&self, mut host: String) -> Result<Option<Node>> {
- if let Some(suf) = host.strip_prefix("http://") {
- host = suf.to_string();
- }
- if let Some(suf) = host.strip_prefix("https://") {
- host = suf.to_string();
- }
-
- let url = Url::parse(&("http://".to_string() + host.as_str()))?;
- let node = Node::from(url).into_host();
- let id = node.id();
- let edges = self.raw_ingoing_edges(id, EdgeLimit::Limit(1)).await?;
-
- if !edges.is_empty() {
- Ok(Some(node))
- } else {
- Ok(None)
- }
- }
+ pub async fn search_initial<Q>(
+ &self,
+ query: &Q,
+ ) -> Result<<Q::Collector as webgraph::Collector>::Fruit>
+ where
+ Q: Query,
+ Result<
+ <Q::Collector as webgraph::Collector>::Fruit,
+ webgraph_server::EncodedError,
+ >: From<<Q as sonic::service::Message<WebGraphService>>::Response>,
+ <<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as webgraph::Collector>::Fruit>,
+ {
+ let collector = query.remote_collector();
- pub async fn get_node(&self, id: NodeID) -> Result<Option<Node>> {
let res = self
.conn()
.await
- .send(
- GetNode { node: id },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
+ .send(query.clone(), &AllShardsSelector, &RandomReplicaSelector)
.await?;
- Ok(res
+ let fruits: Vec<<<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit> = res
.into_iter()
.flatten()
- .flat_map(|(_, res)| res.into_iter().map(|(_, v)| v))
- .find(|n| n.is_some())
- .flatten()
- .clone())
- }
-
- pub async fn batch_get_node(&self, ids: &[NodeID]) -> Result<Vec<Option<Node>>> {
- let reqs = ids.iter().map(|&id| GetNode { node: id }).collect_vec();
-
- let res = self
- .conn()
- .await
- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
+ .flat_map(|(_, reps)| reps)
+ .filter_map(|(_, rep)| {
+ Result::<
+ <Q::Collector as webgraph::Collector>::Fruit,
+ webgraph_server::EncodedError,
+ >::from(rep)
+ .ok()
+ })
+ .map(|fruit| {
+ <<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit::from(fruit)
+ })
+ .collect();
- let mut nodes = vec![None; ids.len()];
+ collector
+ .merge_fruits(fruits)
+ .map_err(|_| anyhow::anyhow!("failed to merge fruits"))
+ }
- for (_, rep) in res {
- debug_assert!(rep.len() <= 1);
+ pub async fn retrieve<Q>(
+ &self,
+ query: Q,
+ fruit: <Q::Collector as webgraph::Collector>::Fruit,
+ ) -> Result<Vec<Q::IntermediateOutput>>
+ where
+ Q: Query,
+ <Q::Collector as webgraph::Collector>::Fruit: Clone,
+ <Q as webgraph_server::Query>::RetrieveReq: sonic::service::Wrapper<WebGraphService>,
+ Result<Q::IntermediateOutput, webgraph_server::EncodedError>: From<
+ <<Q as webgraph_server::Query>::RetrieveReq as sonic::service::Message<
+ WebGraphService,
+ >>::Response,
+ >,
+ {
+ let conn = self.conn().await;
+ let mut results = FuturesUnordered::new();
+ for shard in conn.shards() {
+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());
+ let req = Q::RetrieveReq::new(query.clone(), fruit);
+ results.push(shard.replicas().send(req, &RandomReplicaSelector));
+ }
+ let mut res = Vec::new();
- for (_, rep_nodes) in rep {
- for (i, node) in rep_nodes.into_iter().enumerate() {
- if let Some(node) = node {
- nodes[i] = Some(node);
- }
- }
+ while let Some(shard_res) = results.next().await {
+ if let Ok(shard_res) = shard_res {
+ res.push(shard_res);
}
}
- Ok(nodes)
- }
-
- pub async fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
- let res = self
- .conn()
- .await
- .send(
- IngoingEdges { node, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await?;
-
Ok(res
.into_iter()
.flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
+ .filter_map(|(_, res)| {
+ Result::<Q::IntermediateOutput, webgraph_server::EncodedError>::from(res).ok()
})
.collect())
}
- pub async fn raw_ingoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {
- let res = self
- .conn()
- .await
- .send(
- RawIngoingEdges { node: id, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
- })
- .collect())
+ pub async fn search<Q>(&self, query: Q) -> Result<Q::Output>
+ where
+ Q: Query,
+ <Q::Collector as webgraph::Collector>::Fruit: Clone,
+ Result<
+ <Q::Collector as webgraph::Collector>::Fruit,
+ webgraph_server::EncodedError,
+ >: From<<Q as sonic::service::Message<WebGraphService>>::Response>,
+ <<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as webgraph::Collector>::Fruit>,
+ <Q as webgraph_server::Query>::RetrieveReq: sonic::service::Wrapper<WebGraphService>,
+ Result<Q::IntermediateOutput, webgraph_server::EncodedError>: From<
+ <<Q as webgraph_server::Query>::RetrieveReq as sonic::service::Message<
+ WebGraphService,
+ >>::Response,
+ >,
+ {
+ let fruit = self.search_initial(&query).await?;
+ let res = self.retrieve(query, fruit).await?;
+ let output = Q::merge_results(res);
+ Ok(output)
}
- pub async fn raw_ingoing_edges_with_labels(
+ pub async fn batch_search_initial<Q>(
&self,
- id: NodeID,
- limit: EdgeLimit,
- ) -> Result<Vec<Edge<String>>> {
+ queries: &[Q],
+ ) -> Result<Vec<<Q::Collector as webgraph::Collector>::Fruit>>
+ where
+ Q: Query,
+ <Q::Collector as webgraph::Collector>::Fruit: Clone,
+ Result<<<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, webgraph_server::EncodedError>:
+ From<<Q as sonic::service::Message<WebGraphService>>::Response>,
+ {
let res = self
.conn()
.await
- .send(
- RawIngoingEdgesWithLabels { node: id, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
+ .batch_send(queries, &AllShardsSelector, &RandomReplicaSelector)
.await?;
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
- })
- .collect())
- }
+ let mut fruits = Vec::with_capacity(queries.len());
- pub async fn batch_raw_ingoing_edges_with_labels(
- &self,
- ids: &[NodeID],
- limit: EdgeLimit,
- ) -> Result<Vec<Vec<Edge<String>>>> {
- let reqs: Vec<_> = ids
- .iter()
- .map(|id| RawIngoingEdgesWithLabels { node: *id, limit })
- .collect();
-
- let res = self
- .conn()
- .await
- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
-
- let mut edges = vec![vec![]; ids.len()];
+ for _ in 0..queries.len() {
+ fruits.push(Vec::new());
+ }
- for (_, res) in res {
- debug_assert!(res.len() <= 1);
+ for (_, replica_results) in res.into_iter() {
+ debug_assert_eq!(replica_results.len(), 1);
- for (_, res) in res {
- for (i, rep) in res.into_iter().enumerate() {
- edges[i].extend(rep);
+ for (_, shard_results) in replica_results.into_iter() {
+ for (i, shard_result) in shard_results.into_iter().enumerate() {
+ if let Ok(shard_result) =
+ Result::<_, webgraph_server::EncodedError>::from(shard_result)
+ {
+ fruits[i].push(shard_result);
+ }
}
}
}
- Ok(edges)
+ queries
+ .iter()
+ .zip_eq(fruits.into_iter())
+ .map(|(query, shard_fruits)| query.remote_collector().merge_fruits(shard_fruits))
+ .collect::<Result<Vec<_>, _>>()
}
- pub async fn batch_raw_ingoing_edges(
+ pub async fn batch_retrieve<Q>(
&self,
- ids: &[NodeID],
- limit: EdgeLimit,
- ) -> Result<Vec<Vec<Edge<()>>>> {
- let reqs: Vec<_> = ids
- .iter()
- .map(|id| RawIngoingEdges { node: *id, limit })
- .collect();
-
- let res = self
- .conn()
- .await
- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
+ queries: Vec<(Q, <Q::Collector as webgraph::Collector>::Fruit)>,
+ ) -> Result<Vec<Vec<Q::IntermediateOutput>>>
+ where
+ Q: Query,
+ <Q as webgraph_server::Query>::RetrieveReq: sonic::service::Wrapper<WebGraphService>,
+ Result<Q::IntermediateOutput, webgraph_server::EncodedError>: From<
+ <<Q as webgraph_server::Query>::RetrieveReq as sonic::service::Message<
+ WebGraphService,
+ >>::Response,
+ >,
+ <Q::Collector as webgraph::Collector>::Fruit: Clone,
+ {
+ let conn = self.conn().await;
+ let mut results = FuturesUnordered::new();
+
+ for shard in conn.shards() {
+ let retrieve_requests: Vec<_> = queries
+ .iter()
+ .map(|(query, fruit)| {
+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());
+ Q::RetrieveReq::new(query.clone(), fruit)
+ })
+ .collect();
+
+ results.push(async move {
+ let retrieve_requests = retrieve_requests; // move lifetime
+ shard
+ .replicas()
+ .batch_send(&retrieve_requests, &RandomReplicaSelector)
+ .await
+ });
+ }
- let mut edges = vec![vec![]; ids.len()];
+ let mut res = Vec::new();
- for (_, res) in res {
- debug_assert!(res.len() <= 1);
+ for _ in 0..queries.len() {
+ res.push(Vec::new());
+ }
- for (_, res) in res {
- for (i, rep) in res.into_iter().enumerate() {
- edges[i].extend(rep);
+ while let Some(shard_res) = results.next().await {
+ for (_, shard_res) in shard_res? {
+ assert_eq!(shard_res.len(), queries.len());
+
+ for (i, query_res) in shard_res.into_iter().enumerate() {
+ res[i].push(
+ <Result<Q::IntermediateOutput, webgraph_server::EncodedError>>::from(
+ query_res,
+ )
+ .map_err(|e| anyhow::anyhow!("{e}"))?,
+ );
}
}
}
- Ok(edges)
+ Ok(res)
}
- pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
+ pub async fn batch_search<Q>(&self, queries: Vec<Q>) -> Result<Vec<Q::Output>>
+ where
+ Q: Query,
+ <Q::Collector as webgraph::Collector>::Fruit: Clone,
+ Result<<<Q::Collector as webgraph::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, webgraph_server::EncodedError>:
+ From<<Q as sonic::service::Message<WebGraphService>>::Response>,
+ <Q as webgraph_server::Query>::RetrieveReq: sonic::service::Wrapper<WebGraphService>,
+ Result<Q::IntermediateOutput, webgraph_server::EncodedError>: From<
+ <<Q as webgraph_server::Query>::RetrieveReq as sonic::service::Message<
+ WebGraphService,
+ >>::Response,
+ >,
+ {
+ let res = self.batch_search_initial(&queries).await?;
let res = self
- .conn()
- .await
- .send(
- OutgoingEdges { node, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
+ .batch_retrieve(queries.into_iter().zip(res).collect())
.await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
- })
- .collect())
- }
-
- pub async fn raw_outgoing_edges(&self, id: NodeID, limit: EdgeLimit) -> Result<Vec<Edge<()>>> {
- let res = self
- .conn()
- .await
- .send(
- RawOutgoingEdges { node: id, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
- })
- .collect())
+ Ok(res.into_iter().map(|v| Q::merge_results(v)).collect())
}
- pub async fn raw_outgoing_edges_with_labels(
- &self,
- id: NodeID,
- limit: EdgeLimit,
- ) -> Result<Vec<Edge<String>>> {
- let res = self
- .conn()
- .await
- .send(
- RawOutgoingEdgesWithLabels { node: id, limit },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, reps)| {
- debug_assert!(reps.len() <= 1);
- reps.into_iter().flat_map(|(_, rep)| rep)
- })
- .collect())
- }
-
- pub async fn batch_raw_outgoing_edges(
- &self,
- ids: &[NodeID],
- limit: EdgeLimit,
- ) -> Result<Vec<Vec<Edge<()>>>> {
- let reqs: Vec<_> = ids
- .iter()
- .map(|id| RawOutgoingEdges { node: *id, limit })
- .collect();
+ pub async fn knows(&self, mut host: String) -> Result<Option<Node>> {
+ if let Some(suf) = host.strip_prefix("http://") {
+ host = suf.to_string();
+ }
+ if let Some(suf) = host.strip_prefix("https://") {
+ host = suf.to_string();
+ }
- let res = self
- .conn()
- .await
- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
+ let url = Url::parse(&("http://".to_string() + host.as_str()))?;
+ let node = Node::from(url).into_host();
+ let id = node.id();
+ let edges = self
+ .search(BacklinksQuery::new(id).with_limit(EdgeLimit::Limit(1)))
.await?;
- let mut edges = vec![vec![]; ids.len()];
-
- for (_, res) in res {
- debug_assert!(res.len() <= 1);
-
- for (_, res) in res {
- for (i, rep) in res.into_iter().enumerate() {
- edges[i].extend(rep);
- }
- }
+ if !edges.is_empty() {
+ Ok(Some(node))
+ } else {
+ Ok(None)
}
-
- Ok(edges)
}
- pub async fn stream_node_ids(&self) -> impl futures::Stream<Item = NodeID> {
+ pub async fn stream_page_node_ids(&self) -> impl futures::Stream<Item = NodeID> {
StreamNodeIDs::new(self.conn().await).stream()
}
}
@@ -469,7 +377,7 @@ impl StreamingResponse for StreamNodeIDs {
type Item = NodeID;
async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
- let req = GetNodeIDs {
+ let req = GetPageNodeIDs {
offset: self.offset,
limit: self.limit,
};
diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rs
new file mode 100644
index 00000000..8ba42592
--- /dev/null
+++ b/crates/core/src/webgraph/schema.rs
@@ -0,0 +1,320 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use enum_dispatch::enum_dispatch;
+use strum::{EnumDiscriminants, VariantArray};
+use tantivy::schema::FieldType;
+use tantivy::schema::NumericOptions;
+use tantivy::schema::OwnedValue;
+use tantivy::schema::TextOptions;
+use tantivy::schema::Value;
+
+use crate::enum_dispatch_from_discriminant;
+use crate::Result;
+
+use super::document::{Edge, ReferenceValue};
+use super::Node;
+
+pub fn create_schema() -> tantivy::schema::Schema {
+ let mut schema_builder = tantivy::schema::Schema::builder();
+
+ for field in FieldEnum::iter() {
+ schema_builder.add_field(tantivy::schema::FieldEntry::new(
+ field.name().to_string(),
+ field.field_type(),
+ ));
+ }
+
+ schema_builder.build()
+}
+
+#[enum_dispatch]
+pub trait Field:
+ Into<FieldEnum> + Clone + Copy + std::fmt::Debug + bincode::Encode + bincode::Decode
+{
+ fn name(&self) -> &'static str;
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a>;
+ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()>;
+ fn field_type(&self) -> tantivy::schema::FieldType;
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct FromUrl;
+impl Field for FromUrl {
+ fn name(&self) -> &'static str {
+ "from_url"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::Str(edge.from.as_str())
+ }
+
+ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {
+ let url = value
+ .as_ref()
+ .as_str()
+ .ok_or(anyhow::anyhow!("Invalid URL"))?;
+ edge.from = Node::from_str_not_validated(url);
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::Str(TextOptions::default().set_stored())
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct ToUrl;
+impl Field for ToUrl {
+ fn name(&self) -> &'static str {
+ "to_url"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::Str(edge.to.as_str())
+ }
+
+ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {
+ let url = value
+ .as_ref()
+ .as_str()
+ .ok_or(anyhow::anyhow!("Invalid URL"))?;
+ edge.to = Node::from_str_not_validated(url);
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::Str(TextOptions::default().set_stored())
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct FromId;
+impl Field for FromId {
+ fn name(&self) -> &'static str {
+ "from_id"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::U64(edge.from.id().as_u64())
+ }
+
+ fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::U64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct ToId;
+impl Field for ToId {
+ fn name(&self) -> &'static str {
+ "to_id"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::U64(edge.to.id().as_u64())
+ }
+
+ fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::U64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct FromHostId;
+impl Field for FromHostId {
+ fn name(&self) -> &'static str {
+ "from_host_id"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::U64(edge.from.clone().into_host().id().as_u64())
+ }
+
+ fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::U64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct ToHostId;
+impl Field for ToHostId {
+ fn name(&self) -> &'static str {
+ "to_host_id"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::U64(edge.to.clone().into_host().id().as_u64())
+ }
+
+ fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::U64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct RelFlags;
+impl Field for RelFlags {
+ fn name(&self) -> &'static str {
+ "rel_flags"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::U64(edge.rel_flags.as_u64())
+ }
+
+ fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::U64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct Label;
+impl Field for Label {
+ fn name(&self) -> &'static str {
+ "label"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::Str(edge.label.as_str())
+ }
+
+ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {
+ edge.label = value
+ .as_ref()
+ .as_str()
+ .ok_or(anyhow::anyhow!("Invalid label"))?
+ .to_string();
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::Str(TextOptions::default().set_stored())
+ }
+}
+
+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]
+pub struct SortScore;
+impl Field for SortScore {
+ fn name(&self) -> &'static str {
+ "sort_score"
+ }
+
+ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
+ ReferenceValue::F64(edge.sort_score)
+ }
+
+ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {
+ let sort_score = value
+ .as_ref()
+ .as_f64()
+ .ok_or(anyhow::anyhow!("Invalid sort score"))?;
+ edge.sort_score = sort_score;
+
+ Ok(())
+ }
+
+ fn field_type(&self) -> FieldType {
+ FieldType::F64(
+ NumericOptions::default()
+ .set_indexed()
+ .set_stored()
+ .set_columnar(),
+ )
+ }
+}
+
+#[enum_dispatch(Field)]
+#[derive(Clone, Copy, Debug, EnumDiscriminants, bincode::Encode, bincode::Decode)]
+#[strum_discriminants(derive(VariantArray))]
+pub enum FieldEnum {
+ FromUrl,
+ ToUrl,
+ FromId,
+ ToId,
+ FromHostId,
+ ToHostId,
+ RelFlags,
+ Label,
+ SortScore,
+}
+
+impl FieldEnum {
+ pub fn iter() -> impl Iterator<Item = FieldEnum> {
+ FieldEnumDiscriminants::VARIANTS
+ .iter()
+ .copied()
+ .map(FieldEnum::from)
+ }
+}
+
+enum_dispatch_from_discriminant!(FieldEnumDiscriminants => FieldEnum,
+[
+ FromUrl,
+ ToUrl,
+ FromId,
+ ToId,
+ FromHostId,
+ ToHostId,
+ RelFlags,
+ Label,
+ SortScore,
+]);
diff --git a/crates/core/src/webgraph/compression.rs b/crates/core/src/webgraph/searcher.rs
similarity index 57%
rename from crates/core/src/webgraph/compression.rs
rename to crates/core/src/webgraph/searcher.rs
index 94a98ee8..eaf1ba98 100644
--- a/crates/core/src/webgraph/compression.rs
+++ b/crates/core/src/webgraph/searcher.rs
@@ -12,27 +12,28 @@
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
-#[derive(Default, Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
-pub enum Compression {
- None,
- #[default]
- Lz4,
+use crate::ampc::dht::ShardId;
+
+pub struct Searcher {
+ tantivy_searcher: tantivy::Searcher,
+ shard: ShardId,
}
-impl Compression {
- pub fn compress(&self, bytes: &[u8]) -> Vec<u8> {
- match self {
- Compression::None => bytes.to_vec(),
- Compression::Lz4 => lz4_flex::compress_prepend_size(bytes),
+impl Searcher {
+ pub fn new(tantivy_searcher: tantivy::Searcher, shard: ShardId) -> Self {
+ Self {
+ tantivy_searcher,
+ shard,
}
}
- pub fn decompress(&self, bytes: &[u8]) -> Vec<u8> {
- match self {
- Compression::None => bytes.to_vec(),
- Compression::Lz4 => lz4_flex::decompress_size_prepended(bytes).unwrap(),
- }
+ pub fn shard(&self) -> ShardId {
+ self.shard
+ }
+
+ pub fn tantivy_searcher(&self) -> &tantivy::Searcher {
+ &self.tantivy_searcher
}
}
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
deleted file mode 100644
index 28e43305..00000000
--- a/crates/core/src/webgraph/segment.rs
+++ /dev/null
@@ -1,385 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{
- path::{Path, PathBuf},
- sync::Arc,
- thread,
-};
-
-use super::{
- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, InsertableEdge, NodeID,
- SegmentEdge,
-};
-use crate::Result;
-
-const ADJACENCY_STORE: &str = "adjacency";
-const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";
-
-pub struct SegmentWriter {
- adjacency: EdgeStoreWriter,
- reversed_adjacency: EdgeStoreWriter,
- id: String,
- folder_path: String,
-}
-
-impl SegmentWriter {
- pub fn open<P: AsRef<Path>>(
- folder_path: P,
- id: String,
- compression: Compression,
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
- ) -> Self {
- SegmentWriter {
- adjacency: EdgeStoreWriter::new(
- folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
- compression,
- false,
- host_centrality_rank_store.clone(),
- ),
- reversed_adjacency: EdgeStoreWriter::new(
- folder_path
- .as_ref()
- .join(&id)
- .join(REVERSED_ADJACENCY_STORE),
- compression,
- true,
- host_centrality_rank_store.clone(),
- ),
- folder_path: folder_path
- .as_ref()
- .as_os_str()
- .to_str()
- .unwrap()
- .to_string(),
- id,
- }
- }
-
- pub fn finalize(self) -> Segment {
- Segment {
- adjacency: self.adjacency.finalize(),
- reversed_adjacency: self.reversed_adjacency.finalize(),
- folder_path: self.folder_path,
- id: self.id,
- }
- }
-
- pub fn insert(&mut self, edge: InsertableEdge<String>) {
- self.adjacency.put(edge.clone());
- self.reversed_adjacency.put(edge);
- }
-}
-
-pub struct Segment {
- adjacency: EdgeStore,
- reversed_adjacency: EdgeStore,
- id: String,
- folder_path: String,
-}
-
-impl Segment {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
- Segment {
- adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE), false),
- reversed_adjacency: EdgeStore::open(
- folder_path
- .as_ref()
- .join(&id)
- .join(REVERSED_ADJACENCY_STORE),
- true,
- ),
- folder_path: folder_path
- .as_ref()
- .as_os_str()
- .to_str()
- .unwrap()
- .to_string(),
- id,
- }
- }
-
- pub fn merge<P: AsRef<Path>>(
- segments: Vec<Self>,
- label_compression: Compression,
- folder: P,
- id: String,
- ) -> Result<()> {
- if segments.is_empty() {
- return Ok(());
- }
-
- if segments.len() == 1 {
- let segment = &segments[0];
- std::fs::rename(segment.path(), folder.as_ref().join(&id))?;
- return Ok(());
- }
-
- let old_paths = segments.iter().map(|s| s.path()).collect::<Vec<_>>();
-
- let (adjacency, reversed_adjacency) = segments
- .into_iter()
- .map(|s| (s.adjacency, s.reversed_adjacency))
- .unzip();
-
- let adjacency_path = folder.as_ref().join(&id).join(ADJACENCY_STORE);
- let adjacency =
- thread::spawn(move || EdgeStore::merge(adjacency, label_compression, adjacency_path));
-
- let reversed_adjacency_path = folder.as_ref().join(&id).join(REVERSED_ADJACENCY_STORE);
- let reversed_adjacency = thread::spawn(move || {
- EdgeStore::merge(
- reversed_adjacency,
- label_compression,
- reversed_adjacency_path,
- )
- });
-
- adjacency.join().unwrap()?;
- reversed_adjacency.join().unwrap()?;
-
- for path in old_paths {
- if Path::new(&path).exists() {
- std::fs::remove_dir_all(path)?;
- }
- }
-
- Ok(())
- }
-
- pub fn outgoing_edges_with_label<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {
- self.adjacency.get_with_label(node)
- }
-
- pub fn outgoing_edges<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {
- self.adjacency.get_without_label(node)
- }
-
- pub fn out_degree(&self, node: &NodeID) -> u64 {
- self.adjacency.degree(node)
- }
-
- pub fn ingoing_edges_with_label<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {
- self.reversed_adjacency.get_with_label(node)
- }
-
- pub fn ingoing_edges<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {
- self.reversed_adjacency.get_without_label(node)
- }
-
- pub fn in_degree(&self, node: &NodeID) -> u64 {
- self.reversed_adjacency.degree(node)
- }
-
- pub fn id(&self) -> String {
- self.id.clone()
- }
-
- pub fn path(&self) -> PathBuf {
- Path::new(&self.folder_path).join(&self.id)
- }
-
- pub fn edges(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {
- self.adjacency.iter_without_label()
- }
-
- pub fn optimize_read(&mut self) {
- self.adjacency.optimize_read();
- self.reversed_adjacency.optimize_read();
- }
-}
-
-#[cfg(test)]
-mod test {
- use crate::{
- webgraph::{Edge, FullNodeID, NodeDatum},
- webpage::html::links::RelFlags,
- };
-
- use super::*;
-
- #[test]
- #[allow(clippy::too_many_lines)]
- fn simple_triangle_graph() {
- let temp_dir = file_store::gen_temp_dir().unwrap();
- // ┌────┐
- // │ │
- // ┌───0◄─┐ │
- // │ │ │
- // ▼ │ │
- // 1─────►2◄┘
-
- let mut writer = SegmentWriter::open(
- temp_dir.as_ref(),
- "test".to_string(),
- Compression::default(),
- None,
- );
-
- let mut edges = Vec::new();
-
- let a = FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- };
- let b = FullNodeID {
- id: NodeID::from(1_u64),
- host: NodeID::from(0_u64),
- };
- let c = FullNodeID {
- id: NodeID::from(2_u64),
- host: NodeID::from(0_u64),
- };
-
- edges.push(InsertableEdge {
- from: a.clone(),
- to: b.clone(),
- label: String::new(),
- rel: RelFlags::default(),
- });
- edges.push(InsertableEdge {
- from: b.clone(),
- to: c.clone(),
- label: String::new(),
- rel: RelFlags::default(),
- });
- edges.push(InsertableEdge {
- from: c.clone(),
- to: a.clone(),
- label: String::new(),
- rel: RelFlags::default(),
- });
- edges.push(InsertableEdge {
- from: a.clone(),
- to: c.clone(),
- label: String::new(),
- rel: RelFlags::default(),
- });
-
- for edge in &edges {
- writer.insert(edge.clone());
- }
- let segment = writer.finalize();
-
- let mut out: Vec<_> = segment.outgoing_edges(&a.id).collect();
-
- out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
-
- assert_eq!(
- out,
- vec![
- Edge {
- from: NodeDatum::new(a.id, u64::MAX),
- to: NodeDatum::new(b.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),
- Edge {
- from: NodeDatum::new(a.id, u64::MAX),
- to: NodeDatum::new(c.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),
- ]
- );
-
- let mut out: Vec<_> = segment.outgoing_edges(&b.id).collect();
- out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
- assert_eq!(
- out,
- vec![Edge {
- from: NodeDatum::new(b.id, u64::MAX),
- to: NodeDatum::new(c.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),]
- );
-
- let mut out: Vec<_> = segment.outgoing_edges(&c.id).collect();
- out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));
- assert_eq!(
- out,
- vec![Edge {
- from: NodeDatum::new(c.id, u64::MAX),
- to: NodeDatum::new(a.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),]
- );
-
- let out: Vec<_> = segment.ingoing_edges(&a.id).collect();
- assert_eq!(
- out,
- vec![Edge {
- from: NodeDatum::new(c.id, u64::MAX),
- to: NodeDatum::new(a.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),]
- );
-
- let out: Vec<_> = segment.ingoing_edges(&b.id).collect();
- assert_eq!(
- out,
- vec![Edge {
- from: NodeDatum::new(a.id, u64::MAX),
- to: NodeDatum::new(b.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),]
- );
-
- let mut out: Vec<_> = segment.ingoing_edges(&c.id).collect();
- out.sort_by(|a, b| a.from.node().cmp(&b.from.node()));
- assert_eq!(
- out,
- vec![
- Edge {
- from: NodeDatum::new(a.id, u64::MAX),
- to: NodeDatum::new(c.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),
- Edge {
- from: NodeDatum::new(b.id, u64::MAX),
- to: NodeDatum::new(c.id, u64::MAX),
- label: (),
- rel: RelFlags::default(),
- }
- .into(),
- ]
- );
- }
-}
diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rs
index 536fcc3b..dbe4925f 100644
--- a/crates/core/src/webgraph/shortest_path.rs
+++ b/crates/core/src/webgraph/shortest_path.rs
@@ -19,28 +19,50 @@ use std::{
collections::{BTreeMap, BinaryHeap},
};
-use super::{Edge, EdgeLabel, Node, NodeID, Webgraph};
+use crate::config::WebgraphGranularity;
+
+use super::{query, EdgeLimit, Node, NodeID, SmallEdge, Webgraph};
pub trait ShortestPaths {
- fn distances(&self, source: Node) -> BTreeMap<Node, u8>;
- fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
- fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8>;
- fn raw_reversed_distances_with_max(&self, source: NodeID, max_dist: u8)
- -> BTreeMap<NodeID, u8>;
- fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8>;
- fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;
+ fn distances(&self, source: Node, granularity: WebgraphGranularity) -> BTreeMap<Node, u8>;
+ fn raw_distances(
+ &self,
+ source: NodeID,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8>;
+ fn raw_distances_with_max(
+ &self,
+ source: NodeID,
+ max_dist: u8,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8>;
+ fn raw_reversed_distances_with_max(
+ &self,
+ source: NodeID,
+ max_dist: u8,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8>;
+ fn raw_reversed_distances(
+ &self,
+ source: NodeID,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8>;
+ fn reversed_distances(
+ &self,
+ source: Node,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<Node, u8>;
}
-fn dijkstra_multi<F1, F2, L>(
+fn dijkstra_multi<F1, F2>(
sources: &[NodeID],
node_edges: F1,
edge_node: F2,
max_dist: Option<u8>,
) -> BTreeMap<NodeID, u8>
where
- L: EdgeLabel,
- F1: Fn(NodeID) -> Vec<Edge<L>>,
- F2: Fn(&Edge<L>) -> NodeID,
+ F1: Fn(NodeID) -> Vec<SmallEdge>,
+ F2: Fn(&SmallEdge) -> NodeID,
{
let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();
@@ -81,36 +103,83 @@ where
}
impl ShortestPaths for Webgraph {
- fn distances(&self, source: Node) -> BTreeMap<Node, u8> {
- self.raw_distances(source.id())
+ fn distances(&self, source: Node, granularity: WebgraphGranularity) -> BTreeMap<Node, u8> {
+ self.raw_distances(source.id(), granularity)
.into_iter()
- .filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist)))
+ .filter_map(|(id, dist)| match granularity {
+ WebgraphGranularity::Host => self
+ .search(&query::Id2NodeQuery::Host(id))
+ .unwrap()
+ .map(|node| (node, dist)),
+ WebgraphGranularity::Page => self
+ .search(&query::Id2NodeQuery::Page(id))
+ .unwrap()
+ .map(|node| (node, dist)),
+ })
.collect()
}
- fn raw_distances_with_max(&self, source: NodeID, max_dist: u8) -> BTreeMap<NodeID, u8> {
+ fn raw_distances_with_max(
+ &self,
+ source: NodeID,
+ max_dist: u8,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.to.node(),
+ |node| match granularity {
+ WebgraphGranularity::Host => self
+ .search(
+ &query::HostForwardlinksQuery::new(node).with_limit(EdgeLimit::Unlimited),
+ )
+ .unwrap_or_default(),
+ WebgraphGranularity::Page => self
+ .search(&query::ForwardlinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ },
+ |edge| edge.to,
Some(max_dist),
)
}
- fn raw_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
+ fn raw_distances(
+ &self,
+ source: NodeID,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.to.node(),
+ |node| match granularity {
+ WebgraphGranularity::Host => self
+ .search(
+ &query::HostForwardlinksQuery::new(node).with_limit(EdgeLimit::Unlimited),
+ )
+ .unwrap_or_default(),
+ WebgraphGranularity::Page => self
+ .search(&query::ForwardlinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ },
+ |edge| edge.to,
None,
)
}
- fn raw_reversed_distances(&self, source: NodeID) -> BTreeMap<NodeID, u8> {
+ fn raw_reversed_distances(
+ &self,
+ source: NodeID,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.from.node(),
+ |node| match granularity {
+ WebgraphGranularity::Host => self
+ .search(&query::HostBacklinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ WebgraphGranularity::Page => self
+ .search(&query::BacklinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ },
+ |edge| edge.from,
None,
)
}
@@ -119,19 +188,40 @@ impl ShortestPaths for Webgraph {
&self,
source: NodeID,
max_dist: u8,
+ granularity: WebgraphGranularity,
) -> BTreeMap<NodeID, u8> {
dijkstra_multi(
&[source],
- |node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.from.node(),
+ |node| match granularity {
+ WebgraphGranularity::Host => self
+ .search(&query::HostBacklinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ WebgraphGranularity::Page => self
+ .search(&query::BacklinksQuery::new(node).with_limit(EdgeLimit::Unlimited))
+ .unwrap_or_default(),
+ },
+ |edge| edge.from,
Some(max_dist),
)
}
- fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {
- self.raw_reversed_distances(source.id())
+ fn reversed_distances(
+ &self,
+ source: Node,
+ granularity: WebgraphGranularity,
+ ) -> BTreeMap<Node, u8> {
+ self.raw_reversed_distances(source.id(), granularity)
.into_iter()
- .filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist)))
+ .filter_map(|(id, dist)| match granularity {
+ WebgraphGranularity::Host => self
+ .search(&query::Id2NodeQuery::Host(id))
+ .unwrap()
+ .map(|node| (node, dist)),
+ WebgraphGranularity::Page => self
+ .search(&query::Id2NodeQuery::Page(id))
+ .unwrap()
+ .map(|node| (node, dist)),
+ })
.collect()
}
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index cca6e41e..0439ea98 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -14,724 +14,412 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{fs::File, ops::Range, path::Path};
-
-use crate::{
- webgraph::merge::{EdgeMerger, MergeIter},
- webpage::html::links::RelFlags,
- Result,
-};
-use anyhow::bail;
-use file_store::{
- iterable::{
- ConstIterableStoreReader, ConstIterableStoreWriter, IterableStoreReader,
- IterableStoreWriter,
- },
- ConstSerializable,
+use std::{
+ collections::HashSet,
+ fs,
+ ops::Range,
+ path::{Path, PathBuf},
};
-use itertools::Itertools;
use super::{
- merge::{MergeNode, MergeSegmentOrd},
- Compression, NodeDatum, NodeID, SegmentEdge, StoredEdge,
+ document::SmallEdge,
+ query::collector::{Collector, TantivyCollector},
+ schema::{self, create_schema, Field, FromHostId, FromId, ToHostId, ToId},
+ searcher::Searcher,
+ Edge,
+};
+use crate::{ampc::dht::ShardId, webpage::html::links::RelFlags, Result};
+use itertools::Itertools;
+use tantivy::{
+ columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy, DocId, SegmentReader,
};
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct EdgeRange {
- range: std::ops::Range<u64>,
- host_rank: u64,
-}
-
-impl EdgeRange {
- pub fn new(range: std::ops::Range<u64>, host_rank: u64) -> Self {
- Self { range, host_rank }
- }
-}
-
-#[inline]
-fn usize_range(range: std::ops::Range<u64>) -> std::ops::Range<usize> {
- range.start as usize..range.end as usize
-}
+use super::{query::Query, NodeID};
-pub struct RangesDb {
- edges: speedy_kv::Db<NodeID, EdgeRange>,
- labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,
+pub struct EdgeStore {
+ index: tantivy::Index,
+ writer: Option<tantivy::IndexWriter<Edge>>,
+ reader: tantivy::IndexReader,
+ shard_id: ShardId,
+ path: PathBuf,
}
-impl RangesDb {
- pub fn open<P: AsRef<Path>>(path: P) -> Self {
- let edges = speedy_kv::Db::open_or_create(path.as_ref().join("edges")).unwrap();
- let labels = speedy_kv::Db::open_or_create(path.as_ref().join("labels")).unwrap();
-
- Self { edges, labels }
- }
-
- pub fn optimize_read(&mut self) {
- self.edges.merge_all_segments().unwrap();
- self.labels.merge_all_segments().unwrap();
- }
+impl EdgeStore {
+ pub fn open<P: AsRef<Path>>(path: P, shard_id: ShardId) -> Result<Self> {
+ if !path.as_ref().exists() {
+ fs::create_dir_all(&path)?;
+ }
- fn uncommitted_node_inserts(&self) -> usize {
- self.edges.uncommitted_inserts()
+ let index: tantivy::Index = tantivy::IndexBuilder::new()
+ .schema(create_schema())
+ .settings(tantivy::IndexSettings {
+ sort_by_field: Some(tantivy::IndexSortByField {
+ field: schema::SortScore.name().to_string(),
+ order: tantivy::Order::Desc,
+ }),
+ ..Default::default()
+ })
+ .open_or_create(MmapDirectory::open(&path)?)?;
+
+ Ok(Self {
+ path: path.as_ref().to_path_buf(),
+ writer: None,
+ reader: index.reader()?,
+ index,
+ shard_id,
+ })
}
- pub fn commit(&mut self) {
- self.edges.commit().unwrap();
- self.labels.commit().unwrap();
- }
+ fn prepare_writer(&mut self) -> Result<()> {
+ if self.writer.is_some() {
+ return Ok(());
+ }
- pub fn nodes_get_raw<'a>(
- &'a self,
- key: &'a [u8],
- ) -> Option<speedy_kv::SerializedRef<'a, EdgeRange>> {
- self.edges.get_raw(key)
- }
+ let writer = self.index.writer_with_num_threads(1, 1_000_000_000)?;
- pub fn labels_get_raw<'a>(
- &'a self,
- key: &'a [u8],
- ) -> Option<speedy_kv::SerializedRef<'a, Range<u64>>> {
- self.labels.get_raw(key)
- }
+ let merge_policy = NoMergePolicy;
+ writer.set_merge_policy(Box::new(merge_policy));
- pub fn insert_raw_node(&mut self, node: Vec<u8>, range: Vec<u8>) {
- self.edges.insert_raw(node, range);
- }
+ self.writer = Some(writer);
- pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {
- self.labels.insert_raw(node, range);
+ Ok(())
}
- fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {
- self.edges
- .sorted_iter_raw()
- .zip_eq(self.labels.sorted_iter_raw())
- .map(move |((key_node, val), (key_label, labels))| {
- debug_assert_eq!(key_node, key_label);
-
- let node = NodeID::from_be_bytes(key_node.as_bytes().try_into().unwrap());
- let range = EdgeRange::deserialize(val.as_bytes());
+ pub fn optimize_read(&mut self) -> Result<()> {
+ self.prepare_writer()?;
+ let base_path = Path::new(&self.path);
+ let segments: Vec<_> = self.index.load_metas()?.segments.into_iter().collect();
- let labels = {
- let range = Range::deserialize(labels.as_bytes());
- range.start..range.end
- };
+ tantivy::merge_segments(
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared"),
+ segments,
+ base_path,
+ 1,
+ )?;
- MergeNode::new(node, range, labels)
- })
+ Ok(())
}
-}
-impl ConstSerializable for NodeID {
- const BYTES: usize = std::mem::size_of::<NodeID>();
-
- fn serialize(&self, buf: &mut [u8]) {
- self.as_u64().serialize(buf);
+ pub fn insert(&mut self, edge: Edge) -> Result<()> {
+ self.prepare_writer()?;
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared")
+ .add_document(edge)?;
+ Ok(())
}
- fn deserialize(buf: &[u8]) -> Self {
- let id = u64::deserialize(buf);
- NodeID::from(id)
+ pub fn commit(&mut self) -> Result<()> {
+ self.prepare_writer()?;
+ self.writer
+ .as_mut()
+ .expect("writer should have been prepared")
+ .commit()?;
+ self.reader.reload()?;
+ Ok(())
}
-}
-impl ConstSerializable for RelFlags {
- const BYTES: usize = std::mem::size_of::<RelFlags>();
+ pub fn merge(&mut self, mut other: Self) -> Result<()> {
+ self.prepare_writer()?;
+ other.prepare_writer()?;
- fn serialize(&self, buf: &mut [u8]) {
- self.as_u32().serialize(buf);
- }
+ other.commit()?;
+ self.commit()?;
- fn deserialize(buf: &[u8]) -> Self {
- u32::deserialize(buf).into()
- }
-}
+ let other_meta = other.index.load_metas()?;
-impl ConstSerializable for NodeDatum {
- const BYTES: usize = std::mem::size_of::<NodeDatum>();
+ let mut meta = self.index.load_metas()?;
- fn serialize(&self, buf: &mut [u8]) {
- let node = self.node().to_be_bytes();
- buf[..8].copy_from_slice(&node);
- self.host_rank().serialize(&mut buf[8..]);
- }
+ let other_path = Path::new(&other.path);
+ let other_writer = other.writer.take().unwrap();
+ other_writer.wait_merging_threads().unwrap();
- fn deserialize(buf: &[u8]) -> Self {
- let node = NodeID::from_be_bytes(buf[..8].try_into().unwrap());
- let host_rank = u64::deserialize(&buf[8..]);
+ let self_path = Path::new(&self.path);
+ let self_writer = self.writer.take().unwrap();
+ self_writer.wait_merging_threads().unwrap();
- Self::new(node, host_rank)
- }
-}
+ let ids: HashSet<_> = meta.segments.iter().map(|segment| segment.id()).collect();
-impl ConstSerializable for EdgeRange {
- const BYTES: usize = std::mem::size_of::<EdgeRange>();
+ for segment in other_meta.segments {
+ if ids.contains(&segment.id()) {
+ continue;
+ }
- fn serialize(&self, buf: &mut [u8]) {
- const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
- self.range.serialize(&mut buf[..RANGE_BYTES]);
- self.host_rank.serialize(&mut buf[RANGE_BYTES..]);
- }
+ // TODO: handle case where current index has segment with same name
+ for file in segment.list_files() {
+ let p = other_path.join(&file);
+ if p.exists() {
+ fs::rename(p, self_path.join(&file)).unwrap();
+ }
+ }
+ meta.segments.push(segment);
+ }
- fn deserialize(buf: &[u8]) -> Self {
- const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
- let range: Range<u64> = Range::deserialize(&buf[..RANGE_BYTES]);
- let host_rank = u64::deserialize(&buf[RANGE_BYTES..]);
+ meta.segments
+ .sort_by_key(|a| std::cmp::Reverse(a.max_doc()));
- Self { range, host_rank }
- }
-}
+ fs::remove_dir_all(other_path).ok();
-impl ConstSerializable for StoredEdge {
- const BYTES: usize = std::mem::size_of::<StoredEdge>();
+ let self_path = Path::new(&self.path);
- fn serialize(&self, buf: &mut [u8]) {
- self.other.serialize(&mut buf[..NodeDatum::BYTES]);
- self.rel.serialize(&mut buf[NodeDatum::BYTES..]);
- }
+ std::fs::write(
+ self_path.join("meta.json"),
+ serde_json::to_string_pretty(&meta).unwrap(),
+ )
+ .unwrap();
- fn deserialize(buf: &[u8]) -> Self {
- let other = NodeDatum::deserialize(&buf[..NodeDatum::BYTES]);
- let rel = RelFlags::deserialize(&buf[NodeDatum::BYTES..]);
+ self.re_open()?;
- Self::new(other, rel)
+ Ok(())
}
-}
-
-pub const NUM_LABELS_PER_BLOCK: usize = 128;
-#[derive(bincode::Encode, bincode::Decode)]
-pub struct LabelBlock {
- labels: Vec<String>,
-}
-
-impl LabelBlock {
- pub fn new(labels: Vec<String>) -> Self {
- Self { labels }
+ fn re_open(&mut self) -> Result<()> {
+ *self = Self::open(self.path.clone(), self.shard_id)?;
+ Ok(())
}
- pub fn compress(&self, compression: Compression) -> CompressedLabelBlock {
- let bytes = bincode::encode_to_vec(self, common::bincode_config()).unwrap();
- let compressed = compression.compress(&bytes);
-
- CompressedLabelBlock {
- compressions: compression,
- data: compressed,
- }
+ fn searcher(&self) -> Searcher {
+ Searcher::new(self.reader.searcher(), self.shard_id)
}
-}
-#[derive(bincode::Encode, bincode::Decode)]
-pub struct CompressedLabelBlock {
- compressions: Compression,
- data: Vec<u8>,
-}
+ pub fn search_initial<Q: Query>(
+ &self,
+ query: &Q,
+ ) -> Result<<Q::Collector as Collector>::Fruit> {
+ let res = self.searcher().tantivy_searcher().search(
+ &query.tantivy_query(),
+ &TantivyCollector::from(&query.collector(self.shard_id)),
+ )?;
-impl CompressedLabelBlock {
- pub fn decompress(&self) -> LabelBlock {
- let bytes = self.compressions.decompress(&self.data);
- let (res, _) = bincode::decode_from_slice(&bytes, common::bincode_config()).unwrap();
- res
+ Ok(res)
}
-}
-
-pub struct EdgeStore {
- reversed: bool,
- ranges: RangesDb,
-
- edge_labels: IterableStoreReader<CompressedLabelBlock>,
- edges: ConstIterableStoreReader<StoredEdge>,
-}
-
-impl EdgeStore {
- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
- let ranges = RangesDb::open(path.as_ref().join("ranges"));
- let edge_labels = IterableStoreReader::open(path.as_ref().join("labels")).unwrap();
-
- let edges = ConstIterableStoreReader::open(path.as_ref().join("edges")).unwrap();
-
- Self {
- ranges,
- edge_labels,
- edges,
- reversed,
- }
+ pub fn retrieve<Q: Query>(
+ &self,
+ query: &Q,
+ fruit: <Q::Collector as Collector>::Fruit,
+ ) -> Result<Q::IntermediateOutput> {
+ query.retrieve(&self.searcher(), fruit)
}
- pub fn optimize_read(&mut self) {
- self.ranges.optimize_read();
+ pub fn search<Q>(&self, query: &Q) -> Result<Q::Output>
+ where
+ Q: Query,
+ <<Q::Collector as Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as Collector>::Fruit>,
+ {
+ let fruit = self.search_initial(query)?;
+ let fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;
+ let res = self.retrieve(query, fruit)?;
+ Ok(Q::merge_results(vec![res]))
}
- fn merge_postings_for_node<'a>(
- buf: &[MergeNode<MergeSegmentOrd>],
- stores: &'a [EdgeStore],
- ) -> EdgeMerger<'a> {
- let mut edges = Vec::new();
-
- for node in buf {
- let store = &stores[node.ord().as_usize()];
- let stored_edges = store.edges.slice(usize_range(node.range().range.clone()));
- let labels = store
- .edge_labels
- .slice(usize_range(node.labels()))
- .map(|r| r.decompress())
- .flat_map(|block| block.labels.into_iter());
-
- edges.push(
- stored_edges
- .zip_eq(labels)
- .map(|(edge, label)| edge.with_label(label)),
- );
- }
+ pub fn iter_pages_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {
+ let searcher = self.reader.searcher();
+ let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
- EdgeMerger::new(edges)
+ segment_readers.into_iter().flat_map(|segment| {
+ SmallSegmentEdgesIter::new(&segment, FromId, ToId, 0..segment.max_doc())
+ })
}
- fn merge_postings<P: AsRef<Path>>(
- stores: &[EdgeStore],
- label_compression: Compression,
- folder: P,
- ) -> Result<Self> {
- let reversed = stores[0].reversed;
- let mut ranges = RangesDb::open(folder.as_ref().join("ranges"));
-
- let labels_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(folder.as_ref().join("labels"))
- .unwrap();
- let mut labels_store = IterableStoreWriter::new(labels_file);
-
- let edges_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(folder.as_ref().join("edges"))
- .unwrap();
- let mut edges_store: ConstIterableStoreWriter<StoredEdge, File> =
- ConstIterableStoreWriter::new(edges_file);
-
- let mut merge_iter = MergeIter::new(
- stores
- .iter()
- .map(|store| store.ranges.merge_nodes())
- .collect(),
- );
-
- let mut buf = Vec::new();
-
- while merge_iter.advance(&mut buf) {
- if buf.is_empty() {
- continue;
- }
-
- let edges = Self::merge_postings_for_node(&buf, stores);
-
- // write postings
- let node_host_rank = buf[0].range().host_rank;
- let node_id = buf[0].id();
- debug_assert!(buf.iter().all(|node| node.id() == node_id));
- let mut first_label_offset = None;
- let mut last_label_offset = None;
- let mut first_node_offset = None;
- let mut last_node_offset = None;
-
- for chunk in edges.chunks(NUM_LABELS_PER_BLOCK).into_iter() {
- let (labels, edges): (Vec<_>, Vec<_>) = chunk
- .map(|edge| (edge.label().clone(), edge.with_label(())))
- .unzip();
-
- let label_block = LabelBlock::new(labels).compress(label_compression);
-
- let label_offset = labels_store.write(&label_block).unwrap();
-
- if first_label_offset.is_none() {
- first_label_offset = Some(label_offset);
- }
-
- last_label_offset = Some(label_offset);
-
- for edge in edges {
- let offset = edges_store.write(&edge).unwrap();
-
- if first_node_offset.is_none() {
- first_node_offset = Some(offset);
- }
-
- last_node_offset = Some(offset);
- }
- }
-
- let label_range = Range {
- start: first_label_offset.unwrap().start,
- end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,
- };
+ pub fn iter_hosts_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {
+ let searcher = self.reader.searcher();
+ let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
- let node_range: EdgeRange = EdgeRange::new(
- Range {
- start: first_node_offset.unwrap().start,
- end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
- },
- node_host_rank,
- );
- let node_range_bytes = node_range.serialize_to_vec();
+ segment_readers
+ .into_iter()
+ .flat_map(|segment| {
+ SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, 0..segment.max_doc())
+ })
+ .unique_by(|e| (e.from, e.to))
+ }
- let node_bytes = node_id.as_u64().to_be_bytes().to_vec();
- ranges.insert_raw_node(node_bytes.clone(), node_range_bytes);
+ pub fn iter_page_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {
+ let searcher = self.reader.searcher();
+ let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
- let label_range_bytes = label_range.serialize_to_vec();
+ let range = offset..limit;
- ranges.insert_raw_label(node_bytes, label_range_bytes);
+ segment_readers
+ .into_iter()
+ .flat_map(move |segment| {
+ SmallSegmentEdgesIter::new(&segment, FromId, ToId, range.clone())
+ })
+ .flat_map(|e| [e.from, e.to])
+ .unique()
+ }
- if ranges.uncommitted_node_inserts() > 100_000_000 {
- ranges.commit();
- edges_store.flush().unwrap();
- labels_store.flush().unwrap();
- }
- }
+ pub fn iter_host_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {
+ let searcher = self.reader.searcher();
+ let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
- if ranges.uncommitted_node_inserts() > 0 {
- ranges.commit();
- edges_store.flush().unwrap();
- labels_store.flush().unwrap();
- }
+ let range = offset..limit;
- Ok(Self::open(folder, reversed))
+ segment_readers
+ .into_iter()
+ .flat_map(move |segment| {
+ SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, range.clone())
+ })
+ .flat_map(|e| [e.from, e.to])
+ .unique()
}
+}
- pub fn merge<P: AsRef<Path>>(
- stores: Vec<EdgeStore>,
- label_compression: Compression,
- path: P,
- ) -> Result<()> {
- if stores.is_empty() {
- return Ok(());
- }
+pub struct SmallSegmentEdgesIter {
+ from_id: Column,
+ to_id: Column,
+ rel_flags: Column,
+ doc_range: Range<DocId>,
+ current_doc: DocId,
+}
- if !path.as_ref().exists() {
- std::fs::create_dir_all(&path)?;
+impl SmallSegmentEdgesIter {
+ fn new<F1, F2>(
+ segment: &SegmentReader,
+ from_id: F1,
+ to_id: F2,
+ mut doc_range: Range<DocId>,
+ ) -> Self
+ where
+ F1: Field,
+ F2: Field,
+ {
+ let columns = segment.column_fields();
+
+ if doc_range.end > segment.max_doc() {
+ doc_range.end = segment.max_doc();
}
- let reversed = stores[0].reversed;
- if !stores.iter().all(|store| store.reversed == reversed) {
- bail!("Cannot merge stores with different reversed flags");
+ Self {
+ from_id: columns.u64(from_id.name()).unwrap(),
+ to_id: columns.u64(to_id.name()).unwrap(),
+ rel_flags: columns.u64(schema::RelFlags.name()).unwrap(),
+ current_doc: doc_range.start,
+ doc_range,
}
-
- let mut res = Self::merge_postings(&stores, label_compression, path)?;
- res.optimize_read();
-
- Ok(())
}
+}
- pub fn degree(&self, node: &NodeID) -> u64 {
- let node_bytes = node.as_u64().to_be_bytes();
+impl Iterator for SmallSegmentEdgesIter {
+ type Item = SmallEdge;
- match self.ranges.edges.get_raw(&node_bytes) {
- Some(node_range_bytes) => {
- let range = EdgeRange::deserialize(node_range_bytes.as_bytes());
- self.edges.slice(usize_range(range.range)).len() as u64
- }
- None => 0,
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.current_doc >= self.doc_range.end {
+ return None;
}
- }
- pub fn get_with_label<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {
- let node_bytes = node.as_u64().to_be_bytes();
-
- match (
- self.ranges.edges.get_raw(&node_bytes),
- self.ranges.labels.get_raw(&node_bytes),
- ) {
- (Some(node_range_bytes), Some(edge_range_bytes)) => {
- let node_range = EdgeRange::deserialize(node_range_bytes.as_bytes());
- let label_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());
-
- let labels = self
- .edge_labels
- .slice(usize_range(label_range))
- .map(|r| r.decompress())
- .flat_map(|block| block.labels.into_iter());
-
- let edges = self.edges.slice(usize_range(node_range.range));
-
- Box::new(labels.zip_eq(edges).map(move |(label, edge)| {
- if self.reversed {
- SegmentEdge {
- from: edge.other,
- to: NodeDatum::new(*node, node_range.host_rank),
- rel: edge.rel,
- label,
- }
- } else {
- SegmentEdge {
- from: NodeDatum::new(*node, node_range.host_rank),
- to: edge.other,
- rel: edge.rel,
- label,
- }
- }
- }))
- }
- _ => Box::new(std::iter::empty()),
- }
- }
+ let doc = self.current_doc;
+ self.current_doc += 1;
- pub fn get_without_label<'a>(
- &'a self,
- node: &'a NodeID,
- ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {
- let node_bytes = node.as_u64().to_be_bytes();
-
- match self.ranges.edges.get_raw(&node_bytes) {
- Some(node_range_bytes) => {
- let edge_range = EdgeRange::deserialize(node_range_bytes.as_bytes());
-
- let edges = self.edges.slice(usize_range(edge_range.range));
-
- Box::new(edges.into_iter().map(move |edge| {
- if self.reversed {
- SegmentEdge {
- from: edge.other,
- to: NodeDatum::new(*node, edge_range.host_rank),
- rel: edge.rel,
- label: (),
- }
- } else {
- SegmentEdge {
- from: NodeDatum::new(*node, edge_range.host_rank),
- to: edge.other,
- rel: edge.rel,
- label: (),
- }
- }
- }))
- }
- _ => Box::new(std::iter::empty()),
- }
- }
+ let from = self.from_id.first(doc)?;
+ let to = self.to_id.first(doc)?;
+ let rel_flags = self.rel_flags.first(doc)?;
- pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {
- self.ranges.edges.iter_raw().flat_map(move |(key, val)| {
- let node = u64::from_be_bytes((key.as_bytes()).try_into().unwrap());
- let node = NodeID::from(node);
-
- let edge_range = EdgeRange::deserialize(val.as_bytes());
-
- let edges = self
- .edges
- .slice(usize_range(edge_range.range))
- .collect::<Vec<_>>();
-
- edges.into_iter().map(move |edge| {
- if self.reversed {
- SegmentEdge {
- from: edge.other,
- to: NodeDatum::new(node, edge_range.host_rank),
- rel: edge.rel,
- label: (),
- }
- } else {
- SegmentEdge {
- from: NodeDatum::new(node, edge_range.host_rank),
- to: edge.other,
- rel: edge.rel,
- label: (),
- }
- }
- })
+ Some(SmallEdge {
+ from: NodeID::from(from),
+ to: NodeID::from(to),
+ rel_flags: RelFlags::from(rel_flags),
})
}
}
#[cfg(test)]
mod tests {
- use std::sync::Arc;
-
- use crate::webgraph::{store_writer::EdgeStoreWriter, Edge, FullNodeID, InsertableEdge};
+ use crate::webgraph::{query::backlink::HostBacklinksQuery, Edge, Node};
use super::*;
#[test]
fn test_insert() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut kv: EdgeStoreWriter =
- EdgeStoreWriter::new(&temp_dir, Compression::default(), false, None);
-
- let e = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- },
- to: FullNodeID {
- id: NodeID::from(1_u64),
- host: NodeID::from(0_u64),
- },
+ let mut store = EdgeStore::open(&temp_dir, ShardId::new(0)).unwrap();
+
+ let e = Edge {
+ from: Node::from("https://www.first.com").into_host(),
+ to: Node::from("https://www.second.com").into_host(),
label: "test".to_string(),
- rel: RelFlags::default(),
+ rel_flags: RelFlags::default(),
+ sort_score: 0.0,
};
+ let from_node_id = e.from.id();
+ let to_node_id = e.to.id();
- kv.put(e.clone());
+ store.insert(e.clone()).unwrap();
+ store.commit().unwrap();
- let store = kv.finalize();
-
- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();
+ let query = HostBacklinksQuery::new(to_node_id);
+ let edges = store.search(&query).unwrap();
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0].to.node(), &SegmentEdge::from(e.clone()).to.node());
+ assert_eq!(edges[0].from, from_node_id);
+ assert_eq!(edges[0].to, to_node_id);
- let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64)).collect();
+ let query = HostBacklinksQuery::new(from_node_id);
+ let edges = store.search(&query).unwrap();
assert_eq!(edges.len(), 0);
- let edges = store.iter_without_label().collect::<Vec<_>>();
+ let edges = store.iter_pages_small().collect::<Vec<_>>();
assert_eq!(edges.len(), 1);
}
- #[test]
- fn test_reversed() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut kv: EdgeStoreWriter =
- EdgeStoreWriter::new(&temp_dir, Compression::default(), true, None);
-
- let e = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- },
- to: FullNodeID {
- id: NodeID::from(1_u64),
- host: NodeID::from(0_u64),
- },
- label: "test".to_string(),
- rel: RelFlags::default(),
- };
-
- kv.put(e.clone());
-
- let store = kv.finalize();
-
- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();
- assert_eq!(edges.len(), 0);
-
- let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64)).collect();
- assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));
- }
-
#[test]
fn test_edge_ordering() {
let temp_dir = crate::gen_temp_dir().unwrap();
- let mut rank_store =
- speedy_kv::Db::open_or_create(temp_dir.as_ref().join("test-rank-store")).unwrap();
-
- rank_store.insert(NodeID::from(2_u64), 1).unwrap();
- rank_store.insert(NodeID::from(3_u64), 2).unwrap();
- rank_store.insert(NodeID::from(1_u64), 3).unwrap();
-
- rank_store.commit().unwrap();
-
- let mut kv: EdgeStoreWriter = EdgeStoreWriter::new(
- &temp_dir.as_ref().join("test-segment"),
- Compression::default(),
- true,
- Some(Arc::new(rank_store)),
- );
-
- let e1 = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(1_u64),
- host: NodeID::from(1_u64),
- },
- to: FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- },
- label: "1".to_string(),
- rel: RelFlags::default(),
+
+ let a = Node::from("https://www.first.com").into_host();
+ let b = Node::from("https://www.second.com").into_host();
+ let c = Node::from("https://www.third.com").into_host();
+ let d = Node::from("https://www.fourth.com").into_host();
+
+ let a_centrality = 1.0;
+ let b_centrality = 2.0;
+ let c_centrality = 4.0;
+ let d_centrality = 3.0;
+ let mut store =
+ EdgeStore::open(&temp_dir.as_ref().join("test-segment"), ShardId::new(0)).unwrap();
+
+ let e1 = Edge {
+ from: b.clone(),
+ to: a.clone(),
+ label: "test".to_string(),
+ rel_flags: RelFlags::default(),
+ sort_score: a_centrality + b_centrality,
};
- let e2 = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(2_u64),
- host: NodeID::from(2_u64),
- },
- to: FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- },
+ let e2 = Edge {
+ from: c.clone(),
+ to: a.clone(),
label: "2".to_string(),
- rel: RelFlags::default(),
+ rel_flags: RelFlags::default(),
+ sort_score: a_centrality + c_centrality,
};
- let e3 = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(3_u64),
- host: NodeID::from(3_u64),
- },
- to: FullNodeID {
- id: NodeID::from(0_u64),
- host: NodeID::from(0_u64),
- },
+ let e3 = Edge {
+ from: d.clone(),
+ to: a.clone(),
label: "3".to_string(),
- rel: RelFlags::default(),
+ rel_flags: RelFlags::default(),
+ sort_score: a_centrality + d_centrality,
};
- kv.put(e1.clone());
- kv.put(e2.clone());
- kv.put(e3.clone());
+ store.insert(e1.clone()).unwrap();
+ store.insert(e2.clone()).unwrap();
+ store.insert(e3.clone()).unwrap();
- let store = kv.finalize();
+ store.commit().unwrap();
- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();
+ let query = HostBacklinksQuery::new(a.id());
+ let edges = store.search(&query).unwrap();
assert_eq!(edges.len(), 3);
- assert_eq!(
- Edge::from(edges[0].clone()).from.node(),
- Edge::from(e2.clone()).from.node()
- );
- assert_eq!(
- Edge::from(edges[1].clone()).from.node(),
- Edge::from(e3.clone()).from.node()
- );
- assert_eq!(
- Edge::from(edges[2].clone()).from.node(),
- Edge::from(e1.clone()).from.node()
- );
- }
-
- #[test]
- fn test_degree() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut kv: EdgeStoreWriter =
- EdgeStoreWriter::new(&temp_dir, Compression::default(), false, None);
-
- let e1 = InsertableEdge {
- from: FullNodeID {
- id: NodeID::from(1_u64),
- host: NodeID::from(1_u64),
- },
- to: FullNodeID {
- id: NodeID::from(2_u64),
- host: NodeID::from(2_u64),
- },
- label: "1".to_string(),
- rel: RelFlags::default(),
- };
-
- kv.put(e1.clone());
-
- let store = kv.finalize();
-
- assert_eq!(store.degree(&NodeID::from(1_u64)), 1);
- assert_eq!(store.degree(&NodeID::from(2_u64)), 0);
+ assert_eq!(edges[0].from, e2.from.id());
+ assert_eq!(edges[1].from, e3.from.id());
+ assert_eq!(edges[2].from, e1.from.id());
}
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
deleted file mode 100644
index 71fefc60..00000000
--- a/crates/core/src/webgraph/store_writer.rs
+++ /dev/null
@@ -1,446 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-pub const MAX_BATCH_SIZE: usize = 3_000_000;
-
-use std::{
- collections::BTreeSet,
- fs::File,
- ops::Range,
- path::{Path, PathBuf},
- sync::Arc,
-};
-
-use itertools::Itertools;
-
-use crate::{
- webgraph::store::{EdgeRange, NUM_LABELS_PER_BLOCK},
- Result,
-};
-use file_store::{
- iterable::{
- ConstIterableStoreWriter, IterableStoreReader, IterableStoreWriter,
- SortedIterableStoreReader,
- },
- ConstSerializable,
-};
-
-use super::{
- store::{CompressedLabelBlock, EdgeStore, LabelBlock, RangesDb},
- Compression, EdgeLabel, InsertableEdge, NodeDatum, NodeID, StoredEdge,
-};
-
-#[derive(bincode::Encode, bincode::Decode)]
-struct SortableEdge<L: EdgeLabel> {
- sort_node: NodeID,
- secondary_node: NodeID,
- edge: InsertableEdge<L>,
-}
-
-impl<L: EdgeLabel> PartialOrd for SortableEdge<L> {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl<L: EdgeLabel> Ord for SortableEdge<L> {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.sort_node
- .cmp(&other.sort_node)
- .then(self.secondary_node.cmp(&other.secondary_node))
- }
-}
-
-impl<L: EdgeLabel> PartialEq for SortableEdge<L> {
- fn eq(&self, other: &Self) -> bool {
- self.sort_node == other.sort_node && self.secondary_node == other.secondary_node
- }
-}
-
-impl<L: EdgeLabel> Eq for SortableEdge<L> {}
-
-struct SortedEdgeIterator<M, D>
-where
- M: Iterator<Item = SortableEdge<String>>,
- D: Iterator<Item = SortableEdge<String>>,
-{
- mem: file_store::Peekable<M>,
- file_reader: file_store::Peekable<D>,
-}
-
-impl<M, D> Iterator for SortedEdgeIterator<M, D>
-where
- M: Iterator<Item = SortableEdge<String>>,
- D: Iterator<Item = SortableEdge<String>>,
-{
- type Item = SortableEdge<String>;
-
- fn next(&mut self) -> Option<Self::Item> {
- if let Some(edge) = self.mem.peek() {
- if let Some(file_edge) = self.file_reader.peek() {
- if edge.sort_node < file_edge.sort_node {
- self.mem.next()
- } else {
- self.file_reader.next()
- }
- } else {
- self.mem.next()
- }
- } else {
- self.file_reader.next()
- }
- }
-}
-
-pub struct EdgeStoreWriter {
- reversed: bool,
- path: PathBuf,
- edges: BTreeSet<SortableEdge<String>>,
- stored_writers: Vec<PathBuf>,
- compression: Compression,
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
-}
-
-impl EdgeStoreWriter {
- pub fn new<P: AsRef<Path>>(
- path: P,
- compression: Compression,
- reversed: bool,
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
- ) -> Self {
- let writer_path = path.as_ref().join("writer");
-
- if !writer_path.exists() {
- std::fs::create_dir_all(&writer_path).unwrap();
- }
-
- Self {
- edges: BTreeSet::new(),
- reversed,
- path: path.as_ref().to_path_buf(),
- compression,
- stored_writers: Vec::new(),
- host_centrality_rank_store,
- }
- }
-
- fn flush_to_file(&mut self) -> Result<()> {
- let file_path = self
- .path
- .join("writer")
- .join(format!("{}.store", self.stored_writers.len()));
- let file = File::create(&file_path)?;
-
- let mut writer = file_store::iterable::IterableStoreWriter::new(file);
-
- for edge in &self.edges {
- writer.write(edge)?;
- }
- writer.finalize()?;
-
- self.edges.clear();
-
- self.stored_writers.push(file_path);
-
- Ok(())
- }
-
- pub fn put(&mut self, edge: InsertableEdge<String>) {
- let (sort_node, secondary_node) = if self.reversed {
- (edge.to.id, edge.from.id)
- } else {
- (edge.from.id, edge.to.id)
- };
-
- self.edges.insert(SortableEdge {
- sort_node,
- secondary_node,
- edge,
- });
-
- if self.edges.len() >= MAX_BATCH_SIZE {
- self.flush_to_file().unwrap();
- }
- }
-
- fn sorted_edges(mut self) -> impl Iterator<Item = SortableEdge<String>> {
- let readers = self
- .stored_writers
- .iter()
- .map(|p| IterableStoreReader::open(p).unwrap())
- .collect();
- let file_reader = SortedIterableStoreReader::new(readers);
-
- let edges = std::mem::take(&mut self.edges);
-
- SortedEdgeIterator {
- mem: file_store::Peekable::new(edges.into_iter()),
- file_reader: file_store::Peekable::new(file_reader),
- }
- }
-
- pub fn finalize(mut self) -> EdgeStore {
- self.flush_to_file().unwrap();
-
- let mut final_writer = FinalEdgeStoreWriter::open(
- self.compression,
- self.reversed,
- self.host_centrality_rank_store.clone(),
- &self.path,
- );
-
- let mut store = final_writer.build_store(self.sorted_edges().dedup().map(|e| e.edge));
- store.optimize_read();
-
- store
- }
-}
-
-impl Drop for EdgeStoreWriter {
- fn drop(&mut self) {
- if self.path.join("writer").exists() {
- std::fs::remove_dir_all(self.path.join("writer")).unwrap();
- }
- }
-}
-
-struct FinalEdgeStoreWriter {
- ranges: RangesDb,
-
- labels: IterableStoreWriter<CompressedLabelBlock, File>,
- edges: ConstIterableStoreWriter<StoredEdge, File>,
-
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
-
- compression: Compression,
- reversed: bool,
-
- path: PathBuf,
-}
-
-impl FinalEdgeStoreWriter {
- fn open<P: AsRef<Path>>(
- compression: Compression,
- reversed: bool,
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
- path: P,
- ) -> Self {
- let ranges = RangesDb::open(path.as_ref().join("ranges"));
-
- let labels_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(path.as_ref().join("labels"))
- .unwrap();
- let labels = IterableStoreWriter::new(labels_file);
-
- let edges_file = File::options()
- .read(true)
- .create(true)
- .truncate(false)
- .write(true)
- .open(path.as_ref().join("edges"))
- .unwrap();
- let edges = ConstIterableStoreWriter::new(edges_file);
-
- Self {
- ranges,
- labels,
- edges,
- reversed,
- compression,
- path: path.as_ref().to_path_buf(),
- host_centrality_rank_store,
- }
- }
- /// Insert a batch of edges into the store.
- /// The edges *must* have been de-duplicated by their from/to node.
- /// I.e. if the store is not reversed, there should only ever be a single
- /// put for each from node, and vice versa.
- fn put_store(&mut self, edges: &mut [InsertableEdge<String>]) {
- if edges.is_empty() {
- return;
- }
-
- let node = if self.reversed {
- edges[0].to.clone()
- } else {
- edges[0].from.clone()
- };
-
- // order edges by centrality rank of the other node host
- if let Some(rank_store) = &self.host_centrality_rank_store {
- edges.sort_by_cached_key(|e| {
- if self.reversed {
- rank_store.get(&e.from.host).unwrap().unwrap_or(u64::MAX)
- } else {
- rank_store.get(&e.to.host).unwrap().unwrap_or(u64::MAX)
- }
- });
- }
-
- let node_bytes = node.id.as_u64().to_be_bytes().to_vec();
-
- debug_assert!(self.ranges.nodes_get_raw(&node_bytes).is_none());
- debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());
-
- let mut edge_labels = Vec::new();
- let mut stored_edges: Vec<StoredEdge> = Vec::new();
-
- for edge in edges {
- edge_labels.push(edge.label.clone());
-
- let (node, host) = if self.reversed {
- (edge.from.id, edge.from.host)
- } else {
- (edge.to.id, edge.to.host)
- };
-
- let host_rank = self
- .host_centrality_rank_store
- .as_ref()
- .map(|store| store.get(&host).unwrap().unwrap_or(u64::MAX))
- .unwrap_or(u64::MAX);
-
- let datum = NodeDatum::new(node, host_rank);
- let rel = edge.rel;
- stored_edges.push(StoredEdge::new(datum, rel));
- }
-
- let edge_labels: Vec<_> = edge_labels
- .into_iter()
- .chunks(NUM_LABELS_PER_BLOCK)
- .into_iter()
- .map(|chunk| LabelBlock::new(chunk.collect()).compress(self.compression))
- .collect();
-
- let mut first_label_offset = None;
- let mut last_label_offset = None;
- let mut first_node_offset = None;
- let mut last_node_offset = None;
-
- for block in &edge_labels {
- let offset = self.labels.write(block).unwrap();
-
- if first_label_offset.is_none() {
- first_label_offset = Some(offset);
- }
-
- last_label_offset = Some(offset);
- }
-
- for node in &stored_edges {
- let offset = self.edges.write(node).unwrap();
-
- if first_node_offset.is_none() {
- first_node_offset = Some(offset);
- }
-
- last_node_offset = Some(offset);
- }
-
- let label_range = Range {
- start: first_label_offset.unwrap().start,
- end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,
- };
-
- let sort_key = self
- .host_centrality_rank_store
- .as_ref()
- .map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX))
- .unwrap_or(u64::MAX);
-
- let node_range: EdgeRange = EdgeRange::new(
- Range {
- start: first_node_offset.unwrap().start,
- end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
- },
- sort_key,
- );
- let node_range_bytes = node_range.serialize_to_vec();
-
- self.ranges
- .insert_raw_node(node_bytes.clone(), node_range_bytes);
-
- let label_range_bytes = label_range.serialize_to_vec();
-
- self.ranges.insert_raw_label(node_bytes, label_range_bytes);
- }
-
- /// Build a new edge store from a set of edges.
- ///
- /// **IMPORTANT** The edges must be sorted by
- /// either the from or to node, depending on the value of `reversed`.
- pub fn build_store(
- &mut self,
- edges: impl Iterator<Item = InsertableEdge<String>>,
- ) -> EdgeStore {
- let mut inserts_since_last_flush = 0;
-
- // create batches of consecutive edges with the same from/to node
- let mut batch = Vec::new();
- let mut last_node = None;
- for edge in edges {
- if let Some(last_node) = last_node {
- if (self.reversed && edge.to.id != last_node)
- || (!self.reversed && edge.from.id != last_node)
- {
- batch.sort_unstable_by_key(
- |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },
- );
- batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
- let batch_len = batch.len();
- self.put_store(&mut batch);
- batch.clear();
- inserts_since_last_flush += batch_len;
-
- if inserts_since_last_flush >= 100_000_000 {
- self.flush();
- inserts_since_last_flush = 0;
- }
- }
- }
-
- last_node = Some(if self.reversed {
- edge.to.id
- } else {
- edge.from.id
- });
- batch.push(edge);
- }
-
- if !batch.is_empty() {
- batch.sort_unstable_by_key(
- |e: &InsertableEdge<_>| if self.reversed { e.from.id } else { e.to.id },
- );
- batch.dedup_by_key(|e| if self.reversed { e.from.id } else { e.to.id });
- self.put_store(&mut batch);
- }
-
- self.flush();
-
- EdgeStore::open(&self.path, self.reversed)
- }
-
- fn flush(&mut self) {
- self.ranges.commit();
-
- self.edges.flush().unwrap();
- self.labels.flush().unwrap();
- }
-}
diff --git a/crates/core/src/webgraph/tests.rs b/crates/core/src/webgraph/tests.rs
new file mode 100644
index 00000000..e192c17f
--- /dev/null
+++ b/crates/core/src/webgraph/tests.rs
@@ -0,0 +1,797 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use super::query;
+
+use super::*;
+
+impl Webgraph {
+ pub fn ingoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<Edge> {
+ self.search(&query::FullBacklinksQuery::new(node).with_limit(limit))
+ .unwrap_or_default()
+ }
+
+ pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<SmallEdge> {
+ self.search(&query::BacklinksQuery::new(*node).with_limit(limit))
+ .unwrap_or_default()
+ }
+
+ pub fn raw_ingoing_edges_with_labels(
+ &self,
+ node: &NodeID,
+ limit: EdgeLimit,
+ ) -> Vec<SmallEdgeWithLabel> {
+ self.search(&query::BacklinksWithLabelsQuery::new(*node).with_limit(limit))
+ .unwrap_or_default()
+ }
+
+ pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<Edge> {
+ self.search(&query::FullForwardlinksQuery::new(node).with_limit(limit))
+ .unwrap_or_default()
+ }
+
+ pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<SmallEdge> {
+ self.search(&query::ForwardlinksQuery::new(*node).with_limit(limit))
+ .unwrap_or_default()
+ }
+}
+
+use crate::config::WebgraphGranularity;
+use crate::webpage::html::links::RelFlags;
+
+use file_store::temp::TempDir;
+use proptest::prelude::*;
+
+pub fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ (Node::from("D"), Node::from("C"), String::new()),
+ ]
+}
+
+pub fn test_graph() -> (Webgraph, TempDir) {
+ // ┌-----┐
+ // │ │
+ // ┌───A◄─┐ │
+ // │ │ │
+ // ▼ │ │
+ // B─────►C◄-┘
+ // ▲
+ // │
+ // │
+ // D
+
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, label) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+}
+
+#[test]
+fn distance_calculation() {
+ let (graph, _temp_dir) = test_graph();
+
+ let distances = graph.distances(Node::from("D"), WebgraphGranularity::Page);
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
+}
+
+#[test]
+fn nonexisting_node() {
+ let (graph, _temp_dir) = test_graph();
+ assert_eq!(
+ graph
+ .distances(Node::from("E"), WebgraphGranularity::Page)
+ .len(),
+ 0
+ );
+ assert_eq!(
+ graph
+ .reversed_distances(Node::from("E"), WebgraphGranularity::Page)
+ .len(),
+ 0
+ );
+}
+
+#[test]
+fn reversed_distance_calculation() {
+ let (graph, _temp_dir) = test_graph();
+
+ let distances = graph.reversed_distances(Node::from("D"), WebgraphGranularity::Page);
+
+ assert_eq!(distances.get(&Node::from("C")), None);
+ assert_eq!(distances.get(&Node::from("A")), None);
+ assert_eq!(distances.get(&Node::from("B")), None);
+
+ let distances = graph.reversed_distances(Node::from("A"), WebgraphGranularity::Page);
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("D")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&2));
+}
+
+#[test]
+fn merge_path() {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip([
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("D"), String::new()),
+ (Node::from("D"), Node::from("E"), String::new()),
+ (Node::from("E"), Node::from("F"), String::new()),
+ (Node::from("F"), Node::from("G"), String::new()),
+ (Node::from("G"), Node::from("H"), String::new()),
+ ]) {
+ let mut graph =
+ Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into())
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.optimize_read().unwrap();
+
+ assert_eq!(
+ graph
+ .distances(Node::from("A"), WebgraphGranularity::Page)
+ .get(&Node::from("H")),
+ Some(&7)
+ );
+
+ assert_eq!(
+ graph
+ .reversed_distances(Node::from("H"), WebgraphGranularity::Page)
+ .get(&Node::from("A")),
+ Some(&7)
+ );
+}
+
+#[test]
+fn merge_simple() {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip(test_edges()) {
+ let mut graph =
+ Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into())
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.optimize_read().unwrap();
+
+ let mut res = graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+ assert_eq!(res.len(), 2);
+
+ assert_eq!(res[0].to, Node::from("B"));
+ assert_eq!(res[1].to, Node::from("C"));
+
+ let mut res = graph.outgoing_edges(Node::from("B"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("C"));
+
+ let mut res = graph.outgoing_edges(Node::from("C"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("A"));
+
+ let mut res = graph.outgoing_edges(Node::from("D"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.to.cmp(&b.to));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("C"));
+
+ let mut res = graph.ingoing_edges(Node::from("A"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("C"));
+
+ let mut res = graph.ingoing_edges(Node::from("B"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("A"));
+
+ let mut res = graph.ingoing_edges(Node::from("C"), EdgeLimit::Unlimited);
+ res.sort_by(|a, b| a.from.cmp(&b.from));
+
+ assert_eq!(res.len(), 3);
+ assert_eq!(res[0].from, Node::from("A"));
+ assert_eq!(res[1].from, Node::from("B"));
+ assert_eq!(res[2].from, Node::from("D"));
+}
+
+#[test]
+fn merge_cycle() {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip([
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("C"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ ]) {
+ let mut graph =
+ Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into())
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ assert_eq!(
+ graph
+ .distances(Node::from("A"), WebgraphGranularity::Page)
+ .get(&Node::from("C")),
+ Some(&2)
+ );
+
+ graph.optimize_read().unwrap();
+
+ assert_eq!(
+ graph
+ .distances(Node::from("A"), WebgraphGranularity::Page)
+ .get(&Node::from("C")),
+ Some(&2)
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+}
+
+#[test]
+fn merge_star() {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip([
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ (Node::from("A"), Node::from("D"), String::new()),
+ (Node::from("A"), Node::from("E"), String::new()),
+ ]) {
+ let mut graph =
+ Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into())
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.optimize_read().unwrap();
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 4
+ );
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("D"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+}
+
+#[test]
+fn merge_reverse_star() {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ for (i, (from, to, label)) in (0..).zip([
+ (Node::from("B"), Node::from("A"), String::new()),
+ (Node::from("C"), Node::from("A"), String::new()),
+ (Node::from("D"), Node::from("A"), String::new()),
+ (Node::from("E"), Node::from("A"), String::new()),
+ ]) {
+ let mut graph =
+ Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into())
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ graph.optimize_read().unwrap();
+
+ assert_eq!(
+ graph
+ .ingoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 4
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("B"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("C"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("D"), EdgeLimit::Unlimited)
+ .len(),
+ 1
+ );
+}
+
+proptest! {
+ #![proptest_config(ProptestConfig::with_cases(8))]
+
+ #[test]
+ fn prop_merge(
+ nodes in
+ proptest::collection::vec(
+ ("[a-z]", "[a-z]"), 0..100
+ )
+ ) {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()), 0u64.into())
+ .open()
+ .unwrap();
+ for (from, to) in nodes.clone() {
+ graph.insert(Edge {
+ from: Node::new_for_test(from.as_str()),
+ to: Node::new_for_test(to.as_str()),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ }).unwrap();
+
+ if rand::random::<usize>() % 10 == 0 {
+ graph.commit().unwrap();
+ graphs.push(graph);
+ graph = Webgraph::builder(&temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()), 0u64.into())
+ .open()
+ .unwrap();
+ }
+ }
+
+ if graphs.is_empty() {
+ return Ok(());
+ }
+
+ graphs.push(graph);
+
+ let mut graph = graphs.pop().unwrap();
+
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+
+ for (from, to) in nodes {
+ if from == to {
+ continue;
+ }
+
+ let from = Node::new_for_test(from.as_str());
+ let to = Node::new_for_test(to.as_str());
+
+ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
+
+ prop_assert!(outgoing.iter().any(|e| e.to == to));
+ prop_assert!(ingoing.iter().any(|e| e.from == from));
+ }
+ }
+}
+
+fn proptest_case(nodes: &[(&str, &str)]) {
+ let mut graphs = Vec::new();
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(
+ &temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ 0u64.into(),
+ )
+ .open()
+ .unwrap();
+
+ for (i, (from, to)) in nodes.iter().enumerate() {
+ graph
+ .insert(Edge {
+ from: Node::new_for_test(from),
+ to: Node::new_for_test(to),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ if i % 2 == 0 {
+ graph.commit().unwrap();
+ graphs.push(graph);
+ graph = Webgraph::builder(
+ &temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ 0u64.into(),
+ )
+ .open()
+ .unwrap();
+ }
+ }
+
+ graphs.push(graph);
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ for (from, to) in nodes {
+ if from == to {
+ continue;
+ }
+
+ let from = Node::new_for_test(from);
+ let to = Node::new_for_test(to);
+
+ let outgoing = graph.outgoing_edges(from.clone(), EdgeLimit::Unlimited);
+ let ingoing = graph.ingoing_edges(to.clone(), EdgeLimit::Unlimited);
+
+ assert!(outgoing.iter().any(|e| e.to == to));
+ assert!(ingoing.iter().any(|e| e.from == from));
+ }
+}
+
+#[test]
+fn merge_proptest_case1() {
+ let nodes = [("k", "d"), ("k", "t"), ("y", "m")];
+ proptest_case(&nodes);
+}
+
+#[test]
+fn merge_proptest_case2() {
+ let nodes = [("i", "k"), ("k", "g"), ("y", "m"), ("q", "r"), ("e", "g")];
+
+ proptest_case(&nodes);
+}
+
+#[test]
+fn merge_proptest_case3() {
+ let nodes = [("h", "c"), ("r", "r")];
+
+ proptest_case(&nodes);
+}
+
+#[test]
+fn node_lowercase_name() {
+ let n = Node::from("TEST".to_string());
+ assert_eq!(n.as_str(), "test");
+}
+
+#[test]
+fn host_node_cleanup() {
+ let n = Node::from("https://www.example.com?test").into_host();
+ assert_eq!(n.as_str(), "example.com");
+}
+
+#[test]
+fn remove_protocol() {
+ let n = Node::from("https://www.example.com/?test");
+
+ assert_eq!(n.as_str(), "example.com/?test=");
+}
+
+#[test]
+fn cap_label_length() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(
+ &temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ 0u64.into(),
+ )
+ .open()
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::default(),
+ label: "a".repeat(MAX_LABEL_LENGTH + 1),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ assert_eq!(
+ graph.outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)[0].label,
+ "a".repeat(MAX_LABEL_LENGTH)
+ );
+}
+
+#[test]
+fn test_edge_limits() {
+ let (graph, temp_dir) = test_graph();
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
+ );
+
+ let mut graphs = Vec::new();
+ for (from, to, label) in &[
+ (Node::from("A"), Node::from("B"), String::new()),
+ (Node::from("A"), Node::from("C"), String::new()),
+ ] {
+ let mut graph = Webgraph::builder(
+ &temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ 0u64.into(),
+ )
+ .open()
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: from.clone(),
+ to: to.clone(),
+ rel_flags: RelFlags::default(),
+ label: label.clone(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+ graphs.push(graph);
+ }
+
+ let mut graph = graphs.pop().unwrap();
+
+ for other in graphs {
+ graph.merge(other).unwrap();
+ }
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
+ );
+
+ graph.optimize_read().unwrap();
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Unlimited)
+ .len(),
+ 2
+ );
+
+ assert_eq!(
+ graph
+ .outgoing_edges(Node::from("A"), EdgeLimit::Limit(1))
+ .len(),
+ 1
+ );
+}
+
+#[test]
+fn test_node_normalized() {
+ let n = Node::from("http://www.example.com/abc");
+ assert_eq!(n.as_str(), "example.com/abc");
+
+ let n = Node::from("http://www.example.com/abc#123");
+ assert_eq!(n.as_str(), "example.com/abc");
+}
+
+#[test]
+fn test_rel_flags() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(
+ &temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),
+ 0u64.into(),
+ )
+ .open()
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("A"),
+ to: Node::from("B"),
+ rel_flags: RelFlags::IS_IN_FOOTER | RelFlags::TAG,
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ assert_eq!(
+ graph.raw_outgoing_edges(&Node::from("A").id(), EdgeLimit::Unlimited)[0].rel_flags,
+ RelFlags::IS_IN_FOOTER | RelFlags::TAG,
+ );
+}
+
+#[test]
+fn test_limit_and_offset() {
+ let (graph, _temp_dir) = test_graph();
+
+ let no_offset = graph.raw_outgoing_edges(
+ &Node::from("A").id(),
+ EdgeLimit::LimitAndOffset {
+ limit: 2,
+ offset: 0,
+ },
+ );
+ assert_eq!(no_offset.len(), 2);
+
+ let edges = graph.raw_outgoing_edges(
+ &Node::from("A").id(),
+ EdgeLimit::LimitAndOffset {
+ limit: 2,
+ offset: 1,
+ },
+ );
+ assert_eq!(edges.len(), 1);
+ assert_eq!(edges[0].to, no_offset[1].to);
+}
diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rs
deleted file mode 100644
index 09028264..00000000
--- a/crates/core/src/webgraph/writer.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses
-
-use std::{fs, path::Path, sync::Arc};
-
-use crate::{executor::Executor, webpage::html::links::RelFlags};
-
-use super::{
- id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InsertableEdge, Meta,
- Node, NodeID, Webgraph, MAX_LABEL_LENGTH,
-};
-
-pub struct WebgraphWriter {
- pub path: String,
- segment: SegmentWriter,
- id2node: Id2NodeDb,
- executor: Executor,
- meta: Meta,
-}
-
-impl WebgraphWriter {
- fn meta<P: AsRef<Path>>(path: P) -> Meta {
- let meta_path = path.as_ref().join("metadata.json");
- Meta::open(meta_path)
- }
-
- fn save_metadata(&mut self) {
- let path = Path::new(&self.path).join("metadata.json");
- self.meta.save(path);
- }
-
- pub fn new<P: AsRef<Path>>(
- path: P,
- executor: Executor,
- compression: Compression,
- host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,
- ) -> Self {
- fs::create_dir_all(&path).unwrap();
- let mut meta = Self::meta(&path);
- meta.comitted_segments.clear();
-
- fs::create_dir_all(path.as_ref().join("segments")).unwrap();
-
- let id = uuid::Uuid::new_v4().to_string();
- let segment = SegmentWriter::open(
- path.as_ref().join("segments"),
- id.clone(),
- compression,
- host_centrality_rank_store,
- );
-
- meta.comitted_segments.push(id);
-
- Self {
- path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
- segment,
- id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
- executor,
- meta,
- }
- }
-
- pub fn id2node(&self, id: &NodeID) -> Option<Node> {
- self.id2node.get(id)
- }
-
- fn id_or_assign(&mut self, node: Node) -> FullNodeID {
- let id = FullNodeID::from(node.clone());
-
- self.id2node.put(&id.id, &node);
-
- id
- }
-
- pub fn insert(&mut self, from: Node, to: Node, label: String, rel: RelFlags) {
- if from == to {
- return;
- }
-
- let (from_id, to_id) = (
- self.id_or_assign(from.clone()),
- self.id_or_assign(to.clone()),
- );
-
- let edge = InsertableEdge {
- from: from_id,
- to: to_id,
- label: label.chars().take(MAX_LABEL_LENGTH).collect(),
- rel,
- };
-
- self.segment.insert(edge);
- }
-
- pub fn commit(&mut self) {
- self.save_metadata();
- self.id2node.flush();
- }
-
- pub fn finalize(mut self) -> Webgraph {
- self.commit();
-
- Webgraph {
- path: self.path,
- segments: vec![self.segment.finalize()],
- executor: self.executor.into(),
- id2node: self.id2node,
- meta: self.meta,
- }
- }
-}
diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rs
index 3df237ee..65fdb040 100644
--- a/crates/core/src/webpage/html/links.rs
+++ b/crates/core/src/webpage/html/links.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -39,11 +39,24 @@ pub struct ImageLink {
/// Flags for the `rel` attribute of the `a` element.
/// See https://html.spec.whatwg.org/multipage/links.html#linkTypes
-#[derive(Default, Debug, Clone, Copy, bincode::Encode, bincode::Decode, PartialEq, Eq, Hash)]
-pub struct RelFlags(u32);
+#[derive(
+ Default,
+ Debug,
+ Clone,
+ Copy,
+ bincode::Encode,
+ bincode::Decode,
+ serde::Serialize,
+ serde::Deserialize,
+ PartialEq,
+ Eq,
+ Hash,
+ utoipa::ToSchema,
+)]
+pub struct RelFlags(u64);
impl RelFlags {
- pub fn as_u32(&self) -> u32 {
+ pub fn as_u64(&self) -> u64 {
self.0
}
@@ -91,14 +104,14 @@ impl RelFlags {
}
}
-impl From<u32> for RelFlags {
- fn from(value: u32) -> Self {
+impl From<u64> for RelFlags {
+ fn from(value: u64) -> Self {
Self(value)
}
}
bitflags! {
- impl RelFlags: u32 {
+ impl RelFlags: u64 {
const ALTERNATE = 1 << 0;
const AUTHOR = 1 << 1;
const CANONICAL = 1 << 2;
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index 707afa89..8c4b8a4e 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -18,7 +18,7 @@ use crate::{
backlink_grouper::GroupedBacklinks,
inverted_index::InvertedIndex,
schema::{numerical_field::NumericalField, text_field::TextField, Field},
- webgraph::{Edge, NodeID},
+ webgraph::{NodeID, SmallEdgeWithLabel},
Result,
};
use candle_core::Tensor;
@@ -35,7 +35,7 @@ pub mod region;
pub mod safety_classifier;
pub mod schema_org;
pub mod url_ext;
-use self::html::links::RelFlags;
+pub use self::html::links::RelFlags;
pub use self::html::Html;
pub use region::Region;
@@ -45,10 +45,10 @@ pub struct Webpage {
pub html: Html,
#[cfg(test)]
- pub backlinks: Vec<Edge<String>>,
+ pub backlinks: Vec<SmallEdgeWithLabel>,
#[cfg(not(test))]
- backlinks: Vec<Edge<String>>,
+ backlinks: Vec<SmallEdgeWithLabel>,
#[cfg(test)]
pub grouped_backlinks: GroupedBacklinks,
@@ -145,11 +145,16 @@ impl Webpage {
})
}
- pub fn backlinks(&self) -> &[Edge<String>] {
+ pub fn backlinks(&self) -> &[SmallEdgeWithLabel] {
&self.backlinks
}
- pub fn set_backlinks(&mut self, backlinks: Vec<Edge<String>>) {
+ pub fn set_backlinks(&mut self, backlinks: Vec<SmallEdgeWithLabel>) {
+ let backlinks = backlinks
+ .into_iter()
+ .filter(|e| !e.rel_flags.contains(RelFlags::NOFOLLOW))
+ .collect();
+
self.backlinks = backlinks;
}
diff --git a/crates/tantivy/Cargo.toml b/crates/tantivy/Cargo.toml
index 84377f2e..a5048653 100644
--- a/crates/tantivy/Cargo.toml
+++ b/crates/tantivy/Cargo.toml
@@ -19,25 +19,26 @@ async-trait.workspace = true
base64.workspace = true
bitpacking.workspace = true
byteorder.workspace = true
+bincode.workspace = true
census.workspace = true
crc32fast.workspace = true
crossbeam-channel.workspace = true
downcast-rs.workspace = true
-fail = {workspace = true, optional = true}
+fail = { workspace = true, optional = true }
fastdivide.workspace = true
fnv.workspace = true
-fs4 = {workspace = true, optional = true}
+fs4 = { workspace = true, optional = true }
fst.workspace = true
-futures-util = {workspace = true, optional = true}
+futures-util = { workspace = true, optional = true }
htmlescape.workspace = true
itertools.workspace = true
lending-iter.workspace = true
levenshtein_automata.workspace = true
log.workspace = true
lru.workspace = true
-lz4_flex = {workspace = true, optional = true}
+lz4_flex = { workspace = true, optional = true }
measure_time.workspace = true
-memmap2 = {workspace = true, optional = true}
+memmap2 = { workspace = true, optional = true }
murmurhash32.workspace = true
nom.workspace = true
oneshot.workspace = true
@@ -50,7 +51,7 @@ serde.workspace = true
serde_json.workspace = true
smallvec.workspace = true
strum.workspace = true
-tempfile = {workspace = true, optional = true}
+tempfile = { workspace = true, optional = true }
thiserror.workspace = true
time.workspace = true
uuid.workspace = true
diff --git a/crates/tantivy/src/core/mod.rs b/crates/tantivy/src/core/mod.rs
index da0e7200..fe1defa8 100644
--- a/crates/tantivy/src/core/mod.rs
+++ b/crates/tantivy/src/core/mod.rs
@@ -10,6 +10,8 @@ use std::sync::LazyLock;
pub use self::executor::Executor;
pub use self::searcher::{Searcher, SearcherGeneration};
+pub use std::result;
+
/// The meta file contains all the information about the list of segments and the schema
/// of the index.
pub static META_FILEPATH: LazyLock<&'static Path> = LazyLock::new(|| Path::new("meta.json"));
diff --git a/crates/tantivy/src/docset.rs b/crates/tantivy/src/docset.rs
index 7b480590..04ca4209 100644
--- a/crates/tantivy/src/docset.rs
+++ b/crates/tantivy/src/docset.rs
@@ -152,3 +152,36 @@ impl<TDocSet: DocSet + ?Sized> DocSet for Box<TDocSet> {
unboxed.count()
}
}
+
+pub struct DocSetIter<T>
+where
+ T: DocSet,
+{
+ docset: T,
+}
+
+impl<T> From<T> for DocSetIter<T>
+where
+ T: DocSet,
+{
+ fn from(docset: T) -> Self {
+ DocSetIter { docset }
+ }
+}
+
+impl<T> Iterator for DocSetIter<T>
+where
+ T: DocSet,
+{
+ type Item = DocId;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ let doc = self.docset.doc();
+ if doc == TERMINATED {
+ None
+ } else {
+ self.docset.advance();
+ Some(doc)
+ }
+ }
+}
diff --git a/crates/tantivy/src/index/mod.rs b/crates/tantivy/src/index/mod.rs
index fcd9a147..309cc782 100644
--- a/crates/tantivy/src/index/mod.rs
+++ b/crates/tantivy/src/index/mod.rs
@@ -10,6 +10,10 @@ mod segment_component;
mod segment_id;
mod segment_reader;
+use std::path::Path;
+
+use crate::IndexWriter;
+
pub use self::index::{Index, IndexBuilder};
pub(crate) use self::index_meta::SegmentMetaInventory;
pub use self::index_meta::{IndexMeta, IndexSettings, IndexSortByField, Order, SegmentMeta};
@@ -18,3 +22,60 @@ pub use self::segment::Segment;
pub use self::segment_component::SegmentComponent;
pub use self::segment_id::SegmentId;
pub use self::segment_reader::{FieldMetadata, SegmentReader};
+
+struct SegmentMergeCandidate {
+ num_docs: u32,
+ segments: Vec<SegmentMeta>,
+}
+
+pub fn merge_segments<P: AsRef<Path>, D: crate::Document>(
+ writer: &mut IndexWriter<D>,
+ mut segments: Vec<SegmentMeta>,
+ base_path: P,
+ max_num_segments: u64,
+) -> crate::Result<()> {
+ assert!(max_num_segments > 0);
+
+ if segments.len() <= max_num_segments as usize {
+ return Ok(());
+ }
+
+ let num_segments = (max_num_segments + 1) / 2; // ceil(num_segments/2)
+
+ let mut merge_segments = Vec::new();
+
+ for _ in 0..num_segments {
+ merge_segments.push(SegmentMergeCandidate {
+ num_docs: 0,
+ segments: Vec::new(),
+ });
+ }
+
+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
+
+ for segment in segments {
+ let best_candidate = merge_segments
+ .iter_mut()
+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
+ .unwrap();
+
+ best_candidate.num_docs += segment.num_docs();
+ best_candidate.segments.push(segment);
+ }
+
+ for merge in merge_segments
+ .into_iter()
+ .filter(|merge| !merge.segments.is_empty())
+ {
+ let segment_ids: Vec<_> = merge.segments.iter().map(|segment| segment.id()).collect();
+ writer.merge(&segment_ids[..]).wait()?;
+
+ for segment in merge.segments {
+ for file in segment.list_files() {
+ std::fs::remove_file(base_path.as_ref().join(file)).ok();
+ }
+ }
+ }
+
+ Ok(())
+}
diff --git a/crates/tantivy/src/index/segment_reader.rs b/crates/tantivy/src/index/segment_reader.rs
index f73554e1..ddd42acb 100644
--- a/crates/tantivy/src/index/segment_reader.rs
+++ b/crates/tantivy/src/index/segment_reader.rs
@@ -342,8 +342,8 @@ impl SegmentReader {
}
/// Returns an iterator that will iterate over the document ids
- pub fn doc_ids(&self) -> Box<dyn Iterator<Item = DocId> + Send + '_> {
- Box::new(0u32..self.max_doc)
+ pub fn doc_ids(&self) -> impl Iterator<Item = DocId> + Send + '_ {
+ 0u32..self.max_doc
}
/// Summarize total space usage of this segment.
diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rs
index d1085af1..18093e62 100644
--- a/crates/tantivy/src/lib.rs
+++ b/crates/tantivy/src/lib.rs
@@ -141,8 +141,8 @@ pub use crate::core::json_utils;
pub use crate::core::{Executor, Searcher, SearcherGeneration};
pub use crate::directory::Directory;
pub use crate::index::{
- Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField, InvertedIndexReader, Order,
- Segment, SegmentMeta, SegmentReader,
+ merge_segments, Index, IndexBuilder, IndexMeta, IndexSettings, IndexSortByField,
+ InvertedIndexReader, Order, Segment, SegmentMeta, SegmentReader,
};
pub use crate::indexer::{IndexWriter, SingleSegmentIndexWriter};
pub use crate::schema::{Document, TantivyDocument, Term};
@@ -249,7 +249,20 @@ impl DocAddress {
///
/// The id used for the segment is actually an ordinal
/// in the list of `Segment`s held by a `Searcher`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[derive(
+ Debug,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+ Serialize,
+ Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+)]
pub struct DocAddress {
/// The segment ordinal id that identifies the segment
/// hosting the document in the `Searcher` it is called from.
diff --git a/crates/tantivy/src/postings/block_segment_postings.rs b/crates/tantivy/src/postings/block_segment_postings.rs
index cb175143..b13becd3 100644
--- a/crates/tantivy/src/postings/block_segment_postings.rs
+++ b/crates/tantivy/src/postings/block_segment_postings.rs
@@ -290,7 +290,7 @@ impl BlockSegmentPostings {
/// `.load_block()` needs to be called manually afterwards.
/// If all docs are smaller than target, the block loaded may be empty,
/// or be the last an incomplete VInt block.
- pub(crate) fn shallow_seek(&mut self, target_doc: DocId) {
+ pub fn shallow_seek(&mut self, target_doc: DocId) {
if self.skip_reader.seek(target_doc) {
self.block_max_score_cache = None;
self.block_loaded = false;
@@ -373,7 +373,7 @@ impl BlockSegmentPostings {
}
}
- pub(crate) fn skip_reader(&self) -> &SkipReader {
+ pub fn skip_reader(&self) -> &SkipReader {
&self.skip_reader
}
}
diff --git a/crates/tantivy/src/postings/segment_postings.rs b/crates/tantivy/src/postings/segment_postings.rs
index 36561860..fa7c6ac2 100644
--- a/crates/tantivy/src/postings/segment_postings.rs
+++ b/crates/tantivy/src/postings/segment_postings.rs
@@ -28,6 +28,15 @@ impl SegmentPostings {
}
}
+ /// Returns the underlying block segment postings
+ pub fn block_cursor(&self) -> &BlockSegmentPostings {
+ &self.block_cursor
+ }
+
+ pub fn mut_block_cursor(&mut self) -> &mut BlockSegmentPostings {
+ &mut self.block_cursor
+ }
+
/// Returns the overall number of documents in the block postings.
/// It does not take in account whether documents are deleted or not.
pub fn doc_freq(&self) -> u32 {
@@ -132,6 +141,10 @@ impl SegmentPostings {
position_reader,
}
}
+
+ pub fn reset_cursor_start_block(&mut self) {
+ self.cur = 0;
+ }
}
impl DocSet for SegmentPostings {
diff --git a/crates/tantivy/src/postings/skip.rs b/crates/tantivy/src/postings/skip.rs
index fe5a8df8..e2f664c1 100644
--- a/crates/tantivy/src/postings/skip.rs
+++ b/crates/tantivy/src/postings/skip.rs
@@ -82,7 +82,7 @@ impl SkipSerializer {
}
#[derive(Clone)]
-pub(crate) struct SkipReader {
+pub struct SkipReader {
last_doc_in_block: DocId,
pub(crate) last_doc_in_previous_block: DocId,
owned_read: OwnedBytes,
@@ -96,7 +96,7 @@ pub(crate) struct SkipReader {
}
#[derive(Clone, Eq, PartialEq, Copy, Debug)]
-pub(crate) enum BlockInfo {
+pub enum BlockInfo {
BitPacked {
doc_num_bits: u8,
strict_delta_encoded: bool,
@@ -170,7 +170,7 @@ impl SkipReader {
}
}
- pub(crate) fn last_doc_in_block(&self) -> DocId {
+ pub fn last_doc_in_block(&self) -> DocId {
self.last_doc_in_block
}
diff --git a/crates/tantivy/src/query/mod.rs b/crates/tantivy/src/query/mod.rs
index 4d6957a3..7da100bb 100644
--- a/crates/tantivy/src/query/mod.rs
+++ b/crates/tantivy/src/query/mod.rs
@@ -19,6 +19,7 @@ mod range_query;
mod reqopt_scorer;
mod scorer;
mod set_query;
+mod shortcircuit;
mod term_query;
mod union;
mod weight;
@@ -61,6 +62,7 @@ pub use self::score_combiner::{
};
pub use self::scorer::Scorer;
pub use self::set_query::TermSetQuery;
+pub use self::shortcircuit::ShortCircuitQuery;
pub use self::term_query::TermQuery;
pub use self::union::Union;
#[cfg(test)]
diff --git a/crates/core/src/query/shortcircuit.rs b/crates/tantivy/src/query/shortcircuit.rs
similarity index 72%
rename from crates/core/src/query/shortcircuit.rs
rename to crates/tantivy/src/query/shortcircuit.rs
index 6fa877a0..b0483588 100644
--- a/crates/core/src/query/shortcircuit.rs
+++ b/crates/tantivy/src/query/shortcircuit.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -13,7 +13,7 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::{
+use crate::{
query::{Explanation, Query, Scorer, Weight},
DocSet, Score,
};
@@ -45,26 +45,10 @@ impl Clone for ShortCircuitQuery {
impl Query for ShortCircuitQuery {
fn weight(
&self,
- enable_scoring: tantivy::query::EnableScoring,
- ) -> tantivy::Result<Box<dyn Weight>> {
- let schema = match enable_scoring {
- tantivy::query::EnableScoring::Enabled {
- searcher,
- statistics_provider: _,
- } => searcher.schema(),
- tantivy::query::EnableScoring::Disabled {
- schema,
- searcher_opt: _,
- } => schema,
- };
-
+ enable_scoring: crate::query::EnableScoring,
+ ) -> crate::Result<Box<dyn Weight>> {
Ok(Box::new(ShortCircuitWeight {
- subweight: self
- .subquery
- .weight(tantivy::query::EnableScoring::Disabled {
- schema,
- searcher_opt: None,
- })?,
+ subweight: self.subquery.weight(enable_scoring)?,
max_docs_per_segment: self.max_docs_per_segment,
}))
}
@@ -78,9 +62,9 @@ struct ShortCircuitWeight {
impl Weight for ShortCircuitWeight {
fn scorer(
&self,
- reader: &tantivy::SegmentReader,
+ reader: &crate::SegmentReader,
boost: Score,
- ) -> tantivy::Result<Box<dyn Scorer>> {
+ ) -> crate::Result<Box<dyn Scorer>> {
let subscorer = self.subweight.scorer(reader, boost)?;
Ok(Box::new(ShortCircuitScorer {
subscorer,
@@ -91,12 +75,12 @@ impl Weight for ShortCircuitWeight {
fn explain(
&self,
- reader: &tantivy::SegmentReader,
- doc: tantivy::DocId,
- ) -> tantivy::Result<tantivy::query::Explanation> {
+ reader: &crate::SegmentReader,
+ doc: crate::DocId,
+ ) -> crate::Result<crate::query::Explanation> {
let mut scorer = self.scorer(reader, 1.0)?;
if scorer.seek(doc) != doc {
- return Err(tantivy::TantivyError::InvalidArgument(format!(
+ return Err(crate::TantivyError::InvalidArgument(format!(
"Document #({doc}) does not match"
)));
}
@@ -125,18 +109,18 @@ impl Scorer for ShortCircuitScorer {
}
impl DocSet for ShortCircuitScorer {
- fn advance(&mut self) -> tantivy::DocId {
+ fn advance(&mut self) -> crate::DocId {
if self.num_docs >= self.max_docs_per_segment {
- return tantivy::TERMINATED;
+ return crate::TERMINATED;
}
self.num_docs += 1;
self.subscorer.advance()
}
- fn doc(&self) -> tantivy::DocId {
+ fn doc(&self) -> crate::DocId {
if self.num_docs >= self.max_docs_per_segment {
- return tantivy::TERMINATED;
+ return crate::TERMINATED;
}
self.subscorer.doc()
diff --git a/justfile b/justfile
index 7a193a29..f69517ae 100644
--- a/justfile
+++ b/justfile
@@ -13,7 +13,7 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
@dev-entity-search-server:
cargo watch -i frontend -i tools -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- entity-search-server configs/entity_search_server.toml"
@dev-webgraph:
- cargo watch -i frontend -i tools -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/host_server.toml" && cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/page_server.toml"
+ cargo watch -i frontend -i tools -i crates/client-wasm -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
@dev-frontend:
cd frontend && npm run dev