Skip to content

web search done right

deduplicate urls during indexing

Mikkel Denker385e8375c625
diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rs
index 19373be5..1e1019dc 100644
--- a/crates/core/examples/indexer.rs
+++ b/crates/core/examples/indexer.rs
@@ -1,7 +1,7 @@
use std::path::Path;
use clap::Parser;
-use stract::config::{IndexingGraphConfig, IndexingLocalConfig};
+use stract::config::{IndexerConfig, IndexerGraphConfig};
#[derive(Parser)]
struct Args {
@@ -20,7 +20,7 @@ fn main() -> anyhow::Result<()> {
println!("Indexing...");
let start = std::time::Instant::now();
- stract::entrypoint::indexer::run(&IndexingLocalConfig {
+ stract::entrypoint::indexer::run(&IndexerConfig {
output_path: path.to_str().unwrap().to_string(),
limit_warc_files: None,
skip_warc_files: None,
@@ -32,7 +32,7 @@ fn main() -> anyhow::Result<()> {
.unwrap()
.to_string()],
}),
- page_webgraph: Some(IndexingGraphConfig::Local {
+ page_webgraph: Some(IndexerGraphConfig::Local {
path: data_path
.join("webgraph_page")
.to_str()
@@ -56,7 +56,7 @@ fn main() -> anyhow::Result<()> {
stract::config::defaults::Indexing::autocommit_after_num_inserts(),
dual_encoder: args
.dual_encoder_path
- .map(|p| stract::config::IndexingDualEncoderConfig {
+ .map(|p| stract::config::IndexerDualEncoderConfig {
model_path: p,
page_centrality_rank_threshold: Some(1_000_000),
}),
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 7e388ab5..649a9505 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -26,12 +26,12 @@ use std::io::{self, BufRead};
use std::net::SocketAddr;
#[derive(Debug, serde::Deserialize, Clone)]
-pub struct IndexingLocalConfig {
+pub struct IndexerConfig {
pub output_path: String,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub warc_source: WarcSource,
- pub page_webgraph: Option<IndexingGraphConfig>,
+ pub page_webgraph: Option<IndexerGraphConfig>,
pub host_centrality_threshold: Option<f64>,
pub topics_path: Option<String>,
pub host_centrality_store_path: String,
@@ -45,18 +45,18 @@ pub struct IndexingLocalConfig {
#[serde(default = "defaults::Indexing::autocommit_after_num_inserts")]
pub autocommit_after_num_inserts: usize,
- pub dual_encoder: Option<IndexingDualEncoderConfig>,
+ pub dual_encoder: Option<IndexerDualEncoderConfig>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
#[serde(tag = "type")]
-pub enum IndexingGraphConfig {
+pub enum IndexerGraphConfig {
Local { path: String },
Remote { gossip: GossipConfig },
}
#[derive(Debug, serde::Deserialize, Clone)]
-pub struct IndexingDualEncoderConfig {
+pub struct IndexerDualEncoderConfig {
pub model_path: String,
/// Only compute embeddings for pages that has a
@@ -479,7 +479,7 @@ pub struct LiveIndexConfig {
// indexer
pub host_centrality_store_path: String,
- pub page_webgraph: Option<IndexingGraphConfig>,
+ pub page_webgraph: Option<IndexerGraphConfig>,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub host_centrality_threshold: Option<f64>,
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index af03e0a4..0594da26 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -20,7 +20,7 @@ use tokio_stream::StreamExt;
use tracing::{debug, info};
use crate::config::{
- defaults, IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LocalConfig,
+ defaults, IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LocalConfig,
WebSpellConfig,
};
use crate::entrypoint::indexer::JobSettings;
@@ -180,12 +180,12 @@ fn create_inverted_index() -> Result<()> {
let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");
let centrality_path = Path::new(DATA_PATH).join("centrality");
let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");
- let dual_encoder_path = Path::new(DATA_PATH).join("summarizer").join("dual_encoder");
+ let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");
- let worker = indexer::IndexingWorker::new(IndexingLocalConfig {
+ let worker = indexer::IndexingWorker::new(IndexerConfig {
host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),
page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),
- page_webgraph: Some(IndexingGraphConfig::Local {
+ page_webgraph: Some(IndexerGraphConfig::Local {
path: webgraph_path.to_str().unwrap().to_string(),
}),
topics_path: Some(
@@ -204,7 +204,7 @@ fn create_inverted_index() -> Result<()> {
minimum_clean_words: None,
batch_size: defaults::Indexing::batch_size(),
autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
- dual_encoder: Some(IndexingDualEncoderConfig {
+ dual_encoder: Some(IndexerDualEncoderConfig {
model_path: dual_encoder_path.to_str().unwrap().to_string(),
page_centrality_rank_threshold: Some(100_000),
}),
diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rs
index f5a3d9e8..a17bafa6 100644
--- a/crates/core/src/entrypoint/indexer/job.rs
+++ b/crates/core/src/entrypoint/indexer/job.rs
@@ -68,14 +68,16 @@ impl Job {
for file in warc_files.by_ref() {
let mut batch = Vec::with_capacity(self.settings.batch_size);
- for chunk in &file
+ for chunk in file
.records()
.flatten()
.filter(|record| match &record.response.payload_type {
Some(payload_type) => matches!(payload_type, PayloadType::Html),
None => true,
})
+ .filter(|record| !worker.see(&record.request.url))
.chunks(self.settings.batch_size)
+ .into_iter()
{
batch.clear();
diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rs
index d1e3ef88..04952402 100644
--- a/crates/core/src/entrypoint/indexer/mod.rs
+++ b/crates/core/src/entrypoint/indexer/mod.rs
@@ -40,7 +40,7 @@ impl From<String> for IndexPointer {
}
}
-pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
+pub fn run(config: &config::IndexerConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config: WarcSource = config.warc_source.clone();
@@ -62,7 +62,16 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
autocommit_after_num_inserts: config.autocommit_after_num_inserts,
},
})
- .map(|job| IndexPointer(job.process(&worker).path().as_os_str().to_str().unwrap().to_string()))
+ .map(|job| {
+ IndexPointer(
+ job.process(&worker)
+ .path()
+ .as_os_str()
+ .to_str()
+ .unwrap()
+ .to_string(),
+ )
+ })
.collect();
let index = merge(indexes)?;
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 91641771..45b39326 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -17,13 +17,14 @@
use chrono::Utc;
use itertools::Itertools;
use std::path::Path;
+use std::sync::Mutex;
use tracing::debug;
pub use super::indexable_webpage::IndexableWebpage;
pub use super::job::{Job, JobSettings};
use crate::config::{
- IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LiveIndexConfig,
+ IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,
WebgraphGranularity,
};
use crate::models::dual_encoder::DualEncoder as DualEncoderModel;
@@ -40,14 +41,14 @@ use crate::webpage::{safety_classifier, Html, Webpage};
pub struct Config {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
- pub page_webgraph: Option<IndexingGraphConfig>,
+ pub page_webgraph: Option<IndexerGraphConfig>,
pub topics_path: Option<String>,
pub safety_classifier_path: Option<String>,
- pub dual_encoder: Option<IndexingDualEncoderConfig>,
+ pub dual_encoder: Option<IndexerDualEncoderConfig>,
}
-impl From<IndexingLocalConfig> for Config {
- fn from(config: IndexingLocalConfig) -> Self {
+impl From<IndexerConfig> for Config {
+ fn from(config: IndexerConfig) -> Self {
Self {
host_centrality_store_path: config.host_centrality_store_path,
page_centrality_store_path: config.page_centrality_store_path,
@@ -141,14 +142,14 @@ impl Webgraph {
}
impl Webgraph {
- fn new(config: &IndexingGraphConfig) -> Self {
+ fn new(config: &IndexerGraphConfig) -> Self {
match config {
- IndexingGraphConfig::Local { path } => Self::Local(
+ IndexerGraphConfig::Local { path } => Self::Local(
webgraph::WebgraphBuilder::new(path)
.single_threaded()
.open(),
),
- IndexingGraphConfig::Remote { gossip } => {
+ IndexerGraphConfig::Remote { gossip } => {
let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);
Self::Remote(crate::block_on(RemoteWebgraph::new(
cluster,
@@ -170,6 +171,7 @@ pub struct IndexingWorker {
job_settings: Option<JobSettings>,
rake: RakeModel,
dual_encoder: Option<DualEncoder>,
+ seen_urls: Mutex<bloom::BytesBloomFilter<String>>,
}
impl IndexingWorker {
@@ -216,6 +218,7 @@ impl IndexingWorker {
page_centrality_rank_threshold: dual_encoder.page_centrality_rank_threshold,
}
}),
+ seen_urls: Mutex::new(bloom::BytesBloomFilter::new(10_000_000_000, 0.05)),
}
}
@@ -227,7 +230,20 @@ impl IndexingWorker {
self.page_webgraph.as_ref()
}
- pub fn process(&self, job: &Job) -> Index {
+ /// Returns false if the URL has not been seen before and marks it as seen.
+ /// Returns true if the URL has been seen before.
+ pub(super) fn see(&self, url: &String) -> bool {
+ let mut seen_urls = self.seen_urls.lock().unwrap();
+
+ if seen_urls.contains(url) {
+ true
+ } else {
+ seen_urls.insert(url);
+ false
+ }
+ }
+
+ pub fn process(&mut self, job: &Job) -> Index {
job.process(self)
}
@@ -395,7 +411,7 @@ impl IndexingWorker {
.map(|w| Node::from(w.html.url()).id())
.collect::<Vec<_>>();
- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(256));
+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(512));
for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {
page.backlink_labels = backlink;
@@ -495,13 +511,13 @@ mod tests {
use super::*;
fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {
- IndexingWorker::new(IndexingLocalConfig {
+ IndexingWorker::new(IndexerConfig {
host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
page_centrality_store_path: None,
page_webgraph: None,
topics_path: None,
safety_classifier_path: None,
- dual_encoder: Some(IndexingDualEncoderConfig {
+ dual_encoder: Some(IndexerDualEncoderConfig {
model_path: data_path.to_str().unwrap().to_string(),
page_centrality_rank_threshold: threshold,
}),
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 27a1f47d..42ef0156 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -183,6 +183,10 @@ impl Message<WebGraphService> for PagesByHosts {
pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
+ let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());
+
+ let server = WebGraphService { graph }.bind(addr).await.unwrap();
+
// dropping the handle leaves the cluster
let _cluster = Arc::new(
Cluster::join(
@@ -200,10 +204,6 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
.await?,
);
- let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());
-
- let server = WebGraphService { graph }.bind(addr).await.unwrap();
-
info!("webgraph server is ready to accept requests on {}", addr);
loop {
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index f2950901..1f28b9ad 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -125,7 +125,7 @@ mod tests {
use std::path::Path;
use crate::{
- config::{IndexingDualEncoderConfig, IndexingLocalConfig, WarcSource},
+ config::{IndexerConfig, IndexerDualEncoderConfig, WarcSource},
entrypoint::indexer::IndexingWorker,
index::Index,
models::dual_encoder::DualEncoder,
@@ -744,13 +744,13 @@ mod tests {
}
fn setup_worker(data_path: &Path) -> IndexingWorker {
- IndexingWorker::new(IndexingLocalConfig {
+ IndexingWorker::new(IndexerConfig {
host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
page_centrality_store_path: None,
page_webgraph: None,
topics_path: None,
safety_classifier_path: None,
- dual_encoder: Some(IndexingDualEncoderConfig {
+ dual_encoder: Some(IndexerDualEncoderConfig {
model_path: data_path.to_str().unwrap().to_string(),
page_centrality_rank_threshold: None,
}),
diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoder
index 663f653d..725b6de1 100755
--- a/scripts/export_dual_encoder
+++ b/scripts/export_dual_encoder
@@ -5,14 +5,14 @@ import transformers
MODEL = "sentence-transformers/all-MiniLM-L6-v2"
-if os.path.exists("data/summarizer/"):
- print("data/summarizer/ already exists. Exiting...")
+if os.path.exists("data/dual_encoder/"):
+ print("data/dual_encoder/ already exists. Exiting...")
sys.exit()
-os.system("mkdir -p data/summarizer/dual_encoder")
+os.system("mkdir -p data/dual_encoder")
model = transformers.AutoModel.from_pretrained(MODEL)
tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)
-model.save_pretrained("data/summarizer/dual_encoder")
-tokenizer.save_pretrained("data/summarizer/dual_encoder")
+model.save_pretrained("data/dual_encoder")
+tokenizer.save_pretrained("data/dual_encoder")