web search done right
deduplicate urls during indexing
Mikkel Denker385e8375c625
diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rsindex 19373be5..1e1019dc 100644--- a/crates/core/examples/indexer.rs+++ b/crates/core/examples/indexer.rs@@ -1,7 +1,7 @@use std::path::Path;use clap::Parser;-use stract::config::{IndexingGraphConfig, IndexingLocalConfig};+use stract::config::{IndexerConfig, IndexerGraphConfig};#[derive(Parser)]struct Args {@@ -20,7 +20,7 @@ fn main() -> anyhow::Result<()> {println!("Indexing...");let start = std::time::Instant::now();- stract::entrypoint::indexer::run(&IndexingLocalConfig {+ stract::entrypoint::indexer::run(&IndexerConfig {output_path: path.to_str().unwrap().to_string(),limit_warc_files: None,skip_warc_files: None,@@ -32,7 +32,7 @@ fn main() -> anyhow::Result<()> {.unwrap().to_string()],}),- page_webgraph: Some(IndexingGraphConfig::Local {+ page_webgraph: Some(IndexerGraphConfig::Local {path: data_path.join("webgraph_page").to_str()@@ -56,7 +56,7 @@ fn main() -> anyhow::Result<()> {stract::config::defaults::Indexing::autocommit_after_num_inserts(),dual_encoder: args.dual_encoder_path- .map(|p| stract::config::IndexingDualEncoderConfig {+ .map(|p| stract::config::IndexerDualEncoderConfig {model_path: p,page_centrality_rank_threshold: Some(1_000_000),}),diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 7e388ab5..649a9505 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -26,12 +26,12 @@ use std::io::{self, BufRead};use std::net::SocketAddr;#[derive(Debug, serde::Deserialize, Clone)]-pub struct IndexingLocalConfig {+pub struct IndexerConfig {pub output_path: String,pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,pub warc_source: WarcSource,- pub page_webgraph: Option<IndexingGraphConfig>,+ pub page_webgraph: Option<IndexerGraphConfig>,pub host_centrality_threshold: Option<f64>,pub topics_path: Option<String>,pub host_centrality_store_path: String,@@ -45,18 +45,18 @@ pub struct IndexingLocalConfig {#[serde(default = "defaults::Indexing::autocommit_after_num_inserts")]pub autocommit_after_num_inserts: usize,- pub dual_encoder: Option<IndexingDualEncoderConfig>,+ pub dual_encoder: Option<IndexerDualEncoderConfig>,}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]#[serde(tag = "type")]-pub enum IndexingGraphConfig {+pub enum IndexerGraphConfig {Local { path: String },Remote { gossip: GossipConfig },}#[derive(Debug, serde::Deserialize, Clone)]-pub struct IndexingDualEncoderConfig {+pub struct IndexerDualEncoderConfig {pub model_path: String,/// Only compute embeddings for pages that has a@@ -479,7 +479,7 @@ pub struct LiveIndexConfig {// indexerpub host_centrality_store_path: String,- pub page_webgraph: Option<IndexingGraphConfig>,+ pub page_webgraph: Option<IndexerGraphConfig>,pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,pub host_centrality_threshold: Option<f64>,diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex af03e0a4..0594da26 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -20,7 +20,7 @@ use tokio_stream::StreamExt;use tracing::{debug, info};use crate::config::{- defaults, IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LocalConfig,+ defaults, IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LocalConfig,WebSpellConfig,};use crate::entrypoint::indexer::JobSettings;@@ -180,12 +180,12 @@ fn create_inverted_index() -> Result<()> {let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");let centrality_path = Path::new(DATA_PATH).join("centrality");let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");- let dual_encoder_path = Path::new(DATA_PATH).join("summarizer").join("dual_encoder");+ let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");- let worker = indexer::IndexingWorker::new(IndexingLocalConfig {+ let worker = indexer::IndexingWorker::new(IndexerConfig {host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),- page_webgraph: Some(IndexingGraphConfig::Local {+ page_webgraph: Some(IndexerGraphConfig::Local {path: webgraph_path.to_str().unwrap().to_string(),}),topics_path: Some(@@ -204,7 +204,7 @@ fn create_inverted_index() -> Result<()> {minimum_clean_words: None,batch_size: defaults::Indexing::batch_size(),autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),- dual_encoder: Some(IndexingDualEncoderConfig {+ dual_encoder: Some(IndexerDualEncoderConfig {model_path: dual_encoder_path.to_str().unwrap().to_string(),page_centrality_rank_threshold: Some(100_000),}),diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex f5a3d9e8..a17bafa6 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -68,14 +68,16 @@ impl Job {for file in warc_files.by_ref() {let mut batch = Vec::with_capacity(self.settings.batch_size);- for chunk in &file+ for chunk in file.records().flatten().filter(|record| match &record.response.payload_type {Some(payload_type) => matches!(payload_type, PayloadType::Html),None => true,})+ .filter(|record| !worker.see(&record.request.url)).chunks(self.settings.batch_size)+ .into_iter(){batch.clear();diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex d1e3ef88..04952402 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -40,7 +40,7 @@ impl From<String> for IndexPointer {}}-pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {+pub fn run(config: &config::IndexerConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config: WarcSource = config.warc_source.clone();@@ -62,7 +62,16 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {autocommit_after_num_inserts: config.autocommit_after_num_inserts,},})- .map(|job| IndexPointer(job.process(&worker).path().as_os_str().to_str().unwrap().to_string()))+ .map(|job| {+ IndexPointer(+ job.process(&worker)+ .path()+ .as_os_str()+ .to_str()+ .unwrap()+ .to_string(),+ )+ }).collect();let index = merge(indexes)?;diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 91641771..45b39326 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -17,13 +17,14 @@use chrono::Utc;use itertools::Itertools;use std::path::Path;+use std::sync::Mutex;use tracing::debug;pub use super::indexable_webpage::IndexableWebpage;pub use super::job::{Job, JobSettings};use crate::config::{- IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LiveIndexConfig,+ IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,WebgraphGranularity,};use crate::models::dual_encoder::DualEncoder as DualEncoderModel;@@ -40,14 +41,14 @@ use crate::webpage::{safety_classifier, Html, Webpage};pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,- pub page_webgraph: Option<IndexingGraphConfig>,+ pub page_webgraph: Option<IndexerGraphConfig>,pub topics_path: Option<String>,pub safety_classifier_path: Option<String>,- pub dual_encoder: Option<IndexingDualEncoderConfig>,+ pub dual_encoder: Option<IndexerDualEncoderConfig>,}-impl From<IndexingLocalConfig> for Config {- fn from(config: IndexingLocalConfig) -> Self {+impl From<IndexerConfig> for Config {+ fn from(config: IndexerConfig) -> Self {Self {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,@@ -141,14 +142,14 @@ impl Webgraph {}impl Webgraph {- fn new(config: &IndexingGraphConfig) -> Self {+ fn new(config: &IndexerGraphConfig) -> Self {match config {- IndexingGraphConfig::Local { path } => Self::Local(+ IndexerGraphConfig::Local { path } => Self::Local(webgraph::WebgraphBuilder::new(path).single_threaded().open(),),- IndexingGraphConfig::Remote { gossip } => {+ IndexerGraphConfig::Remote { gossip } => {let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);Self::Remote(crate::block_on(RemoteWebgraph::new(cluster,@@ -170,6 +171,7 @@ pub struct IndexingWorker {job_settings: Option<JobSettings>,rake: RakeModel,dual_encoder: Option<DualEncoder>,+ seen_urls: Mutex<bloom::BytesBloomFilter<String>>,}impl IndexingWorker {@@ -216,6 +218,7 @@ impl IndexingWorker {page_centrality_rank_threshold: dual_encoder.page_centrality_rank_threshold,}}),+ seen_urls: Mutex::new(bloom::BytesBloomFilter::new(10_000_000_000, 0.05)),}}@@ -227,7 +230,20 @@ impl IndexingWorker {self.page_webgraph.as_ref()}- pub fn process(&self, job: &Job) -> Index {+ /// Returns false if the URL has not been seen before and marks it as seen.+ /// Returns true if the URL has been seen before.+ pub(super) fn see(&self, url: &String) -> bool {+ let mut seen_urls = self.seen_urls.lock().unwrap();++ if seen_urls.contains(url) {+ true+ } else {+ seen_urls.insert(url);+ false+ }+ }++ pub fn process(&mut self, job: &Job) -> Index {job.process(self)}@@ -395,7 +411,7 @@ impl IndexingWorker {.map(|w| Node::from(w.html.url()).id()).collect::<Vec<_>>();- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(256));+ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, EdgeLimit::Limit(512));for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {page.backlink_labels = backlink;@@ -495,13 +511,13 @@ mod tests {use super::*;fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {- IndexingWorker::new(IndexingLocalConfig {+ IndexingWorker::new(IndexerConfig {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,page_webgraph: None,topics_path: None,safety_classifier_path: None,- dual_encoder: Some(IndexingDualEncoderConfig {+ dual_encoder: Some(IndexerDualEncoderConfig {model_path: data_path.to_str().unwrap().to_string(),page_centrality_rank_threshold: threshold,}),diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 27a1f47d..42ef0156 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -183,6 +183,10 @@ impl Message<WebGraphService> for PagesByHosts {pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;+ let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());++ let server = WebGraphService { graph }.bind(addr).await.unwrap();+// dropping the handle leaves the clusterlet _cluster = Arc::new(Cluster::join(@@ -200,10 +204,6 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {.await?,);- let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());-- let server = WebGraphService { graph }.bind(addr).await.unwrap();-info!("webgraph server is ready to accept requests on {}", addr);loop {diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex f2950901..1f28b9ad 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -125,7 +125,7 @@ mod tests {use std::path::Path;use crate::{- config::{IndexingDualEncoderConfig, IndexingLocalConfig, WarcSource},+ config::{IndexerConfig, IndexerDualEncoderConfig, WarcSource},entrypoint::indexer::IndexingWorker,index::Index,models::dual_encoder::DualEncoder,@@ -744,13 +744,13 @@ mod tests {}fn setup_worker(data_path: &Path) -> IndexingWorker {- IndexingWorker::new(IndexingLocalConfig {+ IndexingWorker::new(IndexerConfig {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,page_webgraph: None,topics_path: None,safety_classifier_path: None,- dual_encoder: Some(IndexingDualEncoderConfig {+ dual_encoder: Some(IndexerDualEncoderConfig {model_path: data_path.to_str().unwrap().to_string(),page_centrality_rank_threshold: None,}),diff --git a/scripts/export_dual_encoder b/scripts/export_dual_encoderindex 663f653d..725b6de1 100755--- a/scripts/export_dual_encoder+++ b/scripts/export_dual_encoder@@ -5,14 +5,14 @@ import transformersMODEL = "sentence-transformers/all-MiniLM-L6-v2"-if os.path.exists("data/summarizer/"):- print("data/summarizer/ already exists. Exiting...")+if os.path.exists("data/dual_encoder/"):+ print("data/dual_encoder/ already exists. Exiting...")sys.exit()-os.system("mkdir -p data/summarizer/dual_encoder")+os.system("mkdir -p data/dual_encoder")model = transformers.AutoModel.from_pretrained(MODEL)tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)-model.save_pretrained("data/summarizer/dual_encoder")-tokenizer.save_pretrained("data/summarizer/dual_encoder")+model.save_pretrained("data/dual_encoder")+tokenizer.save_pretrained("data/dual_encoder")