web search done right
dedup edges by ids on insert
Mikkel Denkerba464a2d81b5
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 0439ea98..89cc7d16 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -30,6 +30,7 @@ use super::{};use crate::{ampc::dht::ShardId, webpage::html::links::RelFlags, Result};use itertools::Itertools;+use rustc_hash::FxHashSet;use tantivy::{columnar::Column, directory::MmapDirectory, indexer::NoMergePolicy, DocId, SegmentReader,};@@ -39,6 +40,7 @@ use super::{query::Query, NodeID};pub struct EdgeStore {index: tantivy::Index,writer: Option<tantivy::IndexWriter<Edge>>,+ writer_dedup: FxHashSet<(NodeID, NodeID)>,reader: tantivy::IndexReader,shard_id: ShardId,path: PathBuf,@@ -67,6 +69,7 @@ impl EdgeStore {reader: index.reader()?,index,shard_id,+ writer_dedup: FxHashSet::default(),})}@@ -104,6 +107,11 @@ impl EdgeStore {pub fn insert(&mut self, edge: Edge) -> Result<()> {self.prepare_writer()?;++ if !self.writer_dedup.insert((edge.from.id(), edge.to.id())) {+ return Ok(());+ }+self.writer.as_mut().expect("writer should have been prepared")@@ -117,6 +125,7 @@ impl EdgeStore {.as_mut().expect("writer should have been prepared").commit()?;+ self.writer_dedup.clear();self.reader.reload()?;Ok(())}