web search done right
only build host webgraph for now
Mikkel Denker43d7ac4d9395
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 7e45300e..7d4cb2d6 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use clap::Parser;-use cuely::entrypoint::{Indexer, WebgraphBuilder};+use cuely::entrypoint::{Indexer, WebgraphEntrypoint};use cuely::Config;use std::fs;use tracing::Level;@@ -43,7 +43,7 @@ fn main() {.run().expect("Failed to index documents");}- Config::Webgraph(config) => WebgraphBuilder::from(config)+ Config::Webgraph(config) => WebgraphEntrypoint::from(config).run().expect("Failed to build webgraph"),}diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex 608da539..d4f534ca 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -17,4 +17,4 @@ mod indexer;mod webgraph;pub use indexer::Indexer;-pub use webgraph::WebgraphBuilder;+pub use webgraph::WebgraphEntrypoint;diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 086a6261..765d55ae 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -16,7 +16,7 @@use crate::{mapreduce::{Map, MapReduce, Reduce, Worker},warc::WarcFile,- webgraph::{FrozenWebgraph, Node, SledStore, Webgraph},+ webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},webpage::{self, Html},HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,WebgraphMasterConfig, WebgraphWorkerConfig,@@ -25,11 +25,11 @@ use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};use tracing::{debug, info};-pub struct WebgraphBuilder {+pub struct WebgraphEntrypoint {config: WebgraphConfig,}-impl From<WebgraphConfig> for WebgraphBuilder {+impl From<WebgraphConfig> for WebgraphEntrypoint {fn from(config: WebgraphConfig) -> Self {Self { config }}@@ -53,7 +53,9 @@ impl Map<FrozenWebgraph> for Job {info!("processing {}", name);- let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));+ let mut graph = WebgraphBuilder::new(Path::new("webgraph").join(name))+ .with_host_graph()+ .open();let source = match self.config {JobConfig::Http(config) => WarcSource::HTTP(config),@@ -109,7 +111,7 @@ impl Reduce<FrozenWebgraph> for FrozenWebgraph {}}-impl WebgraphBuilder {+impl WebgraphEntrypoint {fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");@@ -171,9 +173,9 @@ impl WebgraphBuilder {pub fn run(&self) -> Result<()> {match &self.config {- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),- WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),+ WebgraphConfig::Master(config) => WebgraphEntrypoint::run_master(config),+ WebgraphConfig::Worker(config) => WebgraphEntrypoint::run_worker(config),+ WebgraphConfig::Local(config) => WebgraphEntrypoint::run_locally(config),}}}diff --git a/src/index.rs b/src/index.rsindex 449fffe3..dd47aca8 100644--- a/src/index.rs+++ b/src/index.rs@@ -25,13 +25,31 @@ use crate::Result;use std::path::Path;pub struct Index {- pub(crate) tantivy_index: tantivy::Index,- pub(crate) writer: tantivy::IndexWriter,- pub(crate) reader: tantivy::IndexReader,- pub(crate) schema: tantivy::schema::Schema,+ tantivy_index: tantivy::Index,+ writer: tantivy::IndexWriter,+ reader: tantivy::IndexReader,+ schema: tantivy::schema::Schema,}impl Index {+ #[cfg(test)]+ pub fn temporary() -> Result<Index> {+ use crate::{schema::create_schema, tokenizer::Tokenizer};++ let schema = create_schema();+ let tantivy_index = tantivy::Index::create_in_ram(schema);++ tantivy_index+ .tokenizers()+ .register("tokenizer", Tokenizer::default());++ Ok(Index {+ writer: tantivy_index.writer(100_000_000)?,+ reader: tantivy_index.reader()?,+ schema: create_schema(),+ tantivy_index,+ })+ }pub fn open<P: AsRef<Path>>(_path: P) -> Result<Self> {todo!();}@@ -129,14 +147,13 @@ impl From<Document> for RetrievedWebpage {#[cfg(test)]mod tests {- use crate::tests::temporary_index;use crate::{ranking::Ranker, webpage::Link};use super::*;#[test]fn simple_search() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("website").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -172,7 +189,7 @@ mod tests {#[test]fn document_not_matching() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("this query should not match").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -201,7 +218,7 @@ mod tests {#[test]fn english_stemming() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runner").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -230,7 +247,7 @@ mod tests {#[test]fn stemmed_query_english() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runners").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -259,7 +276,7 @@ mod tests {#[test]fn searchable_backlinks() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("great site").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -314,7 +331,7 @@ mod tests {#[test]fn limited_top_docs() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("runner").expect("Failed to parse query");let ranker = Ranker::new(query.clone());@@ -345,7 +362,7 @@ mod tests {#[test]fn host_search() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");let query = Query::parse("dr").expect("Failed to parse query");let ranker = Ranker::new(query.clone());diff --git a/src/lib.rs b/src/lib.rsindex e31d8ecc..cac32acb 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -170,25 +170,3 @@ pub enum Error {}pub(crate) type Result<T> = std::result::Result<T, Error>;--#[cfg(test)]-pub(crate) mod tests {- use super::*;- use crate::{index::Index, schema::create_schema, tokenizer::Tokenizer};-- pub fn temporary_index() -> Result<Index> {- let schema = create_schema();- let tantivy_index = tantivy::Index::create_in_ram(schema);-- tantivy_index- .tokenizers()- .register("tokenizer", Tokenizer::default());-- Ok(Index {- writer: tantivy_index.writer(100_000_000)?,- reader: tantivy_index.reader()?,- schema: create_schema(),- tantivy_index,- })- }-}diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex bf1348fe..43934c9f 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -42,15 +42,15 @@ impl Ranker {#[cfg(test)]mod tests {use crate::{+ index::Index,searcher::Searcher,- tests::temporary_index,webpage::{Link, Webpage},};#[test]fn harmonic_ranking() {for _ in 0..10 {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");index.insert(Webpage::new(@@ -97,7 +97,7 @@ mod tests {#[test]fn navigational_search() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");index.insert(Webpage::new(diff --git a/src/snippet.rs b/src/snippet.rsindex 20c78f99..c7355802 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -297,8 +297,7 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu#[cfg(test)]mod tests {use super::*;- use crate::tests::temporary_index;- use crate::{searcher::Searcher, webpage::Webpage};+ use crate::{index::Index, searcher::Searcher, webpage::Webpage};use maplit::btreemap;use tantivy::tokenizer::SimpleTokenizer;@@ -316,7 +315,7 @@ Survey in 2016, 2017, and 2018."#;#[test]fn snippet_during_search() {- let mut index = temporary_index().expect("Unable to open index");+ let mut index = Index::temporary().expect("Unable to open index");index.insert(Webpage::new(