Skip to content

web search done right

only build host webgraph for now

Mikkel Denker43d7ac4d9395
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index 7e45300e..7d4cb2d6 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use clap::Parser;
-use cuely::entrypoint::{Indexer, WebgraphBuilder};
+use cuely::entrypoint::{Indexer, WebgraphEntrypoint};
use cuely::Config;
use std::fs;
use tracing::Level;
@@ -43,7 +43,7 @@ fn main() {
.run()
.expect("Failed to index documents");
}
- Config::Webgraph(config) => WebgraphBuilder::from(config)
+ Config::Webgraph(config) => WebgraphEntrypoint::from(config)
.run()
.expect("Failed to build webgraph"),
}
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index 608da539..d4f534ca 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -17,4 +17,4 @@ mod indexer;
mod webgraph;
pub use indexer::Indexer;
-pub use webgraph::WebgraphBuilder;
+pub use webgraph::WebgraphEntrypoint;
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 086a6261..765d55ae 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -16,7 +16,7 @@
use crate::{
mapreduce::{Map, MapReduce, Reduce, Worker},
warc::WarcFile,
- webgraph::{FrozenWebgraph, Node, SledStore, Webgraph},
+ webgraph::{FrozenWebgraph, Node, Webgraph, WebgraphBuilder},
webpage::{self, Html},
HttpConfig, LocalConfig, Result, WarcSource, WebgraphConfig, WebgraphLocalConfig,
WebgraphMasterConfig, WebgraphWorkerConfig,
@@ -25,11 +25,11 @@ use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
use tracing::{debug, info};
-pub struct WebgraphBuilder {
+pub struct WebgraphEntrypoint {
config: WebgraphConfig,
}
-impl From<WebgraphConfig> for WebgraphBuilder {
+impl From<WebgraphConfig> for WebgraphEntrypoint {
fn from(config: WebgraphConfig) -> Self {
Self { config }
}
@@ -53,7 +53,9 @@ impl Map<FrozenWebgraph> for Job {
info!("processing {}", name);
- let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));
+ let mut graph = WebgraphBuilder::new(Path::new("webgraph").join(name))
+ .with_host_graph()
+ .open();
let source = match self.config {
JobConfig::Http(config) => WarcSource::HTTP(config),
@@ -109,7 +111,7 @@ impl Reduce<FrozenWebgraph> for FrozenWebgraph {
}
}
-impl WebgraphBuilder {
+impl WebgraphEntrypoint {
fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
info!("Running master for webgraph construction");
@@ -171,9 +173,9 @@ impl WebgraphBuilder {
pub fn run(&self) -> Result<()> {
match &self.config {
- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),
- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),
- WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),
+ WebgraphConfig::Master(config) => WebgraphEntrypoint::run_master(config),
+ WebgraphConfig::Worker(config) => WebgraphEntrypoint::run_worker(config),
+ WebgraphConfig::Local(config) => WebgraphEntrypoint::run_locally(config),
}
}
}
diff --git a/src/index.rs b/src/index.rs
index 449fffe3..dd47aca8 100644
--- a/src/index.rs
+++ b/src/index.rs
@@ -25,13 +25,31 @@ use crate::Result;
use std::path::Path;
pub struct Index {
- pub(crate) tantivy_index: tantivy::Index,
- pub(crate) writer: tantivy::IndexWriter,
- pub(crate) reader: tantivy::IndexReader,
- pub(crate) schema: tantivy::schema::Schema,
+ tantivy_index: tantivy::Index,
+ writer: tantivy::IndexWriter,
+ reader: tantivy::IndexReader,
+ schema: tantivy::schema::Schema,
}
impl Index {
+ #[cfg(test)]
+ pub fn temporary() -> Result<Index> {
+ use crate::{schema::create_schema, tokenizer::Tokenizer};
+
+ let schema = create_schema();
+ let tantivy_index = tantivy::Index::create_in_ram(schema);
+
+ tantivy_index
+ .tokenizers()
+ .register("tokenizer", Tokenizer::default());
+
+ Ok(Index {
+ writer: tantivy_index.writer(100_000_000)?,
+ reader: tantivy_index.reader()?,
+ schema: create_schema(),
+ tantivy_index,
+ })
+ }
pub fn open<P: AsRef<Path>>(_path: P) -> Result<Self> {
todo!();
}
@@ -129,14 +147,13 @@ impl From<Document> for RetrievedWebpage {
#[cfg(test)]
mod tests {
- use crate::tests::temporary_index;
use crate::{ranking::Ranker, webpage::Link};
use super::*;
#[test]
fn simple_search() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("website").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -172,7 +189,7 @@ mod tests {
#[test]
fn document_not_matching() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("this query should not match").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -201,7 +218,7 @@ mod tests {
#[test]
fn english_stemming() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runner").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -230,7 +247,7 @@ mod tests {
#[test]
fn stemmed_query_english() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runners").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -259,7 +276,7 @@ mod tests {
#[test]
fn searchable_backlinks() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("great site").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -314,7 +331,7 @@ mod tests {
#[test]
fn limited_top_docs() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("runner").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
@@ -345,7 +362,7 @@ mod tests {
#[test]
fn host_search() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
let query = Query::parse("dr").expect("Failed to parse query");
let ranker = Ranker::new(query.clone());
diff --git a/src/lib.rs b/src/lib.rs
index e31d8ecc..cac32acb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -170,25 +170,3 @@ pub enum Error {
}
pub(crate) type Result<T> = std::result::Result<T, Error>;
-
-#[cfg(test)]
-pub(crate) mod tests {
- use super::*;
- use crate::{index::Index, schema::create_schema, tokenizer::Tokenizer};
-
- pub fn temporary_index() -> Result<Index> {
- let schema = create_schema();
- let tantivy_index = tantivy::Index::create_in_ram(schema);
-
- tantivy_index
- .tokenizers()
- .register("tokenizer", Tokenizer::default());
-
- Ok(Index {
- writer: tantivy_index.writer(100_000_000)?,
- reader: tantivy_index.reader()?,
- schema: create_schema(),
- tantivy_index,
- })
- }
-}
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index bf1348fe..43934c9f 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -42,15 +42,15 @@ impl Ranker {
#[cfg(test)]
mod tests {
use crate::{
+ index::Index,
searcher::Searcher,
- tests::temporary_index,
webpage::{Link, Webpage},
};
#[test]
fn harmonic_ranking() {
for _ in 0..10 {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
@@ -97,7 +97,7 @@ mod tests {
#[test]
fn navigational_search() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(
diff --git a/src/snippet.rs b/src/snippet.rs
index 20c78f99..c7355802 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -297,8 +297,7 @@ pub fn generate(query: &Query, text: &str, searcher: &tantivy::Searcher) -> Resu
#[cfg(test)]
mod tests {
use super::*;
- use crate::tests::temporary_index;
- use crate::{searcher::Searcher, webpage::Webpage};
+ use crate::{index::Index, searcher::Searcher, webpage::Webpage};
use maplit::btreemap;
use tantivy::tokenizer::SimpleTokenizer;
@@ -316,7 +315,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn snippet_during_search() {
- let mut index = temporary_index().expect("Unable to open index");
+ let mut index = Index::temporary().expect("Unable to open index");
index
.insert(Webpage::new(