Skip to content

web search done right

webgraph CLI merge segments

Mikkel Denker29fe3ad65278
diff --git a/core/src/main.rs b/core/src/main.rs
index 62fe25cb..6188ffbf 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -20,6 +20,7 @@ use cuely::entrypoint::autosuggest_scrape::{self, Gl};
use cuely::entrypoint::configure;
use cuely::entrypoint::indexer::IndexPointer;
use cuely::entrypoint::{self, frontend, search_server};
+use cuely::webgraph::WebgraphBuilder;
use cuely::{FrontendConfig, SearchServerConfig};
use serde::de::DeserializeOwned;
use std::fs;
@@ -85,9 +86,20 @@ enum Commands {
#[derive(Subcommand)]
enum WebgraphOptions {
- Master { config_path: String },
- Worker { address: String },
- Local { config_path: String },
+ Master {
+ config_path: String,
+ },
+ Worker {
+ address: String,
+ },
+ Local {
+ config_path: String,
+ },
+ Merge {
+ path: String,
+ other_path: Option<String>,
+ num_segments: Option<usize>,
+ },
}
#[derive(Subcommand)]
@@ -181,6 +193,22 @@ fn main() -> Result<()> {
let config = load_toml_config(config_path);
entrypoint::Webgraph::run_locally(&config)?;
}
+ WebgraphOptions::Merge {
+ path,
+ other_path,
+ num_segments,
+ } => {
+ let mut webgraph = WebgraphBuilder::new(path).open();
+
+ if let Some(other_path) = other_path {
+ let other = WebgraphBuilder::new(other_path).open();
+ webgraph.merge(other);
+ }
+
+ if let Some(num_segments) = num_segments {
+ webgraph.merge_segments(num_segments)
+ }
+ }
},
Commands::Frontend { config_path } => {
let config: FrontendConfig = load_toml_config(config_path);
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 05dfb3f0..3bd3088a 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -35,10 +35,8 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};
-
-#[cfg(test)]
use super::WebsitesResult;
+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};
const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;
@@ -311,7 +309,6 @@ impl LocalSearcher {
self.index.retrieve_websites(websites, &query)
}
- #[cfg(test)]
pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
use std::{sync::Arc, time::Instant};
diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rs
index 46226517..81750750 100644
--- a/core/src/webgraph/executor.rs
+++ b/core/src/webgraph/executor.rs
@@ -38,6 +38,7 @@ impl Executor {
pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {
let pool = ThreadPoolBuilder::new()
.num_threads(num_threads)
+ .stack_size(8_000_000)
.thread_name(move |num| format!("{prefix}{num}"))
.build()?;
Ok(Executor::ThreadPool(pool))
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 1075cd31..b969bb10 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -333,7 +333,8 @@ impl<S: Store> Webgraph<S> {
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
live_segment: None,
segments,
- executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
+ // executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),
+ executor: Arc::new(Executor::single_thread()),
node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),
id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),
meta,
diff --git a/justfile b/justfile
index 8c662067..619f920d 100644
--- a/justfile
+++ b/justfile
@@ -18,6 +18,7 @@
cd frontend; npm run dev
@configure *ARGS:
+ ./scripts/export_crossencoder
cd frontend; npm install; npm run build
cargo run --release --all-features -- configure {{ARGS}}
diff --git a/optics-lsp/src/lib.rs b/optics-lsp/src/lib.rs
index bd81a0a6..f650b378 100644
--- a/optics-lsp/src/lib.rs
+++ b/optics-lsp/src/lib.rs
@@ -47,7 +47,7 @@ pub enum Error {
impl From<Error> for JsValue {
fn from(val: Error) -> Self {
- JsValue::from_str(&format!("{:?}", val))
+ JsValue::from_str(&format!("{val:?}"))
}
}
@@ -120,13 +120,13 @@ impl OpticsBackend {
);
self.send_diagnostics(params.text_document.uri);
}
- _ => log(&format!("on_notification {} {:?}", method, params)),
+ _ => log(&format!("on_notification {method} {params:?}")),
}
}
#[wasm_bindgen(js_name = onHover)]
pub fn on_hover(&mut self, params: JsValue) -> Result<JsValue, Error> {
- log(&format!("on_hover {:?}", params));
+ log(&format!("on_hover {params:?}"));
let params: HoverParams = serde_wasm_bindgen::from_value(params).unwrap();
Ok(serde_wasm_bindgen::to_value(&self.handle_hover(params))?)
@@ -271,13 +271,12 @@ impl OpticsBackend {
.unwrap_or_default(),
version: None,
};
- log(&format!("Sending diagnostic {:?}", params));
+ log(&format!("Sending diagnostic {params:?}"));
let params = &serde_wasm_bindgen::to_value(&params).unwrap();
if let Err(e) = self.diagnostic_callback.call1(this, params) {
log(&format!(
- "send_diagnostics params:\n\t{:?}\n\tJS error: {:?}",
- params, e
+ "send_diagnostics params:\n\t{params:?}\n\tJS error: {e:?}",
));
}
}