web search done right
webgraph CLI merge segments
Mikkel Denker29fe3ad65278
diff --git a/core/src/main.rs b/core/src/main.rsindex 62fe25cb..6188ffbf 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -20,6 +20,7 @@ use cuely::entrypoint::autosuggest_scrape::{self, Gl};use cuely::entrypoint::configure;use cuely::entrypoint::indexer::IndexPointer;use cuely::entrypoint::{self, frontend, search_server};+use cuely::webgraph::WebgraphBuilder;use cuely::{FrontendConfig, SearchServerConfig};use serde::de::DeserializeOwned;use std::fs;@@ -85,9 +86,20 @@ enum Commands {#[derive(Subcommand)]enum WebgraphOptions {- Master { config_path: String },- Worker { address: String },- Local { config_path: String },+ Master {+ config_path: String,+ },+ Worker {+ address: String,+ },+ Local {+ config_path: String,+ },+ Merge {+ path: String,+ other_path: Option<String>,+ num_segments: Option<usize>,+ },}#[derive(Subcommand)]@@ -181,6 +193,22 @@ fn main() -> Result<()> {let config = load_toml_config(config_path);entrypoint::Webgraph::run_locally(&config)?;}+ WebgraphOptions::Merge {+ path,+ other_path,+ num_segments,+ } => {+ let mut webgraph = WebgraphBuilder::new(path).open();++ if let Some(other_path) = other_path {+ let other = WebgraphBuilder::new(other_path).open();+ webgraph.merge(other);+ }++ if let Some(num_segments) = num_segments {+ webgraph.merge_segments(num_segments)+ }+ }},Commands::Frontend { config_path } => {let config: FrontendConfig = load_toml_config(config_path);diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 05dfb3f0..3bd3088a 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -35,10 +35,8 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};--#[cfg(test)]use super::WebsitesResult;+use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;@@ -311,7 +309,6 @@ impl LocalSearcher {self.index.retrieve_websites(websites, &query)}- #[cfg(test)]pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {use std::{sync::Arc, time::Instant};diff --git a/core/src/webgraph/executor.rs b/core/src/webgraph/executor.rsindex 46226517..81750750 100644--- a/core/src/webgraph/executor.rs+++ b/core/src/webgraph/executor.rs@@ -38,6 +38,7 @@ impl Executor {pub fn with_threads(num_threads: usize, prefix: &'static str) -> Result<Executor> {let pool = ThreadPoolBuilder::new().num_threads(num_threads)+ .stack_size(8_000_000).thread_name(move |num| format!("{prefix}{num}")).build()?;Ok(Executor::ThreadPool(pool))diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 1075cd31..b969bb10 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -333,7 +333,8 @@ impl<S: Store> Webgraph<S> {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),live_segment: None,segments,- executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),+ // executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),+ executor: Arc::new(Executor::single_thread()),node2id: CachedStore::new(RocksDbStore::open(path.as_ref().join("node2id")), 100_000),id2node: CachedStore::new(RocksDbStore::open(path.as_ref().join("id2node")), 100_000),meta,diff --git a/justfile b/justfileindex 8c662067..619f920d 100644--- a/justfile+++ b/justfile@@ -18,6 +18,7 @@cd frontend; npm run dev@configure *ARGS:+ ./scripts/export_crossencodercd frontend; npm install; npm run buildcargo run --release --all-features -- configure {{ARGS}}diff --git a/optics-lsp/src/lib.rs b/optics-lsp/src/lib.rsindex bd81a0a6..f650b378 100644--- a/optics-lsp/src/lib.rs+++ b/optics-lsp/src/lib.rs@@ -47,7 +47,7 @@ pub enum Error {impl From<Error> for JsValue {fn from(val: Error) -> Self {- JsValue::from_str(&format!("{:?}", val))+ JsValue::from_str(&format!("{val:?}"))}}@@ -120,13 +120,13 @@ impl OpticsBackend {);self.send_diagnostics(params.text_document.uri);}- _ => log(&format!("on_notification {} {:?}", method, params)),+ _ => log(&format!("on_notification {method} {params:?}")),}}#[wasm_bindgen(js_name = onHover)]pub fn on_hover(&mut self, params: JsValue) -> Result<JsValue, Error> {- log(&format!("on_hover {:?}", params));+ log(&format!("on_hover {params:?}"));let params: HoverParams = serde_wasm_bindgen::from_value(params).unwrap();Ok(serde_wasm_bindgen::to_value(&self.handle_hover(params))?)@@ -271,13 +271,12 @@ impl OpticsBackend {.unwrap_or_default(),version: None,};- log(&format!("Sending diagnostic {:?}", params));+ log(&format!("Sending diagnostic {params:?}"));let params = &serde_wasm_bindgen::to_value(¶ms).unwrap();if let Err(e) = self.diagnostic_callback.call1(this, params) {log(&format!(- "send_diagnostics params:\n\t{:?}\n\tJS error: {:?}",- params, e+ "send_diagnostics params:\n\t{params:?}\n\tJS error: {e:?}",));}}