web search done right
make live index searchable
Mikkel Denkera6ea0d734cb8
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 99e1561d..4fe4ad91 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -41,7 +41,10 @@ use crate::{};use anyhow::Result;-use std::{net::SocketAddr, sync::Arc};+use std::{+ net::{IpAddr, SocketAddr},+ sync::Arc,+};use axum::{http::StatusCode,@@ -253,11 +256,12 @@ async fn search_metric<B>(// It is very important that the ip address is not stored. It is only used// for a probabilistic estimate of the number of unique users using a hyperloglog datastructure.let mut ip = None;+if let Some(forwarded_for) = request.headers().get("x-forwarded-for") {let forwarded_for = forwarded_for.to_str().unwrap_or_default();if let Some(client_ip) = forwarded_for.split(',').next() {- if let Ok(client_ip) = client_ip.trim().parse::<SocketAddr>() {- ip = Some(client_ip.ip());+ if let Ok(client_ip) = client_ip.trim().parse::<IpAddr>() {+ ip = Some(client_ip);}}}diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rsindex d38c75f8..48cfae15 100644--- a/core/src/distributed/member.rs+++ b/core/src/distributed/member.rs@@ -22,10 +22,23 @@ use crate::searcher::ShardId;#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]pub enum Service {- Searcher { host: SocketAddr, shard: ShardId },- Api { host: SocketAddr },- Webgraph { host: SocketAddr },- Alice { host: SocketAddr },+ Searcher {+ host: SocketAddr,+ shard: ShardId,+ },+ LiveIndex {+ host: SocketAddr,+ split_id: crate::feed::scheduler::SplitId,+ },+ Api {+ host: SocketAddr,+ },+ Webgraph {+ host: SocketAddr,+ },+ Alice {+ host: SocketAddr,+ },}#[derive(PartialEq, Eq, Hash, Clone, Debug)]diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rsindex 3e19bf26..d6e93a1a 100644--- a/core/src/entrypoint/live_index.rs+++ b/core/src/entrypoint/live_index.rs@@ -14,22 +14,108 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::path::Path;+use std::{path::Path, sync::Arc};use crate::{config::{LiveIndexConfig, LiveIndexSchedulerConfig},+ distributed::{+ cluster::Cluster,+ member::{Member, Service},+ sonic,+ },feed::{self, index::FeedIndex},+ inverted_index,kv::rocksdb_store::RocksDbStore,- live_index::IndexManager,+ live_index::{Index, IndexManager},+ ranking::inbound_similarity::InboundSimilarity,+ searcher::{InitialWebsiteResult, LocalSearcher},+ sonic_service,webgraph::WebgraphBuilder,};use anyhow::Result;+use tracing::info;++use super::search_server::{RetrieveWebsites, Search};++sonic_service!(SearchService, [RetrieveWebsites, Search]);++pub struct SearchService {+ local_searcher: LocalSearcher<Arc<Index>>,+ // dropping the handle leaves the cluster+ #[allow(unused)]+ cluster_handle: Cluster,+}++impl SearchService {+ async fn new(config: LiveIndexConfig) -> Result<Self> {+ let inbound_similarity = InboundSimilarity::open(+ Path::new(&config.host_centrality_store_path).join("inbound_similarity"),+ )?;++ let manager = IndexManager::new(config.clone())?;+ let mut local_searcher = LocalSearcher::new(manager.index());++ local_searcher.set_inbound_similarity(inbound_similarity);++ tokio::task::spawn(manager.run());++ let cluster_handle = Cluster::join(+ Member {+ id: config.cluster_id,+ service: Service::LiveIndex {+ host: config.host,+ split_id: config.split_id,+ },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.unwrap_or_default(),+ )+ .await?;++ Ok(Self {+ local_searcher,+ cluster_handle,+ })+ }+}++#[async_trait::async_trait]+impl sonic::service::Message<SearchService> for RetrieveWebsites {+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ match server+ .local_searcher+ .retrieve_websites(&self.websites, &self.query)+ {+ Ok(response) => Ok(Some(response)),+ Err(_) => Ok(None),+ }+ }+}++#[async_trait::async_trait]+impl sonic::service::Message<SearchService> for Search {+ type Response = Option<InitialWebsiteResult>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ match server.local_searcher.search_initial(&self.query, true) {+ Ok(result) => Ok(Some(result)),+ Err(_) => Ok(None),+ }+ }+}pub async fn serve(config: LiveIndexConfig) -> Result<()> {- let manager = IndexManager::new(config)?;- tokio::task::spawn(manager.run());+ let addr = config.host;++ let server = SearchService::new(config).await?.bind(&addr).await.unwrap();++ info!("live index is ready to accept requests on {}", addr);- todo!("accept search requests");+ loop {+ if let Err(e) = server.accept().await {+ tracing::error!("{:?}", e);+ }+ }}pub fn schedule(config: LiveIndexSchedulerConfig) -> Result<()> {diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex b2a85aea..254c19cd 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -51,7 +51,7 @@ sonic_service!();pub struct SearchService {- local_searcher: LocalSearcher,+ local_searcher: LocalSearcher<Index>,// dropping the handle leaves the cluster#[allow(unused)]cluster_handle: Cluster,diff --git a/core/src/feed/parser.rs b/core/src/feed/parser.rsindex de3734fa..cfb56920 100644--- a/core/src/feed/parser.rs+++ b/core/src/feed/parser.rs@@ -83,7 +83,6 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {.filter(std::result::Result::is_ok).find(|attr| attr.as_ref().unwrap().key.as_ref() == b"href"){- dbg!(&href);if let Ok(href) = href.unescape_value().map_err(|e| anyhow!(e))diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rsindex 0f876704..8a60cbbd 100644--- a/core/src/feed/scheduler.rs+++ b/core/src/feed/scheduler.rs@@ -46,7 +46,7 @@ impl From<&Url> for Domain {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub struct SplitId(uuid::Uuid);#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]diff --git a/core/src/index.rs b/core/src/index.rsindex 4e9ff281..7a7c5129 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -17,12 +17,10 @@use std::collections::HashSet;use std::fs;use std::path::Path;-use std::sync::{Arc, Mutex};+use std::sync::Mutex;use std::time::SystemTime;-use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;-use url::Url;use crate::collector::MainCollector;use crate::inverted_index::{self, InvertedIndex};@@ -147,22 +145,6 @@ impl Index {Self::open(&self.path).expect("failed to open index")}-- pub fn schema(&self) -> Arc<Schema> {- self.inverted_index.schema()- }-- pub fn num_segments(&self) -> usize {- self.inverted_index.num_segments()- }-- pub(crate) fn get_webpage(&self, url: &str) -> Option<inverted_index::RetrievedWebpage> {- self.inverted_index.get_webpage(url)- }-- pub fn get_homepage(&self, url: &Url) -> Option<inverted_index::RetrievedWebpage> {- self.inverted_index.get_homepage(url)- }}#[cfg(test)]diff --git a/core/src/live_index.rs b/core/src/live_index.rsindex f66169e2..f69a28c5 100644--- a/core/src/live_index.rs+++ b/core/src/live_index.rs@@ -17,7 +17,7 @@use anyhow::Result;use std::{path::Path,- sync::{Arc, RwLock},+ sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},time::{Duration, SystemTime},};@@ -215,7 +215,7 @@ impl Crawler {}}-struct Index {+pub struct Index {search_index: Arc<RwLock<crate::index::Index>>,}@@ -248,6 +248,14 @@ impl Index {fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {self.search_index.clone()}++ pub fn read(&self) -> RwLockReadGuard<'_, crate::index::Index> {+ self.search_index.read().unwrap_or_else(|e| e.into_inner())+ }++ pub fn write(&self) -> RwLockWriteGuard<'_, crate::index::Index> {+ self.search_index.write().unwrap_or_else(|e| e.into_inner())+ }}#[async_trait::async_trait]@@ -364,4 +372,8 @@ impl IndexManager {tokio::time::sleep(EVENT_LOOP_INTERVAL).await;}}++ pub fn index(&self) -> Arc<Index> {+ self.index.clone()+ }}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 38d41d9a..ea541634 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::collections::HashMap;-use std::sync::Arc;+use std::sync::{Arc, RwLockReadGuard};use tantivy::schema::Value;use tantivy::TantivyDocument;@@ -25,7 +25,7 @@ use crate::config::{CollectorConfig, SnippetConfig};use crate::entity_index::{EntityIndex, EntityMatch};use crate::image_store::Image;use crate::index::Index;-use crate::inverted_index::RetrievedWebpage;+use crate::inverted_index::{InvertedIndex, RetrievedWebpage};use crate::query::Query;use crate::ranking::inbound_similarity::InboundSimilarity;#[cfg(not(feature = "libtorch"))]@@ -40,13 +40,84 @@ use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpeuse crate::spell::Spell;use crate::webgraph::Node;use crate::webpage::region::Region;-use crate::{inverted_index, Error, Result};+use crate::{inverted_index, live_index, Error, Result};use super::WebsitesResult;use super::{InitialWebsiteResult, SearchQuery};-pub struct LocalSearcher {- index: Index,+pub trait SearchableIndex {+ type SearchGuard<'a>: SearchGuard<'a>+ where+ Self: 'a;++ fn guard(&self) -> Self::SearchGuard<'_>;+ fn optimize_for_search(&mut self);+ fn set_snippet_config(&mut self, config: SnippetConfig);+}++pub trait SearchGuard<'a> {+ fn search_index(&self) -> &'_ Index;+ fn inverted_index(&self) -> &'_ InvertedIndex {+ &self.search_index().inverted_index+ }+}++impl SearchableIndex for Index {+ type SearchGuard<'a> = NormalIndexSearchGuard<'a>;++ fn guard(&self) -> Self::SearchGuard<'_> {+ NormalIndexSearchGuard { search_index: self }+ }++ fn optimize_for_search(&mut self) {+ self.optimize_for_search().unwrap();+ }++ fn set_snippet_config(&mut self, config: SnippetConfig) {+ self.inverted_index.set_snippet_config(config);+ }+}++pub struct NormalIndexSearchGuard<'a> {+ search_index: &'a Index,+}++impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {+ fn search_index(&self) -> &'_ Index {+ self.search_index+ }+}++impl SearchableIndex for Arc<live_index::Index> {+ type SearchGuard<'a> = LiveIndexSearchGuard<'a>;++ fn guard(&self) -> Self::SearchGuard<'_> {+ LiveIndexSearchGuard {+ lock_guard: self.read(),+ }+ }++ fn optimize_for_search(&mut self) {+ self.write().optimize_for_search().unwrap();+ }++ fn set_snippet_config(&mut self, config: SnippetConfig) {+ self.write().inverted_index.set_snippet_config(config);+ }+}++pub struct LiveIndexSearchGuard<'a> {+ lock_guard: RwLockReadGuard<'a, crate::index::Index>,+}++impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {+ fn search_index(&self) -> &'_ Index {+ &self.lock_guard+ }+}++pub struct LocalSearcher<I: SearchableIndex> {+ index: I,spell: Option<Spell>,entity_index: Option<EntityIndex>,inbound_similarity: Option<InboundSimilarity>,@@ -55,8 +126,11 @@ pub struct LocalSearcher {collector_config: CollectorConfig,}-impl From<Index> for LocalSearcher {- fn from(index: Index) -> Self {+impl<I> From<I> for LocalSearcher<I>+where+ I: SearchableIndex,+{+ fn from(index: I) -> Self {Self::new(index)}}@@ -67,10 +141,13 @@ struct InvertedIndexResult {has_more: bool,}-impl LocalSearcher {- pub fn new(index: Index) -> Self {+impl<I> LocalSearcher<I>+where+ I: SearchableIndex,+{+ pub fn new(index: I) -> Self {let mut index = index;- index.optimize_for_search().unwrap();+ index.optimize_for_search();LocalSearcher {index,@@ -84,7 +161,7 @@ impl LocalSearcher {}pub fn build_spell_dict(&mut self) {- self.spell = Some(Spell::for_index(&self.index));+ self.spell = Some(Spell::for_index(self.index.guard().search_index()));}pub fn set_entity_index(&mut self, entity_index: EntityIndex) {@@ -108,11 +185,16 @@ impl LocalSearcher {}pub fn set_snippet_config(&mut self, config: SnippetConfig) {- self.index.inverted_index.set_snippet_config(config);+ self.index.set_snippet_config(config);}- fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {- let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;+ fn parse_query<'a, G: SearchGuard<'a>>(+ &'a self,+ ctx: &Ctx,+ guard: &G,+ query: &SearchQuery,+ ) -> Result<Query> {+ let parsed_query = Query::parse(ctx, query, guard.inverted_index())?;if parsed_query.is_empty() {Err(Error::EmptyQuery.into())@@ -121,10 +203,11 @@ impl LocalSearcher {}}- fn ranker(- &self,+ fn ranker<'a, G: SearchGuard<'a>>(+ &'a self,query: &Query,ctx: &Ctx,+ guard: &G,de_rank_similar: bool,aggregator: SignalAggregator,) -> Result<Ranker> {@@ -132,7 +215,7 @@ impl LocalSearcher {let mut ranker = Ranker::new(aggregator,- self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),+ guard.inverted_index().fastfield_reader(&ctx.tv_searcher),self.collector_config.clone(),);@@ -147,11 +230,12 @@ impl LocalSearcher {if query_centrality_coeff > 0.0 {if let Some(inbound_sim) = self.inbound_similarity.as_ref() {ranker = ranker- .with_max_docs(1_000, self.index.num_segments())+ .with_max_docs(1_000, guard.inverted_index().num_segments()).with_num_results(100);let top_host_nodes =- self.index+ guard+ .search_index().top_nodes(query, ctx, ranker.collector(ctx.clone()))?;if !top_host_nodes.is_empty() {@@ -167,15 +251,16 @@ impl LocalSearcher {Ok(ranker.with_max_docs(self.collector_config.max_docs_considered,- self.index.num_segments(),+ guard.inverted_index().num_segments(),).with_num_results(query.num_results()).with_offset(query.offset()))}- fn search_inverted_index(- &self,+ fn search_inverted_index<'a, G: SearchGuard<'a>>(+ &'a self,ctx: &Ctx,+ guard: &G,query: &SearchQuery,de_rank_similar: bool,) -> Result<InvertedIndexResult> {@@ -185,7 +270,7 @@ impl LocalSearcher {self.lambda_model.clone(),self.collector_config.clone(),);- let parsed_query = self.parse_query(ctx, &query)?;+ let parsed_query = self.parse_query(ctx, guard, &query)?;let mut aggregator = SignalAggregator::new(Some(&parsed_query));@@ -212,7 +297,8 @@ impl LocalSearcher {}aggregator.set_region_count(- self.index+ guard+ .search_index().region_count.lock().unwrap_or_else(|e| e.into_inner())@@ -223,17 +309,17 @@ impl LocalSearcher {aggregator.set_linear_model(model.clone());}- let ranker = self.ranker(&parsed_query, ctx, de_rank_similar, aggregator)?;+ let ranker = self.ranker(&parsed_query, ctx, guard, de_rank_similar, aggregator)?;- let res = self.index.inverted_index.search_initial(+ let res = guard.inverted_index().search_initial(&parsed_query,ctx,ranker.collector(ctx.clone()),)?;- let fastfield_reader = self.index.inverted_index.fastfield_reader(&ctx.tv_searcher);+ let fastfield_reader = guard.inverted_index().fastfield_reader(&ctx.tv_searcher);- let ranking_websites = self.index.inverted_index.retrieve_ranking_websites(+ let ranking_websites = guard.inverted_index().retrieve_ranking_websites(ctx,res.top_websites,ranker.aggregator(),@@ -243,7 +329,7 @@ impl LocalSearcher {let pipe_top_n = pipeline.top_n;let mut ranking_websites = pipeline.apply(ranking_websites);- let schema = self.index.schema();+ let schema = guard.inverted_index().schema();for website in &mut ranking_websites {let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;website.title = Some(@@ -282,8 +368,10 @@ impl LocalSearcher {query: &SearchQuery,de_rank_similar: bool,) -> Result<InitialWebsiteResult> {- let ctx = self.index.inverted_index.local_search_ctx();- let inverted_index_result = self.search_inverted_index(&ctx, query, de_rank_similar)?;+ let guard = self.index.guard();+ let ctx = guard.inverted_index().local_search_ctx();+ let inverted_index_result =+ self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;let correction = self.spell.as_ref().and_then(|s| s.correction(query));let sidebar = self.entity_sidebar(query);@@ -301,18 +389,19 @@ impl LocalSearcher {websites: &[inverted_index::WebsitePointer],query: &str,) -> Result<Vec<inverted_index::RetrievedWebpage>> {- let ctx = self.index.inverted_index.local_search_ctx();+ let guard = self.index.guard();+ let ctx = guard.inverted_index().local_search_ctx();let query = SearchQuery {query: query.to_string(),..Default::default()};- let query = Query::parse(&ctx, &query, &self.index.inverted_index)?;+ let query = Query::parse(&ctx, &query, guard.inverted_index())?;if query.is_empty() {return Err(Error::EmptyQuery.into());}- self.index.retrieve_websites(websites, &query)+ guard.inverted_index().retrieve_websites(websites, &query)}/// This function is mainly used for tests and benchmarks@@ -405,11 +494,11 @@ impl LocalSearcher {}pub fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {- self.index.get_webpage(url)+ self.index.guard().inverted_index().get_webpage(url)}pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {- self.index.get_homepage(url)+ self.index.guard().inverted_index().get_homepage(url)}pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex ff0b264c..6f934291 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -7,13 +7,14 @@ export const setGlobalApiBase = (apiBase: string) => (GLOBAL_API_BASE = apiBase)export type ApiOptions = {fetch?: typeof fetch;apiBase?: string;+ headers?: Record<string, string>;};export const requestPlain = (method: Method,url: string,body?: unknown,- options?: ApiOptions & { headers?: Record<string, string> },+ options?: ApiOptions,): {data: Promise<string>;cancel: (reason?: string) => void;diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.tsindex 3ce22391..5b3cd94c 100644--- a/frontend/src/routes/search/+page.server.ts+++ b/frontend/src/routes/search/+page.server.ts@@ -4,7 +4,7 @@ import type { Actions } from './$types';import { api } from '$lib/api';import { fetchRemoteOptic } from '$lib/optics';-export const load = async ({ locals, fetch, url }) => {+export const load = async ({ locals, fetch, url, getClientAddress }) => {const searchParams: SearchParams | undefined =(locals['form'] && extractSearchParams(locals['form'])) || undefined;@@ -34,7 +34,7 @@ export const load = async ({ locals, fetch, url }) => {fetchDiscussions: false,countResults: true,},- { fetch },+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },);const results = await data;