Skip to content

web search done right

make live index searchable

Mikkel Denkera6ea0d734cb8
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 99e1561d..4fe4ad91 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -41,7 +41,10 @@ use crate::{
};
use anyhow::Result;
-use std::{net::SocketAddr, sync::Arc};
+use std::{
+ net::{IpAddr, SocketAddr},
+ sync::Arc,
+};
use axum::{
http::StatusCode,
@@ -253,11 +256,12 @@ async fn search_metric<B>(
// It is very important that the ip address is not stored. It is only used
// for a probabilistic estimate of the number of unique users using a hyperloglog datastructure.
let mut ip = None;
+
if let Some(forwarded_for) = request.headers().get("x-forwarded-for") {
let forwarded_for = forwarded_for.to_str().unwrap_or_default();
if let Some(client_ip) = forwarded_for.split(',').next() {
- if let Ok(client_ip) = client_ip.trim().parse::<SocketAddr>() {
- ip = Some(client_ip.ip());
+ if let Ok(client_ip) = client_ip.trim().parse::<IpAddr>() {
+ ip = Some(client_ip);
}
}
}
diff --git a/core/src/distributed/member.rs b/core/src/distributed/member.rs
index d38c75f8..48cfae15 100644
--- a/core/src/distributed/member.rs
+++ b/core/src/distributed/member.rs
@@ -22,10 +22,23 @@ use crate::searcher::ShardId;
#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
pub enum Service {
- Searcher { host: SocketAddr, shard: ShardId },
- Api { host: SocketAddr },
- Webgraph { host: SocketAddr },
- Alice { host: SocketAddr },
+ Searcher {
+ host: SocketAddr,
+ shard: ShardId,
+ },
+ LiveIndex {
+ host: SocketAddr,
+ split_id: crate::feed::scheduler::SplitId,
+ },
+ Api {
+ host: SocketAddr,
+ },
+ Webgraph {
+ host: SocketAddr,
+ },
+ Alice {
+ host: SocketAddr,
+ },
}
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rs
index 3e19bf26..d6e93a1a 100644
--- a/core/src/entrypoint/live_index.rs
+++ b/core/src/entrypoint/live_index.rs
@@ -14,22 +14,108 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::path::Path;
+use std::{path::Path, sync::Arc};
use crate::{
config::{LiveIndexConfig, LiveIndexSchedulerConfig},
+ distributed::{
+ cluster::Cluster,
+ member::{Member, Service},
+ sonic,
+ },
feed::{self, index::FeedIndex},
+ inverted_index,
kv::rocksdb_store::RocksDbStore,
- live_index::IndexManager,
+ live_index::{Index, IndexManager},
+ ranking::inbound_similarity::InboundSimilarity,
+ searcher::{InitialWebsiteResult, LocalSearcher},
+ sonic_service,
webgraph::WebgraphBuilder,
};
use anyhow::Result;
+use tracing::info;
+
+use super::search_server::{RetrieveWebsites, Search};
+
+sonic_service!(SearchService, [RetrieveWebsites, Search]);
+
+pub struct SearchService {
+ local_searcher: LocalSearcher<Arc<Index>>,
+ // dropping the handle leaves the cluster
+ #[allow(unused)]
+ cluster_handle: Cluster,
+}
+
+impl SearchService {
+ async fn new(config: LiveIndexConfig) -> Result<Self> {
+ let inbound_similarity = InboundSimilarity::open(
+ Path::new(&config.host_centrality_store_path).join("inbound_similarity"),
+ )?;
+
+ let manager = IndexManager::new(config.clone())?;
+ let mut local_searcher = LocalSearcher::new(manager.index());
+
+ local_searcher.set_inbound_similarity(inbound_similarity);
+
+ tokio::task::spawn(manager.run());
+
+ let cluster_handle = Cluster::join(
+ Member {
+ id: config.cluster_id,
+ service: Service::LiveIndex {
+ host: config.host,
+ split_id: config.split_id,
+ },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.unwrap_or_default(),
+ )
+ .await?;
+
+ Ok(Self {
+ local_searcher,
+ cluster_handle,
+ })
+ }
+}
+
+#[async_trait::async_trait]
+impl sonic::service::Message<SearchService> for RetrieveWebsites {
+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ match server
+ .local_searcher
+ .retrieve_websites(&self.websites, &self.query)
+ {
+ Ok(response) => Ok(Some(response)),
+ Err(_) => Ok(None),
+ }
+ }
+}
+
+#[async_trait::async_trait]
+impl sonic::service::Message<SearchService> for Search {
+ type Response = Option<InitialWebsiteResult>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ match server.local_searcher.search_initial(&self.query, true) {
+ Ok(result) => Ok(Some(result)),
+ Err(_) => Ok(None),
+ }
+ }
+}
pub async fn serve(config: LiveIndexConfig) -> Result<()> {
- let manager = IndexManager::new(config)?;
- tokio::task::spawn(manager.run());
+ let addr = config.host;
+
+ let server = SearchService::new(config).await?.bind(&addr).await.unwrap();
+
+ info!("live index is ready to accept requests on {}", addr);
- todo!("accept search requests");
+ loop {
+ if let Err(e) = server.accept().await {
+ tracing::error!("{:?}", e);
+ }
+ }
}
pub fn schedule(config: LiveIndexSchedulerConfig) -> Result<()> {
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index b2a85aea..254c19cd 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -51,7 +51,7 @@ sonic_service!(
);
pub struct SearchService {
- local_searcher: LocalSearcher,
+ local_searcher: LocalSearcher<Index>,
// dropping the handle leaves the cluster
#[allow(unused)]
cluster_handle: Cluster,
diff --git a/core/src/feed/parser.rs b/core/src/feed/parser.rs
index de3734fa..cfb56920 100644
--- a/core/src/feed/parser.rs
+++ b/core/src/feed/parser.rs
@@ -83,7 +83,6 @@ fn parse_atom(feed: &str) -> Result<ParsedFeed> {
.filter(std::result::Result::is_ok)
.find(|attr| attr.as_ref().unwrap().key.as_ref() == b"href")
{
- dbg!(&href);
if let Ok(href) = href
.unescape_value()
.map_err(|e| anyhow!(e))
diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rs
index 0f876704..8a60cbbd 100644
--- a/core/src/feed/scheduler.rs
+++ b/core/src/feed/scheduler.rs
@@ -46,7 +46,7 @@ impl From<&Url> for Domain {
}
}
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub struct SplitId(uuid::Uuid);
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
diff --git a/core/src/index.rs b/core/src/index.rs
index 4e9ff281..7a7c5129 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -17,12 +17,10 @@
use std::collections::HashSet;
use std::fs;
use std::path::Path;
-use std::sync::{Arc, Mutex};
+use std::sync::Mutex;
use std::time::SystemTime;
-use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
-use url::Url;
use crate::collector::MainCollector;
use crate::inverted_index::{self, InvertedIndex};
@@ -147,22 +145,6 @@ impl Index {
Self::open(&self.path).expect("failed to open index")
}
-
- pub fn schema(&self) -> Arc<Schema> {
- self.inverted_index.schema()
- }
-
- pub fn num_segments(&self) -> usize {
- self.inverted_index.num_segments()
- }
-
- pub(crate) fn get_webpage(&self, url: &str) -> Option<inverted_index::RetrievedWebpage> {
- self.inverted_index.get_webpage(url)
- }
-
- pub fn get_homepage(&self, url: &Url) -> Option<inverted_index::RetrievedWebpage> {
- self.inverted_index.get_homepage(url)
- }
}
#[cfg(test)]
diff --git a/core/src/live_index.rs b/core/src/live_index.rs
index f66169e2..f69a28c5 100644
--- a/core/src/live_index.rs
+++ b/core/src/live_index.rs
@@ -17,7 +17,7 @@
use anyhow::Result;
use std::{
path::Path,
- sync::{Arc, RwLock},
+ sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
time::{Duration, SystemTime},
};
@@ -215,7 +215,7 @@ impl Crawler {
}
}
-struct Index {
+pub struct Index {
search_index: Arc<RwLock<crate::index::Index>>,
}
@@ -248,6 +248,14 @@ impl Index {
fn clone_inner_index(&self) -> Arc<RwLock<crate::index::Index>> {
self.search_index.clone()
}
+
+ pub fn read(&self) -> RwLockReadGuard<'_, crate::index::Index> {
+ self.search_index.read().unwrap_or_else(|e| e.into_inner())
+ }
+
+ pub fn write(&self) -> RwLockWriteGuard<'_, crate::index::Index> {
+ self.search_index.write().unwrap_or_else(|e| e.into_inner())
+ }
}
#[async_trait::async_trait]
@@ -364,4 +372,8 @@ impl IndexManager {
tokio::time::sleep(EVENT_LOOP_INTERVAL).await;
}
}
+
+ pub fn index(&self) -> Arc<Index> {
+ self.index.clone()
+ }
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 38d41d9a..ea541634 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{Arc, RwLockReadGuard};
use tantivy::schema::Value;
use tantivy::TantivyDocument;
@@ -25,7 +25,7 @@ use crate::config::{CollectorConfig, SnippetConfig};
use crate::entity_index::{EntityIndex, EntityMatch};
use crate::image_store::Image;
use crate::index::Index;
-use crate::inverted_index::RetrievedWebpage;
+use crate::inverted_index::{InvertedIndex, RetrievedWebpage};
use crate::query::Query;
use crate::ranking::inbound_similarity::InboundSimilarity;
#[cfg(not(feature = "libtorch"))]
@@ -40,13 +40,84 @@ use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpe
use crate::spell::Spell;
use crate::webgraph::Node;
use crate::webpage::region::Region;
-use crate::{inverted_index, Error, Result};
+use crate::{inverted_index, live_index, Error, Result};
use super::WebsitesResult;
use super::{InitialWebsiteResult, SearchQuery};
-pub struct LocalSearcher {
- index: Index,
+pub trait SearchableIndex {
+ type SearchGuard<'a>: SearchGuard<'a>
+ where
+ Self: 'a;
+
+ fn guard(&self) -> Self::SearchGuard<'_>;
+ fn optimize_for_search(&mut self);
+ fn set_snippet_config(&mut self, config: SnippetConfig);
+}
+
+pub trait SearchGuard<'a> {
+ fn search_index(&self) -> &'_ Index;
+ fn inverted_index(&self) -> &'_ InvertedIndex {
+ &self.search_index().inverted_index
+ }
+}
+
+impl SearchableIndex for Index {
+ type SearchGuard<'a> = NormalIndexSearchGuard<'a>;
+
+ fn guard(&self) -> Self::SearchGuard<'_> {
+ NormalIndexSearchGuard { search_index: self }
+ }
+
+ fn optimize_for_search(&mut self) {
+ self.optimize_for_search().unwrap();
+ }
+
+ fn set_snippet_config(&mut self, config: SnippetConfig) {
+ self.inverted_index.set_snippet_config(config);
+ }
+}
+
+pub struct NormalIndexSearchGuard<'a> {
+ search_index: &'a Index,
+}
+
+impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {
+ fn search_index(&self) -> &'_ Index {
+ self.search_index
+ }
+}
+
+impl SearchableIndex for Arc<live_index::Index> {
+ type SearchGuard<'a> = LiveIndexSearchGuard<'a>;
+
+ fn guard(&self) -> Self::SearchGuard<'_> {
+ LiveIndexSearchGuard {
+ lock_guard: self.read(),
+ }
+ }
+
+ fn optimize_for_search(&mut self) {
+ self.write().optimize_for_search().unwrap();
+ }
+
+ fn set_snippet_config(&mut self, config: SnippetConfig) {
+ self.write().inverted_index.set_snippet_config(config);
+ }
+}
+
+pub struct LiveIndexSearchGuard<'a> {
+ lock_guard: RwLockReadGuard<'a, crate::index::Index>,
+}
+
+impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {
+ fn search_index(&self) -> &'_ Index {
+ &self.lock_guard
+ }
+}
+
+pub struct LocalSearcher<I: SearchableIndex> {
+ index: I,
spell: Option<Spell>,
entity_index: Option<EntityIndex>,
inbound_similarity: Option<InboundSimilarity>,
@@ -55,8 +126,11 @@ pub struct LocalSearcher {
collector_config: CollectorConfig,
}
-impl From<Index> for LocalSearcher {
- fn from(index: Index) -> Self {
+impl<I> From<I> for LocalSearcher<I>
+where
+ I: SearchableIndex,
+{
+ fn from(index: I) -> Self {
Self::new(index)
}
}
@@ -67,10 +141,13 @@ struct InvertedIndexResult {
has_more: bool,
}
-impl LocalSearcher {
- pub fn new(index: Index) -> Self {
+impl<I> LocalSearcher<I>
+where
+ I: SearchableIndex,
+{
+ pub fn new(index: I) -> Self {
let mut index = index;
- index.optimize_for_search().unwrap();
+ index.optimize_for_search();
LocalSearcher {
index,
@@ -84,7 +161,7 @@ impl LocalSearcher {
}
pub fn build_spell_dict(&mut self) {
- self.spell = Some(Spell::for_index(&self.index));
+ self.spell = Some(Spell::for_index(self.index.guard().search_index()));
}
pub fn set_entity_index(&mut self, entity_index: EntityIndex) {
@@ -108,11 +185,16 @@ impl LocalSearcher {
}
pub fn set_snippet_config(&mut self, config: SnippetConfig) {
- self.index.inverted_index.set_snippet_config(config);
+ self.index.set_snippet_config(config);
}
- fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {
- let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;
+ fn parse_query<'a, G: SearchGuard<'a>>(
+ &'a self,
+ ctx: &Ctx,
+ guard: &G,
+ query: &SearchQuery,
+ ) -> Result<Query> {
+ let parsed_query = Query::parse(ctx, query, guard.inverted_index())?;
if parsed_query.is_empty() {
Err(Error::EmptyQuery.into())
@@ -121,10 +203,11 @@ impl LocalSearcher {
}
}
- fn ranker(
- &self,
+ fn ranker<'a, G: SearchGuard<'a>>(
+ &'a self,
query: &Query,
ctx: &Ctx,
+ guard: &G,
de_rank_similar: bool,
aggregator: SignalAggregator,
) -> Result<Ranker> {
@@ -132,7 +215,7 @@ impl LocalSearcher {
let mut ranker = Ranker::new(
aggregator,
- self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),
+ guard.inverted_index().fastfield_reader(&ctx.tv_searcher),
self.collector_config.clone(),
);
@@ -147,11 +230,12 @@ impl LocalSearcher {
if query_centrality_coeff > 0.0 {
if let Some(inbound_sim) = self.inbound_similarity.as_ref() {
ranker = ranker
- .with_max_docs(1_000, self.index.num_segments())
+ .with_max_docs(1_000, guard.inverted_index().num_segments())
.with_num_results(100);
let top_host_nodes =
- self.index
+ guard
+ .search_index()
.top_nodes(query, ctx, ranker.collector(ctx.clone()))?;
if !top_host_nodes.is_empty() {
@@ -167,15 +251,16 @@ impl LocalSearcher {
Ok(ranker
.with_max_docs(
self.collector_config.max_docs_considered,
- self.index.num_segments(),
+ guard.inverted_index().num_segments(),
)
.with_num_results(query.num_results())
.with_offset(query.offset()))
}
- fn search_inverted_index(
- &self,
+ fn search_inverted_index<'a, G: SearchGuard<'a>>(
+ &'a self,
ctx: &Ctx,
+ guard: &G,
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InvertedIndexResult> {
@@ -185,7 +270,7 @@ impl LocalSearcher {
self.lambda_model.clone(),
self.collector_config.clone(),
);
- let parsed_query = self.parse_query(ctx, &query)?;
+ let parsed_query = self.parse_query(ctx, guard, &query)?;
let mut aggregator = SignalAggregator::new(Some(&parsed_query));
@@ -212,7 +297,8 @@ impl LocalSearcher {
}
aggregator.set_region_count(
- self.index
+ guard
+ .search_index()
.region_count
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -223,17 +309,17 @@ impl LocalSearcher {
aggregator.set_linear_model(model.clone());
}
- let ranker = self.ranker(&parsed_query, ctx, de_rank_similar, aggregator)?;
+ let ranker = self.ranker(&parsed_query, ctx, guard, de_rank_similar, aggregator)?;
- let res = self.index.inverted_index.search_initial(
+ let res = guard.inverted_index().search_initial(
&parsed_query,
ctx,
ranker.collector(ctx.clone()),
)?;
- let fastfield_reader = self.index.inverted_index.fastfield_reader(&ctx.tv_searcher);
+ let fastfield_reader = guard.inverted_index().fastfield_reader(&ctx.tv_searcher);
- let ranking_websites = self.index.inverted_index.retrieve_ranking_websites(
+ let ranking_websites = guard.inverted_index().retrieve_ranking_websites(
ctx,
res.top_websites,
ranker.aggregator(),
@@ -243,7 +329,7 @@ impl LocalSearcher {
let pipe_top_n = pipeline.top_n;
let mut ranking_websites = pipeline.apply(ranking_websites);
- let schema = self.index.schema();
+ let schema = guard.inverted_index().schema();
for website in &mut ranking_websites {
let doc: TantivyDocument = ctx.tv_searcher.doc(website.pointer.address.into())?;
website.title = Some(
@@ -282,8 +368,10 @@ impl LocalSearcher {
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InitialWebsiteResult> {
- let ctx = self.index.inverted_index.local_search_ctx();
- let inverted_index_result = self.search_inverted_index(&ctx, query, de_rank_similar)?;
+ let guard = self.index.guard();
+ let ctx = guard.inverted_index().local_search_ctx();
+ let inverted_index_result =
+ self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;
let correction = self.spell.as_ref().and_then(|s| s.correction(query));
let sidebar = self.entity_sidebar(query);
@@ -301,18 +389,19 @@ impl LocalSearcher {
websites: &[inverted_index::WebsitePointer],
query: &str,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let ctx = self.index.inverted_index.local_search_ctx();
+ let guard = self.index.guard();
+ let ctx = guard.inverted_index().local_search_ctx();
let query = SearchQuery {
query: query.to_string(),
..Default::default()
};
- let query = Query::parse(&ctx, &query, &self.index.inverted_index)?;
+ let query = Query::parse(&ctx, &query, guard.inverted_index())?;
if query.is_empty() {
return Err(Error::EmptyQuery.into());
}
- self.index.retrieve_websites(websites, &query)
+ guard.inverted_index().retrieve_websites(websites, &query)
}
/// This function is mainly used for tests and benchmarks
@@ -405,11 +494,11 @@ impl LocalSearcher {
}
pub fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- self.index.get_webpage(url)
+ self.index.guard().inverted_index().get_webpage(url)
}
pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
- self.index.get_homepage(url)
+ self.index.guard().inverted_index().get_homepage(url)
}
pub fn get_entity_image(&self, image_id: &str) -> Option<Image> {
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index ff0b264c..6f934291 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -7,13 +7,14 @@ export const setGlobalApiBase = (apiBase: string) => (GLOBAL_API_BASE = apiBase)
export type ApiOptions = {
fetch?: typeof fetch;
apiBase?: string;
+ headers?: Record<string, string>;
};
export const requestPlain = (
method: Method,
url: string,
body?: unknown,
- options?: ApiOptions & { headers?: Record<string, string> },
+ options?: ApiOptions,
): {
data: Promise<string>;
cancel: (reason?: string) => void;
diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.ts
index 3ce22391..5b3cd94c 100644
--- a/frontend/src/routes/search/+page.server.ts
+++ b/frontend/src/routes/search/+page.server.ts
@@ -4,7 +4,7 @@ import type { Actions } from './$types';
import { api } from '$lib/api';
import { fetchRemoteOptic } from '$lib/optics';
-export const load = async ({ locals, fetch, url }) => {
+export const load = async ({ locals, fetch, url, getClientAddress }) => {
const searchParams: SearchParams | undefined =
(locals['form'] && extractSearchParams(locals['form'])) || undefined;
@@ -34,7 +34,7 @@ export const load = async ({ locals, fetch, url }) => {
fetchDiscussions: false,
countResults: true,
},
- { fetch },
+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },
);
const results = await data;