web search done right
admin cli endpoint to get number of pages in index
Mikkel Denkere68b7fba19b2
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rsindex 625e2eaf..281f99f6 100644--- a/crates/core/src/entrypoint/admin/mod.rs+++ b/crates/core/src/entrypoint/admin/mod.rs@@ -114,3 +114,15 @@ pub async fn top_keyphrases(top: usize) -> Result<()> {Ok(())}++pub async fn index_size() -> Result<()> {+ let config = Config::load()?;+ let mut conn: sonic::service::Connection<api::ManagementService> =+ sonic::service::Connection::create(config.host).await?;++ let size: api::SizeResponse = conn.send_without_timeout(api::Size).await?;++ println!("Number of pages in index: {}", size.pages);++ Ok(())+}diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rsindex 71564989..5f4025af 100644--- a/crates/core/src/entrypoint/api.rs+++ b/crates/core/src/entrypoint/api.rs@@ -34,6 +34,8 @@ use crate::{searcher::{DistributedSearcher, SearchClient},};+use super::search_server::SearchService;+fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counters> {let search_counter_success = crate::metrics::Counter::default();let search_counter_fail = crate::metrics::Counter::default();@@ -102,7 +104,7 @@ pub struct ManagementService {cluster: Arc<Cluster>,searcher: DistributedSearcher,}-sonic_service!(ManagementService, [TopKeyphrases, ClusterStatus]);+sonic_service!(ManagementService, [TopKeyphrases, ClusterStatus, Size]);impl ManagementService {pub async fn new(cluster: Arc<Cluster>) -> Result<Self> {@@ -138,6 +140,55 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {}}+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]+pub struct Size;++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct SizeResponse {+ pub pages: u64,+}++impl std::ops::Add for SizeResponse {+ type Output = Self;+ fn add(mut self, other: Self) -> Self {+ self += other;+ self+ }+}++impl std::ops::AddAssign for SizeResponse {+ fn add_assign(&mut self, other: Self) {+ self.pages += other.pages;+ }+}++impl sonic::service::Message<ManagementService> for Size {+ type Response = SizeResponse;+ async fn handle(self, server: &ManagementService) -> Self::Response {+ let mut res = SizeResponse { pages: 0 };++ let mut checked_shards = std::collections::HashSet::new();++ for member in server.cluster.members().await {+ if let Service::Searcher { host, shard } = member.service {+ if checked_shards.contains(&shard) {+ continue;+ }++ let mut client: sonic::service::Connection<SearchService> =+ sonic::service::Connection::create(host).await.unwrap();+ let size = client.send_without_timeout(Size).await.unwrap();++ res += size;++ checked_shards.insert(shard);+ }+ }++ res+ }+}+async fn run_management(addr: SocketAddr, cluster: Arc<Cluster>) -> Result<()> {let server = ManagementService::new(cluster).await?.bind(addr).await?;diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex ac6907ef..6630cf87 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -30,10 +30,12 @@ use crate::{inverted_index::{self, KeyPhrase, RetrievedWebpage},models::dual_encoder::DualEncoder,ranking::models::{lambdamart::LambdaMART, linear::LinearRegression},- searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},+ searcher::{InitialWebsiteResult, LocalSearcher, SearchGuard, SearchQuery, SearchableIndex},Result,};+use super::api::{Size, SizeResponse};+sonic_service!(SearchService,[@@ -42,6 +44,7 @@ sonic_service!(GetWebpage,GetHomepageDescriptions,TopKeyPhrases,+ Size,]);@@ -175,3 +178,17 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {}}}++impl sonic::service::Message<SearchService> for Size {+ type Response = SizeResponse;+ async fn handle(self, server: &SearchService) -> Self::Response {+ SizeResponse {+ pages: server+ .local_searcher+ .index()+ .guard()+ .inverted_index()+ .num_documents(),+ }+ }+}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex cf03025f..1fef548c 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -206,6 +206,10 @@ impl InvertedIndex {self.tantivy_index.searchable_segments().unwrap().len()}+ pub fn num_documents(&self) -> u64 {+ self.reader.searcher().num_docs()+ }+#[cfg(test)]pub fn temporary() -> Result<Self> {let path = crate::gen_temp_path();diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 6072fb55..f2e02c34 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -150,9 +150,22 @@ enum AmpcOptions {#[derive(Subcommand)]enum AdminOptions {- Init { host: SocketAddr },+ Init {+ host: SocketAddr,+ },Status,- TopKeyphrases { top: usize },+ TopKeyphrases {+ top: usize,+ },++ #[clap(subcommand)]+ Index(AdminIndexOptions),+}++#[derive(Subcommand)]+enum AdminIndexOptions {+ /// Get the size of the index+ Size,}#[derive(Subcommand)]@@ -496,6 +509,15 @@ fn main() -> Result<()> {.build()?.block_on(entrypoint::admin::top_keyphrases(top))?;}++ AdminOptions::Index(index_options) => match index_options {+ AdminIndexOptions::Size => {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::admin::index_size())?;+ }+ },},}