Skip to content

web search done right

admin cli endpoint to get number of pages in index

Mikkel Denkere68b7fba19b2
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rs
index 625e2eaf..281f99f6 100644
--- a/crates/core/src/entrypoint/admin/mod.rs
+++ b/crates/core/src/entrypoint/admin/mod.rs
@@ -114,3 +114,15 @@ pub async fn top_keyphrases(top: usize) -> Result<()> {
Ok(())
}
+
+pub async fn index_size() -> Result<()> {
+ let config = Config::load()?;
+ let mut conn: sonic::service::Connection<api::ManagementService> =
+ sonic::service::Connection::create(config.host).await?;
+
+ let size: api::SizeResponse = conn.send_without_timeout(api::Size).await?;
+
+ println!("Number of pages in index: {}", size.pages);
+
+ Ok(())
+}
diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rs
index 71564989..5f4025af 100644
--- a/crates/core/src/entrypoint/api.rs
+++ b/crates/core/src/entrypoint/api.rs
@@ -34,6 +34,8 @@ use crate::{
searcher::{DistributedSearcher, SearchClient},
};
+use super::search_server::SearchService;
+
fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counters> {
let search_counter_success = crate::metrics::Counter::default();
let search_counter_fail = crate::metrics::Counter::default();
@@ -102,7 +104,7 @@ pub struct ManagementService {
cluster: Arc<Cluster>,
searcher: DistributedSearcher,
}
-sonic_service!(ManagementService, [TopKeyphrases, ClusterStatus]);
+sonic_service!(ManagementService, [TopKeyphrases, ClusterStatus, Size]);
impl ManagementService {
pub async fn new(cluster: Arc<Cluster>) -> Result<Self> {
@@ -138,6 +140,55 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {
}
}
+#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
+pub struct Size;
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct SizeResponse {
+ pub pages: u64,
+}
+
+impl std::ops::Add for SizeResponse {
+ type Output = Self;
+ fn add(mut self, other: Self) -> Self {
+ self += other;
+ self
+ }
+}
+
+impl std::ops::AddAssign for SizeResponse {
+ fn add_assign(&mut self, other: Self) {
+ self.pages += other.pages;
+ }
+}
+
+impl sonic::service::Message<ManagementService> for Size {
+ type Response = SizeResponse;
+ async fn handle(self, server: &ManagementService) -> Self::Response {
+ let mut res = SizeResponse { pages: 0 };
+
+ let mut checked_shards = std::collections::HashSet::new();
+
+ for member in server.cluster.members().await {
+ if let Service::Searcher { host, shard } = member.service {
+ if checked_shards.contains(&shard) {
+ continue;
+ }
+
+ let mut client: sonic::service::Connection<SearchService> =
+ sonic::service::Connection::create(host).await.unwrap();
+ let size = client.send_without_timeout(Size).await.unwrap();
+
+ res += size;
+
+ checked_shards.insert(shard);
+ }
+ }
+
+ res
+ }
+}
+
async fn run_management(addr: SocketAddr, cluster: Arc<Cluster>) -> Result<()> {
let server = ManagementService::new(cluster).await?.bind(addr).await?;
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index ac6907ef..6630cf87 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -30,10 +30,12 @@ use crate::{
inverted_index::{self, KeyPhrase, RetrievedWebpage},
models::dual_encoder::DualEncoder,
ranking::models::{lambdamart::LambdaMART, linear::LinearRegression},
- searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
+ searcher::{InitialWebsiteResult, LocalSearcher, SearchGuard, SearchQuery, SearchableIndex},
Result,
};
+use super::api::{Size, SizeResponse};
+
sonic_service!(
SearchService,
[
@@ -42,6 +44,7 @@ sonic_service!(
GetWebpage,
GetHomepageDescriptions,
TopKeyPhrases,
+ Size,
]
);
@@ -175,3 +178,17 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
}
}
}
+
+impl sonic::service::Message<SearchService> for Size {
+ type Response = SizeResponse;
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ SizeResponse {
+ pages: server
+ .local_searcher
+ .index()
+ .guard()
+ .inverted_index()
+ .num_documents(),
+ }
+ }
+}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index cf03025f..1fef548c 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -206,6 +206,10 @@ impl InvertedIndex {
self.tantivy_index.searchable_segments().unwrap().len()
}
+ pub fn num_documents(&self) -> u64 {
+ self.reader.searcher().num_docs()
+ }
+
#[cfg(test)]
pub fn temporary() -> Result<Self> {
let path = crate::gen_temp_path();
diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs
index 6072fb55..f2e02c34 100644
--- a/crates/core/src/main.rs
+++ b/crates/core/src/main.rs
@@ -150,9 +150,22 @@ enum AmpcOptions {
#[derive(Subcommand)]
enum AdminOptions {
- Init { host: SocketAddr },
+ Init {
+ host: SocketAddr,
+ },
Status,
- TopKeyphrases { top: usize },
+ TopKeyphrases {
+ top: usize,
+ },
+
+ #[clap(subcommand)]
+ Index(AdminIndexOptions),
+}
+
+#[derive(Subcommand)]
+enum AdminIndexOptions {
+ /// Get the size of the index
+ Size,
}
#[derive(Subcommand)]
@@ -496,6 +509,15 @@ fn main() -> Result<()> {
.build()?
.block_on(entrypoint::admin::top_keyphrases(top))?;
}
+
+ AdminOptions::Index(index_options) => match index_options {
+ AdminIndexOptions::Size => {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()?
+ .block_on(entrypoint::admin::index_size())?;
+ }
+ },
},
}