Skip to content

web search done right

remove aggregations for simplicity

Mikkel Denker4fb6af6fed4f
diff --git a/crates/tantivy/examples/aggregation.rs b/crates/tantivy/examples/aggregation.rs
deleted file mode 100644
index 52080b8d..00000000
--- a/crates/tantivy/examples/aggregation.rs
+++ /dev/null
@@ -1,266 +0,0 @@
-// # Aggregation example
-//
-// This example shows how you can use built-in aggregations.
-// We will use nested aggregations with buckets and metrics:
-// - Range buckets and compute the average in each bucket.
-// - Term aggregation and compute the min price in each bucket
-// ---
-
-use serde_json::{Deserializer, Value};
-use tantivy::aggregation::agg_req::Aggregations;
-use tantivy::aggregation::agg_result::AggregationResults;
-use tantivy::aggregation::AggregationCollector;
-use tantivy::query::AllQuery;
-use tantivy::schema::{self, IndexRecordOption, Schema, TextFieldIndexing, FAST};
-use tantivy::{Index, IndexWriter, TantivyDocument};
-
-fn main() -> tantivy::Result<()> {
- // # Create Schema
- //
- // Lets create a schema for a footwear shop, with 4 fields: name, category, stock and price.
- // category, stock and price will be fast fields as that's the requirement
- // for aggregation queries.
- //
-
- let mut schema_builder = Schema::builder();
- // In preparation of the `TermsAggregation`, the category field is configured with:
- // - `set_fast`
- // - `raw` tokenizer
- //
- // The tokenizer is set to "raw", because the fast field uses the same dictionary as the
- // inverted index. (This behaviour will change in tantivy 0.20, where the fast field will
- // always be raw tokenized independent from the regular tokenizing)
- //
- let text_fieldtype = schema::TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_index_option(IndexRecordOption::WithFreqs)
- .set_tokenizer("raw"),
- )
- .set_fast(None)
- .set_stored();
- schema_builder.add_text_field("category", text_fieldtype);
- schema_builder.add_f64_field("stock", FAST);
- schema_builder.add_f64_field("price", FAST);
-
- let schema = schema_builder.build();
-
- // # Indexing documents
- //
- // Lets index a bunch of documents for this example.
- let index = Index::create_in_ram(schema.clone());
-
- let data = r#"{
- "name": "Almond Toe Court Shoes, Patent Black",
- "category": "Womens Footwear",
- "price": 99.00,
- "stock": 5
- }
- {
- "name": "Suede Shoes, Blue",
- "category": "Womens Footwear",
- "price": 42.00,
- "stock": 4
- }
- {
- "name": "Leather Driver Saddle Loafers, Tan",
- "category": "Mens Footwear",
- "price": 34.00,
- "stock": 12
- }
- {
- "name": "Flip Flops, Red",
- "category": "Mens Footwear",
- "price": 19.00,
- "stock": 6
- }
- {
- "name": "Flip Flops, Blue",
- "category": "Mens Footwear",
- "price": 19.00,
- "stock": 0
- }
- {
- "name": "Gold Button Cardigan, Black",
- "category": "Womens Casualwear",
- "price": 167.00,
- "stock": 6
- }
- {
- "name": "Cotton Shorts, Medium Red",
- "category": "Womens Casualwear",
- "price": 30.00,
- "stock": 5
- }
- {
- "name": "Fine Stripe Short SleeveShirt, Grey",
- "category": "Mens Casualwear",
- "price": 49.99,
- "stock": 9
- }
- {
- "name": "Fine Stripe Short SleeveShirt, Green",
- "category": "Mens Casualwear",
- "price": 49.99,
- "offer": 39.99,
- "stock": 9
- }
- {
- "name": "Sharkskin Waistcoat, Charcoal",
- "category": "Mens Formalwear",
- "price": 75.00,
- "stock": 2
- }
- {
- "name": "Lightweight Patch PocketBlazer, Deer",
- "category": "Mens Formalwear",
- "price": 175.50,
- "stock": 1
- }
- {
- "name": "Bird Print Dress, Black",
- "category": "Womens Formalwear",
- "price": 270.00,
- "stock": 10
- }
- {
- "name": "Mid Twist Cut-Out Dress, Pink",
- "category": "Womens Formalwear",
- "price": 540.00,
- "stock": 5
- }"#;
-
- let stream = Deserializer::from_str(data).into_iter::<Value>();
-
- let mut index_writer: IndexWriter = index.writer(50_000_000)?;
- let mut num_indexed = 0;
- for value in stream {
- let doc = TantivyDocument::parse_json(&schema, &serde_json::to_string(&value.unwrap())?)?;
- index_writer.add_document(doc)?;
- num_indexed += 1;
- if num_indexed > 4 {
- // Writing the first segment
- index_writer.commit()?;
- }
- }
-
- // Writing the second segment
- index_writer.commit()?;
-
- // We have two segments now. The `AggregationCollector` will run the aggregation on each
- // segment and then merge the results into an `IntermediateAggregationResult`.
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- // ---
- // # Aggregation Query
- //
- //
- // We can construct the query by building the request structure or by deserializing from JSON.
- // The JSON API is more stable and therefore recommended.
- //
- // ## Request 1
-
- let agg_req_str = r#"
- {
- "group_by_stock": {
- "aggs": {
- "average_price": { "avg": { "field": "price" } }
- },
- "range": {
- "field": "stock",
- "ranges": [
- { "key": "few", "to": 1.0 },
- { "key": "some", "from": 1.0, "to": 10.0 },
- { "key": "many", "from": 10.0 }
- ]
- }
- }
- } "#;
-
- // In this Aggregation we want to get the average price for different groups, depending on how
- // many items are in stock. We define custom ranges `few`, `some`, `many` via the
- // range aggregation.
- // For every bucket we want the average price, so we create a nested metric aggregation on the
- // range bucket aggregation. Only buckets support nested aggregations.
- // ### Request JSON API
- //
-
- let agg_req: Aggregations = serde_json::from_str(agg_req_str)?;
- let collector = AggregationCollector::from_aggs(agg_req, Default::default());
-
- // We use the `AllQuery` which will pass all documents to the AggregationCollector.
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let res: Value = serde_json::to_value(agg_res)?;
-
- // ### Aggregation Result
- //
- // The resulting structure deserializes in the same JSON format as elastic search.
- //
- let expected_res = r#"
- {
- "group_by_stock":{
- "buckets":[
- {"average_price":{"value":19.0},"doc_count":1,"key":"few","to":1.0},
- {"average_price":{"value":124.748},"doc_count":10,"from":1.0,"key":"some","to":10.0},
- {"average_price":{"value":152.0},"doc_count":2,"from":10.0,"key":"many"}
- ]
- }
- }
- "#;
- let expected_json: Value = serde_json::from_str(expected_res)?;
- assert_eq!(expected_json, res);
-
- // ### Request 2
- //
- // Now we are interested in the minimum price per category, so we create a bucket per
- // category via `TermsAggregation`. We are interested in the highest minimum prices, and set the
- // order of the buckets `"order": { "min_price": "desc" }` to be sorted by the the metric of
- // the sub aggregation. (awesome)
- //
- let agg_req_str = r#"
- {
- "min_price_per_category": {
- "aggs": {
- "min_price": { "min": { "field": "price" } }
- },
- "terms": {
- "field": "category",
- "min_doc_count": 1,
- "order": { "min_price": "desc" }
- }
- }
- } "#;
-
- let agg_req: Aggregations = serde_json::from_str(agg_req_str)?;
-
- let collector = AggregationCollector::from_aggs(agg_req, Default::default());
-
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let res: Value = serde_json::to_value(agg_res)?;
-
- // Minimum price per category, sorted by minimum price descending
- //
- // As you can see, the starting prices for `Formalwear` are higher than `Casualwear`.
- //
- let expected_res = r#"
- {
- "min_price_per_category": {
- "buckets": [
- { "doc_count": 2, "key": "Womens Formalwear", "min_price": { "value": 270.0 } },
- { "doc_count": 2, "key": "Mens Formalwear", "min_price": { "value": 75.0 } },
- { "doc_count": 2, "key": "Mens Casualwear", "min_price": { "value": 49.99 } },
- { "doc_count": 2, "key": "Womens Footwear", "min_price": { "value": 42.0 } },
- { "doc_count": 2, "key": "Womens Casualwear", "min_price": { "value": 30.0 } },
- { "doc_count": 3, "key": "Mens Footwear", "min_price": { "value": 19.0 } }
- ],
- "sum_other_doc_count": 0
- }
- }
- "#;
- let expected_json: Value = serde_json::from_str(expected_res)?;
-
- assert_eq!(expected_json, res);
-
- Ok(())
-}
diff --git a/crates/tantivy/src/aggregation/README.md b/crates/tantivy/src/aggregation/README.md
deleted file mode 100644
index c96454d0..00000000
--- a/crates/tantivy/src/aggregation/README.md
+++ /dev/null
@@ -1,36 +0,0 @@
-# Contributing
-
-When adding new bucket aggregation make sure to extend the "test_aggregation_flushing" test for at least 2 levels.
-
-
-
-# Code Organization
-
-Tantivy's aggregations have been designed to mimic the
-[aggregations of elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html).
-
-The code is organized in submodules:
-
-## bucket
-Contains all bucket aggregations, like range aggregation. These bucket aggregations group documents into buckets and can contain sub-aggregations.
-
-## metric
-Contains all metric aggregations, like average aggregation. Metric aggregations do not have sub aggregations.
-
-#### agg_req
-agg_req contains the users aggregation request. Deserialization from json is compatible with elasticsearch aggregation requests.
-
-#### agg_req_with_accessor
-agg_req_with_accessor contains the users aggregation request enriched with fast field accessors etc, which are
-used during collection.
-
-#### segment_agg_result
-segment_agg_result contains the aggregation result tree, which is used for collection of a segment.
-The tree from agg_req_with_accessor is passed during collection.
-
-#### intermediate_agg_result
-intermediate_agg_result contains the aggregation tree for merging with other trees.
-
-#### agg_result
-agg_result contains the final aggregation tree.
-
diff --git a/crates/tantivy/src/aggregation/agg_limits.rs b/crates/tantivy/src/aggregation/agg_limits.rs
deleted file mode 100644
index 44d802cc..00000000
--- a/crates/tantivy/src/aggregation/agg_limits.rs
+++ /dev/null
@@ -1,138 +0,0 @@
-use std::collections::HashMap;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::Arc;
-
-use crate::common::ByteCount;
-
-use super::collector::DEFAULT_MEMORY_LIMIT;
-use super::{AggregationError, DEFAULT_BUCKET_LIMIT};
-
-/// An estimate for memory consumption. Non recursive
-pub trait MemoryConsumption {
- fn memory_consumption(&self) -> usize;
-}
-
-impl<K, V, S> MemoryConsumption for HashMap<K, V, S> {
- fn memory_consumption(&self) -> usize {
- let capacity = self.capacity();
- (std::mem::size_of::<K>() + std::mem::size_of::<V>() + 1) * capacity
- }
-}
-
-/// Aggregation memory limit after which the request fails. Defaults to DEFAULT_MEMORY_LIMIT
-/// (500MB). The limit is shared by all SegmentCollectors
-pub struct AggregationLimits {
- /// The counter which is shared between the aggregations for one request.
- memory_consumption: Arc<AtomicU64>,
- /// The memory_limit in bytes
- memory_limit: ByteCount,
- /// The maximum number of buckets _returned_
- /// This is not counting intermediate buckets.
- bucket_limit: u32,
-}
-impl Clone for AggregationLimits {
- fn clone(&self) -> Self {
- Self {
- memory_consumption: Arc::clone(&self.memory_consumption),
- memory_limit: self.memory_limit,
- bucket_limit: self.bucket_limit,
- }
- }
-}
-
-impl Default for AggregationLimits {
- fn default() -> Self {
- Self {
- memory_consumption: Default::default(),
- memory_limit: DEFAULT_MEMORY_LIMIT.into(),
- bucket_limit: DEFAULT_BUCKET_LIMIT,
- }
- }
-}
-
-impl AggregationLimits {
- /// *memory_limit*
- /// memory_limit is defined in bytes.
- /// Aggregation fails when the estimated memory consumption of the aggregation is higher than
- /// memory_limit.
- /// memory_limit will default to `DEFAULT_MEMORY_LIMIT` (500MB)
- ///
- /// *bucket_limit*
- /// Limits the maximum number of buckets returned from an aggregation request.
- /// bucket_limit will default to `DEFAULT_BUCKET_LIMIT` (65000)
- ///
- /// Note: The returned instance contains a Arc shared counter to track memory consumption.
- pub fn new(memory_limit: Option<u64>, bucket_limit: Option<u32>) -> Self {
- Self {
- memory_consumption: Default::default(),
- memory_limit: memory_limit.unwrap_or(DEFAULT_MEMORY_LIMIT).into(),
- bucket_limit: bucket_limit.unwrap_or(DEFAULT_BUCKET_LIMIT),
- }
- }
-
- /// Create a new ResourceLimitGuard, that will release the memory when dropped.
- pub fn new_guard(&self) -> ResourceLimitGuard {
- ResourceLimitGuard {
- // The counter which is shared between the aggregations for one request.
- memory_consumption: Arc::clone(&self.memory_consumption),
- // The memory_limit in bytes
- memory_limit: self.memory_limit,
- allocated_with_the_guard: 0,
- }
- }
-
- pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {
- let prev_value = self
- .memory_consumption
- .fetch_add(add_num_bytes, Ordering::Relaxed);
- validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;
- Ok(())
- }
-
- pub(crate) fn get_bucket_limit(&self) -> u32 {
- self.bucket_limit
- }
-}
-
-fn validate_memory_consumption(
- memory_consumption: u64,
- memory_limit: ByteCount,
-) -> Result<(), AggregationError> {
- // Load the estimated memory consumed by the aggregations
- let memory_consumed: ByteCount = memory_consumption.into();
- if memory_consumed > memory_limit {
- return Err(AggregationError::MemoryExceeded {
- limit: memory_limit,
- current: memory_consumed,
- });
- }
- Ok(())
-}
-
-pub struct ResourceLimitGuard {
- /// The counter which is shared between the aggregations for one request.
- memory_consumption: Arc<AtomicU64>,
- /// The memory_limit in bytes
- memory_limit: ByteCount,
- /// Allocated memory with this guard.
- allocated_with_the_guard: u64,
-}
-
-impl ResourceLimitGuard {
- pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {
- let prev_value = self
- .memory_consumption
- .fetch_add(add_num_bytes, Ordering::Relaxed);
- validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;
- Ok(())
- }
-}
-
-impl Drop for ResourceLimitGuard {
- /// Removes the memory consumed tracked by this _instance_ of AggregationLimits.
- /// This is used to clear the segment specific memory consumption all at once.
- fn drop(&mut self) {
- self.memory_consumption
- .fetch_sub(self.allocated_with_the_guard, Ordering::Relaxed);
- }
-}
diff --git a/crates/tantivy/src/aggregation/agg_req.rs b/crates/tantivy/src/aggregation/agg_req.rs
deleted file mode 100644
index f849cad3..00000000
--- a/crates/tantivy/src/aggregation/agg_req.rs
+++ /dev/null
@@ -1,219 +0,0 @@
-//! Contains the aggregation request tree. Used to build an
-//! [`AggregationCollector`](super::AggregationCollector).
-//!
-//! [`Aggregations`] is the top level entry point to create a request, which is a `HashMap<String,
-//! Aggregation>`.
-//!
-//! Requests are compatible with the json format of elasticsearch.
-//!
-//! # Example
-//!
-//! ```
-//! use tantivy::aggregation::agg_req::Aggregations;
-//!
-//! let elasticsearch_compatible_json_req = r#"
-//! {
-//! "range": {
-//! "range": {
-//! "field": "score",
-//! "ranges": [
-//! { "from": 3.0, "to": 7.0 },
-//! { "from": 7.0, "to": 20.0 }
-//! ]
-//! }
-//! }
-//! }"#;
-//! let _agg_req: Aggregations = serde_json::from_str(elasticsearch_compatible_json_req).unwrap();
-//! ```
-
-use std::collections::{HashMap, HashSet};
-
-use serde::{Deserialize, Serialize};
-
-use super::bucket::{
- DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,
-};
-use super::metric::{
- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
- PercentilesAggregationReq, StatsAggregation, SumAggregation, TopHitsAggregation,
-};
-
-/// The top-level aggregation request structure, which contains [`Aggregation`] and their user
-/// defined names. It is also used in buckets aggregations to define sub-aggregations.
-///
-/// The key is the user defined name of the aggregation.
-pub type Aggregations = HashMap<String, Aggregation>;
-
-/// Aggregation request.
-///
-/// An aggregation is either a bucket or a metric.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(try_from = "AggregationForDeserialization")]
-pub struct Aggregation {
- /// The aggregation variant, which can be either a bucket or a metric.
- #[serde(flatten)]
- pub agg: AggregationVariants,
- /// on the document set in the bucket.
- #[serde(rename = "aggs")]
- #[serde(skip_serializing_if = "Aggregations::is_empty")]
- pub sub_aggregation: Aggregations,
-}
-
-/// In order to display proper error message, we cannot rely on flattening
-/// the json enum. Instead we introduce an intermediary struct to separate
-/// the aggregation from the subaggregation.
-#[derive(Deserialize)]
-struct AggregationForDeserialization {
- #[serde(flatten)]
- pub aggs_remaining_json: serde_json::Value,
- #[serde(rename = "aggs")]
- #[serde(default)]
- pub sub_aggregation: Aggregations,
-}
-
-impl TryFrom<AggregationForDeserialization> for Aggregation {
- type Error = serde_json::Error;
-
- fn try_from(value: AggregationForDeserialization) -> serde_json::Result<Self> {
- let AggregationForDeserialization {
- aggs_remaining_json,
- sub_aggregation,
- } = value;
- let agg: AggregationVariants = serde_json::from_value(aggs_remaining_json)?;
- Ok(Aggregation {
- agg,
- sub_aggregation,
- })
- }
-}
-
-impl Aggregation {
- pub(crate) fn sub_aggregation(&self) -> &Aggregations {
- &self.sub_aggregation
- }
-
- fn get_fast_field_names(&self, fast_field_names: &mut HashSet<String>) {
- fast_field_names.extend(
- self.agg
- .get_fast_field_names()
- .iter()
- .map(|s| s.to_string()),
- );
- fast_field_names.extend(get_fast_field_names(&self.sub_aggregation));
- }
-}
-
-/// Extract all fast field names used in the tree.
-pub fn get_fast_field_names(aggs: &Aggregations) -> HashSet<String> {
- let mut fast_field_names = Default::default();
- for el in aggs.values() {
- el.get_fast_field_names(&mut fast_field_names)
- }
- fast_field_names
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// All aggregation types.
-pub enum AggregationVariants {
- // Bucket aggregation types
- /// Put data into buckets of user-defined ranges.
- #[serde(rename = "range")]
- Range(RangeAggregation),
- /// Put data into a histogram.
- #[serde(rename = "histogram")]
- Histogram(HistogramAggregation),
- /// Put data into a date histogram.
- #[serde(rename = "date_histogram")]
- DateHistogram(DateHistogramAggregationReq),
- /// Put data into buckets of terms.
- #[serde(rename = "terms")]
- Terms(TermsAggregation),
-
- // Metric aggregation types
- /// Computes the average of the extracted values.
- #[serde(rename = "avg")]
- Average(AverageAggregation),
- /// Counts the number of extracted values.
- #[serde(rename = "value_count")]
- Count(CountAggregation),
- /// Finds the maximum value.
- #[serde(rename = "max")]
- Max(MaxAggregation),
- /// Finds the minimum value.
- #[serde(rename = "min")]
- Min(MinAggregation),
- /// Computes a collection of statistics (`min`, `max`, `sum`, `count`, and `avg`) over the
- /// extracted values.
- #[serde(rename = "stats")]
- Stats(StatsAggregation),
- /// Computes a collection of estended statistics (`min`, `max`, `sum`, `count`, `avg`,
- /// `sum_of_squares`, `variance`, `variance_sampling`, `std_deviation`,
- /// `std_deviation_sampling`) over the extracted values.
- #[serde(rename = "extended_stats")]
- ExtendedStats(ExtendedStatsAggregation),
- /// Computes the sum of the extracted values.
- #[serde(rename = "sum")]
- Sum(SumAggregation),
- /// Computes the sum of the extracted values.
- #[serde(rename = "percentiles")]
- Percentiles(PercentilesAggregationReq),
- /// Finds the top k values matching some order
- #[serde(rename = "top_hits")]
- TopHits(TopHitsAggregation),
-}
-
-impl AggregationVariants {
- /// Returns the name of the fields used by the aggregation.
- pub fn get_fast_field_names(&self) -> Vec<&str> {
- match self {
- AggregationVariants::Terms(terms) => vec![terms.field.as_str()],
- AggregationVariants::Range(range) => vec![range.field.as_str()],
- AggregationVariants::Histogram(histogram) => vec![histogram.field.as_str()],
- AggregationVariants::DateHistogram(histogram) => vec![histogram.field.as_str()],
- AggregationVariants::Average(avg) => vec![avg.field_name()],
- AggregationVariants::Count(count) => vec![count.field_name()],
- AggregationVariants::Max(max) => vec![max.field_name()],
- AggregationVariants::Min(min) => vec![min.field_name()],
- AggregationVariants::Stats(stats) => vec![stats.field_name()],
- AggregationVariants::ExtendedStats(extended_stats) => vec![extended_stats.field_name()],
- AggregationVariants::Sum(sum) => vec![sum.field_name()],
- AggregationVariants::Percentiles(per) => vec![per.field_name()],
- AggregationVariants::TopHits(top_hits) => top_hits.field_names(),
- }
- }
-
- pub(crate) fn as_range(&self) -> Option<&RangeAggregation> {
- match &self {
- AggregationVariants::Range(range) => Some(range),
- _ => None,
- }
- }
- pub(crate) fn as_histogram(&self) -> crate::Result<Option<HistogramAggregation>> {
- match &self {
- AggregationVariants::Histogram(histogram) => Ok(Some(histogram.clone())),
- AggregationVariants::DateHistogram(histogram) => {
- Ok(Some(histogram.to_histogram_req()?))
- }
- _ => Ok(None),
- }
- }
- pub(crate) fn as_term(&self) -> Option<&TermsAggregation> {
- match &self {
- AggregationVariants::Terms(terms) => Some(terms),
- _ => None,
- }
- }
- pub(crate) fn as_top_hits(&self) -> Option<&TopHitsAggregation> {
- match &self {
- AggregationVariants::TopHits(top_hits) => Some(top_hits),
- _ => None,
- }
- }
-
- pub(crate) fn as_percentile(&self) -> Option<&PercentilesAggregationReq> {
- match &self {
- AggregationVariants::Percentiles(percentile_req) => Some(percentile_req),
- _ => None,
- }
- }
-}
diff --git a/crates/tantivy/src/aggregation/agg_req_with_accessor.rs b/crates/tantivy/src/aggregation/agg_req_with_accessor.rs
deleted file mode 100644
index 54534213..00000000
--- a/crates/tantivy/src/aggregation/agg_req_with_accessor.rs
+++ /dev/null
@@ -1,430 +0,0 @@
-//! This will enhance the request tree with access to the fastfield and metadata.
-
-use std::collections::HashMap;
-use std::io;
-
-use crate::columnar::{Column, ColumnBlockAccessor, ColumnType, DynamicColumn, StrColumn};
-
-use super::agg_limits::ResourceLimitGuard;
-use super::agg_req::{Aggregation, AggregationVariants, Aggregations};
-use super::bucket::{
- DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,
-};
-use super::metric::{
- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
- StatsAggregation, SumAggregation,
-};
-use super::segment_agg_result::AggregationLimits;
-use super::VecWithNames;
-use crate::aggregation::{f64_to_fastfield_u64, Key};
-use crate::index::SegmentReader;
-use crate::SegmentOrdinal;
-
-#[derive(Default)]
-pub(crate) struct AggregationsWithAccessor {
- pub aggs: VecWithNames<AggregationWithAccessor>,
-}
-
-impl AggregationsWithAccessor {
- fn from_data(aggs: VecWithNames<AggregationWithAccessor>) -> Self {
- Self { aggs }
- }
-
- pub fn is_empty(&self) -> bool {
- self.aggs.is_empty()
- }
-}
-
-pub struct AggregationWithAccessor {
- pub(crate) segment_ordinal: SegmentOrdinal,
- /// In general there can be buckets without fast field access, e.g. buckets that are created
- /// based on search terms. That is not that case currently, but eventually this needs to be
- /// Option or moved.
- pub(crate) accessor: Column<u64>,
- /// Load insert u64 for missing use case
- pub(crate) missing_value_for_accessor: Option<u64>,
- pub(crate) str_dict_column: Option<StrColumn>,
- pub(crate) field_type: ColumnType,
- pub(crate) sub_aggregation: AggregationsWithAccessor,
- pub(crate) limits: ResourceLimitGuard,
- pub(crate) column_block_accessor: ColumnBlockAccessor<u64>,
- /// Used for missing term aggregation, which checks all columns for existence.
- /// And also for `top_hits` aggregation, which may sort on multiple fields.
- /// By convention the missing aggregation is chosen, when this property is set
- /// (instead bein set in `agg`).
- /// If this needs to used by other aggregations, we need to refactor this.
- // NOTE: we can make all other aggregations use this instead of the `accessor` and `field_type`
- // (making them obsolete) But will it have a performance impact?
- pub(crate) accessors: Vec<(Column<u64>, ColumnType)>,
- /// Map field names to all associated column accessors.
- /// This field is used for `docvalue_fields`, which is currently only supported for `top_hits`.
- pub(crate) value_accessors: HashMap<String, Vec<DynamicColumn>>,
- pub(crate) agg: Aggregation,
-}
-
-impl AggregationWithAccessor {
- /// May return multiple accessors if the aggregation is e.g. on mixed field types.
- fn try_from_agg(
- agg: &Aggregation,
- sub_aggregation: &Aggregations,
- reader: &SegmentReader,
- segment_ordinal: SegmentOrdinal,
- limits: AggregationLimits,
- ) -> crate::Result<Vec<AggregationWithAccessor>> {
- let mut agg = agg.clone();
-
- let add_agg_with_accessor = |agg: &Aggregation,
- accessor: Column<u64>,
- column_type: ColumnType,
- aggs: &mut Vec<AggregationWithAccessor>|
- -> crate::Result<()> {
- let res = AggregationWithAccessor {
- segment_ordinal,
- accessor,
- accessors: Default::default(),
- value_accessors: Default::default(),
- field_type: column_type,
- sub_aggregation: get_aggs_with_segment_accessor_and_validate(
- sub_aggregation,
- reader,
- segment_ordinal,
- &limits,
- )?,
- agg: agg.clone(),
- limits: limits.new_guard(),
- missing_value_for_accessor: None,
- str_dict_column: None,
- column_block_accessor: Default::default(),
- };
- aggs.push(res);
- Ok(())
- };
-
- let add_agg_with_accessors = |agg: &Aggregation,
- accessors: Vec<(Column<u64>, ColumnType)>,
- aggs: &mut Vec<AggregationWithAccessor>,
- value_accessors: HashMap<String, Vec<DynamicColumn>>|
- -> crate::Result<()> {
- let (accessor, field_type) = accessors.first().expect("at least one accessor");
- let res = AggregationWithAccessor {
- segment_ordinal,
- // TODO: We should do away with the `accessor` field altogether
- accessor: accessor.clone(),
- value_accessors,
- field_type: *field_type,
- accessors,
- sub_aggregation: get_aggs_with_segment_accessor_and_validate(
- sub_aggregation,
- reader,
- segment_ordinal,
- &limits,
- )?,
- agg: agg.clone(),
- limits: limits.new_guard(),
- missing_value_for_accessor: None,
- str_dict_column: None,
- column_block_accessor: Default::default(),
- };
- aggs.push(res);
- Ok(())
- };
-
- let mut res: Vec<AggregationWithAccessor> = Vec::new();
- use AggregationVariants::*;
-
- match agg.agg {
- Range(RangeAggregation {
- field: ref field_name,
- ..
- }) => {
- let (accessor, column_type) =
- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
- }
- Histogram(HistogramAggregation {
- field: ref field_name,
- ..
- }) => {
- let (accessor, column_type) =
- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
- }
- DateHistogram(DateHistogramAggregationReq {
- field: ref field_name,
- ..
- }) => {
- let (accessor, column_type) =
- // Only DateTime is supported for DateHistogram
- get_ff_reader(reader, field_name, Some(&[ColumnType::DateTime]))?;
- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
- }
- Terms(TermsAggregation {
- field: ref field_name,
- ref missing,
- ..
- }) => {
- let str_dict_column = reader.fast_fields().str(field_name)?;
- let allowed_column_types = [
- ColumnType::I64,
- ColumnType::U64,
- ColumnType::F64,
- ColumnType::Str,
- ColumnType::DateTime,
- ColumnType::Bool,
- ColumnType::IpAddr,
- // ColumnType::Bytes Unsupported
- ];
-
- // In case the column is empty we want the shim column to match the missing type
- let fallback_type = missing
- .as_ref()
- .map(|missing| match missing {
- Key::Str(_) => ColumnType::Str,
- Key::F64(_) => ColumnType::F64,
- })
- .unwrap_or(ColumnType::U64);
- let column_and_types = get_all_ff_reader_or_empty(
- reader,
- field_name,
- Some(&allowed_column_types),
- fallback_type,
- )?;
- let missing_and_more_than_one_col = column_and_types.len() > 1 && missing.is_some();
- let text_on_non_text_col = column_and_types.len() == 1
- && column_and_types[0].1.numerical_type().is_some()
- && missing
- .as_ref()
- .map(|m| matches!(m, Key::Str(_)))
- .unwrap_or(false);
-
- // Actually we could convert the text to a number and have the fast path, if it is
- // provided in Rfc3339 format. But this use case is probably common
- // enough to justify the effort.
- let text_on_date_col = column_and_types.len() == 1
- && column_and_types[0].1 == ColumnType::DateTime
- && missing
- .as_ref()
- .map(|m| matches!(m, Key::Str(_)))
- .unwrap_or(false);
-
- let use_special_missing_agg =
- missing_and_more_than_one_col || text_on_non_text_col || text_on_date_col;
- if use_special_missing_agg {
- let column_and_types =
- get_all_ff_reader_or_empty(reader, field_name, None, fallback_type)?;
-
- let accessors = column_and_types
- .iter()
- .map(|c_t| (c_t.0.clone(), c_t.1))
- .collect();
- add_agg_with_accessors(&agg, accessors, &mut res, Default::default())?;
- }
-
- for (accessor, column_type) in column_and_types {
- let missing_value_term_agg = if use_special_missing_agg {
- None
- } else {
- missing.clone()
- };
-
- let missing_value_for_accessor = if let Some(missing) =
- missing_value_term_agg.as_ref()
- {
- get_missing_val(column_type, missing, agg.agg.get_fast_field_names()[0])?
- } else {
- None
- };
-
- let agg = AggregationWithAccessor {
- segment_ordinal,
- missing_value_for_accessor,
- accessor,
- accessors: Default::default(),
- value_accessors: Default::default(),
- field_type: column_type,
- sub_aggregation: get_aggs_with_segment_accessor_and_validate(
- sub_aggregation,
- reader,
- segment_ordinal,
- &limits,
- )?,
- agg: agg.clone(),
- str_dict_column: str_dict_column.clone(),
- limits: limits.new_guard(),
- column_block_accessor: Default::default(),
- };
- res.push(agg);
- }
- }
- Average(AverageAggregation {
- field: ref field_name,
- ..
- })
- | Count(CountAggregation {
- field: ref field_name,
- ..
- })
- | Max(MaxAggregation {
- field: ref field_name,
- ..
- })
- | Min(MinAggregation {
- field: ref field_name,
- ..
- })
- | Stats(StatsAggregation {
- field: ref field_name,
- ..
- })
- | ExtendedStats(ExtendedStatsAggregation {
- field: ref field_name,
- ..
- })
- | Sum(SumAggregation {
- field: ref field_name,
- ..
- }) => {
- let (accessor, column_type) =
- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;
- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
- }
- Percentiles(ref percentiles) => {
- let (accessor, column_type) = get_ff_reader(
- reader,
- percentiles.field_name(),
- Some(get_numeric_or_date_column_types()),
- )?;
- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;
- }
- TopHits(ref mut top_hits) => {
- top_hits.validate_and_resolve_field_names(reader.fast_fields().columnar())?;
- let accessors: Vec<(Column<u64>, ColumnType)> = top_hits
- .field_names()
- .iter()
- .map(|field| {
- get_ff_reader(reader, field, Some(get_numeric_or_date_column_types()))
- })
- .collect::<crate::Result<_>>()?;
-
- let value_accessors = top_hits
- .value_field_names()
- .iter()
- .map(|field_name| {
- Ok((
- field_name.to_string(),
- get_dynamic_columns(reader, field_name)?,
- ))
- })
- .collect::<crate::Result<_>>()?;
-
- add_agg_with_accessors(&agg, accessors, &mut res, value_accessors)?;
- }
- };
-
- Ok(res)
- }
-}
-
-fn get_missing_val(
- column_type: ColumnType,
- missing: &Key,
- field_name: &str,
-) -> crate::Result<Option<u64>> {
- let missing_val = match missing {
- Key::Str(_) if column_type == ColumnType::Str => Some(u64::MAX),
- // Allow fallback to number on text fields
- Key::F64(_) if column_type == ColumnType::Str => Some(u64::MAX),
- Key::F64(val) if column_type.numerical_type().is_some() => {
- f64_to_fastfield_u64(*val, &column_type)
- }
- _ => {
- return Err(crate::TantivyError::InvalidArgument(format!(
- "Missing value {missing:?} for field {field_name} is not supported for column \
- type {column_type:?}"
- )));
- }
- };
- Ok(missing_val)
-}
-
-fn get_numeric_or_date_column_types() -> &'static [ColumnType] {
- &[
- ColumnType::F64,
- ColumnType::U64,
- ColumnType::I64,
- ColumnType::DateTime,
- ]
-}
-
-pub(crate) fn get_aggs_with_segment_accessor_and_validate(
- aggs: &Aggregations,
- reader: &SegmentReader,
- segment_ordinal: SegmentOrdinal,
- limits: &AggregationLimits,
-) -> crate::Result<AggregationsWithAccessor> {
- let mut aggss = Vec::new();
- for (key, agg) in aggs.iter() {
- let aggs = AggregationWithAccessor::try_from_agg(
- agg,
- agg.sub_aggregation(),
- reader,
- segment_ordinal,
- limits.clone(),
- )?;
- for agg in aggs {
- aggss.push((key.to_string(), agg));
- }
- }
- Ok(AggregationsWithAccessor::from_data(
- VecWithNames::from_entries(aggss),
- ))
-}
-
-/// Get fast field reader or empty as default.
-fn get_ff_reader(
- reader: &SegmentReader,
- field_name: &str,
- allowed_column_types: Option<&[ColumnType]>,
-) -> crate::Result<(crate::columnar::Column<u64>, ColumnType)> {
- let ff_fields = reader.fast_fields();
- let ff_field_with_type = ff_fields
- .u64_lenient_for_type(allowed_column_types, field_name)?
- .unwrap_or_else(|| {
- (
- Column::build_empty_column(reader.num_docs()),
- ColumnType::U64,
- )
- });
- Ok(ff_field_with_type)
-}
-
-fn get_dynamic_columns(
- reader: &SegmentReader,
- field_name: &str,
-) -> crate::Result<Vec<crate::columnar::DynamicColumn>> {
- let ff_fields = reader.fast_fields().dynamic_column_handles(field_name)?;
- let cols = ff_fields
- .iter()
- .map(|h| h.open())
- .collect::<io::Result<_>>()?;
- assert!(!ff_fields.is_empty(), "field {field_name} not found");
- Ok(cols)
-}
-
-/// Get all fast field reader or empty as default.
-///
-/// Is guaranteed to return at least one column.
-fn get_all_ff_reader_or_empty(
- reader: &SegmentReader,
- field_name: &str,
- allowed_column_types: Option<&[ColumnType]>,
- fallback_type: ColumnType,
-) -> crate::Result<Vec<(crate::columnar::Column<u64>, ColumnType)>> {
- let ff_fields = reader.fast_fields();
- let mut ff_field_with_type =
- ff_fields.u64_lenient_for_type_all(allowed_column_types, field_name)?;
- if ff_field_with_type.is_empty() {
- ff_field_with_type.push((Column::build_empty_column(reader.num_docs()), fallback_type));
- }
- Ok(ff_field_with_type)
-}
diff --git a/crates/tantivy/src/aggregation/agg_result.rs b/crates/tantivy/src/aggregation/agg_result.rs
deleted file mode 100644
index aac1efac..00000000
--- a/crates/tantivy/src/aggregation/agg_result.rs
+++ /dev/null
@@ -1,306 +0,0 @@
-//! Contains the final aggregation tree.
-//! This tree can be converted via the `into()` method from `IntermediateAggregationResults`.
-//! This conversion computes the final result. For example: The intermediate result contains
-//! intermediate average results, which is the sum and the number of values. The actual average is
-//! calculated on the step from intermediate to final aggregation result tree.
-
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-
-use super::bucket::GetDocCount;
-use super::metric::{
- ExtendedStats, PercentilesMetricResult, SingleMetricResult, Stats, TopHitsMetricResult,
-};
-use super::{AggregationError, Key};
-use crate::TantivyError;
-
-#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)]
-/// The final aggegation result.
-pub struct AggregationResults(pub FxHashMap<String, AggregationResult>);
-
-impl AggregationResults {
- pub(crate) fn get_bucket_count(&self) -> u64 {
- self.0
- .values()
- .map(|agg| agg.get_bucket_count())
- .sum::<u64>()
- }
-
- pub(crate) fn get_value_from_aggregation(
- &self,
- name: &str,
- agg_property: &str,
- ) -> crate::Result<Option<f64>> {
- if let Some(agg) = self.0.get(name) {
- agg.get_value_from_aggregation(name, agg_property)
- } else {
- // Validation is be done during request parsing, so we can't reach this state.
- Err(TantivyError::InternalError(format!(
- "Can't find aggregation {name:?} in sub-aggregations"
- )))
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(untagged)]
-/// An aggregation is either a bucket or a metric.
-pub enum AggregationResult {
- /// Bucket result variant.
- BucketResult(BucketResult),
- /// Metric result variant.
- MetricResult(MetricResult),
-}
-
-impl AggregationResult {
- pub(crate) fn get_bucket_count(&self) -> u64 {
- match self {
- AggregationResult::BucketResult(bucket) => bucket.get_bucket_count(),
- AggregationResult::MetricResult(_) => 0,
- }
- }
-
- pub(crate) fn get_value_from_aggregation(
- &self,
- _name: &str,
- agg_property: &str,
- ) -> crate::Result<Option<f64>> {
- match self {
- AggregationResult::BucketResult(_bucket) => Err(TantivyError::InternalError(
- "Tried to retrieve value from bucket aggregation. This is not supported and \
- should not happen during collection phase, but should be caught during validation"
- .to_string(),
- )),
- AggregationResult::MetricResult(metric) => metric.get_value(agg_property),
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(untagged)]
-/// MetricResult
-pub enum MetricResult {
- /// Average metric result.
- Average(SingleMetricResult),
- /// Count metric result.
- Count(SingleMetricResult),
- /// Max metric result.
- Max(SingleMetricResult),
- /// Min metric result.
- Min(SingleMetricResult),
- /// Stats metric result.
- Stats(Stats),
- /// ExtendedStats metric result.
- ExtendedStats(Box<ExtendedStats>),
- /// Sum metric result.
- Sum(SingleMetricResult),
- /// Percentiles metric result.
- Percentiles(PercentilesMetricResult),
- /// Top hits metric result
- TopHits(TopHitsMetricResult),
-}
-
-impl MetricResult {
- fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {
- match self {
- MetricResult::Average(avg) => Ok(avg.value),
- MetricResult::Count(count) => Ok(count.value),
- MetricResult::Max(max) => Ok(max.value),
- MetricResult::Min(min) => Ok(min.value),
- MetricResult::Stats(stats) => stats.get_value(agg_property),
- MetricResult::ExtendedStats(extended_stats) => extended_stats.get_value(agg_property),
- MetricResult::Sum(sum) => Ok(sum.value),
- MetricResult::Percentiles(_) => Err(TantivyError::AggregationError(
- AggregationError::InvalidRequest("percentiles can't be used to order".to_string()),
- )),
- MetricResult::TopHits(_) => Err(TantivyError::AggregationError(
- AggregationError::InvalidRequest("top_hits can't be used to order".to_string()),
- )),
- }
- }
-}
-
-/// BucketEntry holds bucket aggregation result types.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum BucketResult {
- /// This is the range entry for a bucket, which contains a key, count, from, to, and optionally
- /// sub-aggregations.
- Range {
- /// The range buckets sorted by range.
- buckets: BucketEntries<RangeBucketEntry>,
- },
- /// This is the histogram entry for a bucket, which contains a key, count, and optionally
- /// sub-aggregations.
- Histogram {
- /// The buckets.
- ///
- /// If there are holes depends on the request, if min_doc_count is 0, then there are no
- /// holes between the first and last bucket.
- /// See [`HistogramAggregation`](super::bucket::HistogramAggregation)
- buckets: BucketEntries<BucketEntry>,
- },
- /// This is the term result
- Terms {
- /// The buckets.
- ///
- /// See [`TermsAggregation`](super::bucket::TermsAggregation)
- buckets: Vec<BucketEntry>,
- /// The number of documents that didn’t make it into to TOP N due to shard_size or size
- sum_other_doc_count: u64,
- #[serde(skip_serializing_if = "Option::is_none")]
- /// The upper bound error for the doc count of each term.
- doc_count_error_upper_bound: Option<u64>,
- },
-}
-
-impl BucketResult {
- pub(crate) fn get_bucket_count(&self) -> u64 {
- match self {
- BucketResult::Range { buckets } => {
- buckets.iter().map(|bucket| bucket.get_bucket_count()).sum()
- }
- BucketResult::Histogram { buckets } => {
- buckets.iter().map(|bucket| bucket.get_bucket_count()).sum()
- }
- BucketResult::Terms {
- buckets,
- sum_other_doc_count: _,
- doc_count_error_upper_bound: _,
- } => buckets.iter().map(|bucket| bucket.get_bucket_count()).sum(),
- }
- }
-}
-
-/// This is the wrapper of buckets entries, which can be vector or hashmap
-/// depending on if it's keyed or not.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum BucketEntries<T> {
- /// Vector format bucket entries
- Vec(Vec<T>),
- /// HashMap format bucket entries
- HashMap(FxHashMap<String, T>),
-}
-
-impl<T> BucketEntries<T> {
- fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = &T> + 'a> {
- match self {
- BucketEntries::Vec(vec) => Box::new(vec.iter()),
- BucketEntries::HashMap(map) => Box::new(map.values()),
- }
- }
-}
-
-/// This is the default entry for a bucket, which contains a key, count, and optionally
-/// sub-aggregations.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// ...
-/// "my_histogram": {
-/// "buckets": [
-/// {
-/// "key": "2.0",
-/// "doc_count": 5
-/// },
-/// {
-/// "key": "4.0",
-/// "doc_count": 2
-/// },
-/// {
-/// "key": "6.0",
-/// "doc_count": 3
-/// }
-/// ]
-/// }
-/// ...
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct BucketEntry {
- #[serde(skip_serializing_if = "Option::is_none")]
- /// The string representation of the bucket.
- pub key_as_string: Option<String>,
- /// The identifier of the bucket.
- pub key: Key,
- /// Number of documents in the bucket.
- pub doc_count: u64,
- #[serde(flatten)]
- /// Sub-aggregations in this bucket.
- pub sub_aggregation: AggregationResults,
-}
-impl BucketEntry {
- pub(crate) fn get_bucket_count(&self) -> u64 {
- 1 + self.sub_aggregation.get_bucket_count()
- }
-}
-impl GetDocCount for &BucketEntry {
- fn doc_count(&self) -> u64 {
- self.doc_count
- }
-}
-impl GetDocCount for BucketEntry {
- fn doc_count(&self) -> u64 {
- self.doc_count
- }
-}
-
-/// This is the range entry for a bucket, which contains a key, count, and optionally
-/// sub-aggregations.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// ...
-/// "my_ranges": {
-/// "buckets": [
-/// {
-/// "key": "*-10",
-/// "to": 10,
-/// "doc_count": 5
-/// },
-/// {
-/// "key": "10-20",
-/// "from": 10,
-/// "to": 20,
-/// "doc_count": 2
-/// },
-/// {
-/// "key": "20-*",
-/// "from": 20,
-/// "doc_count": 3
-/// }
-/// ]
-/// }
-/// ...
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct RangeBucketEntry {
- /// The identifier of the bucket.
- pub key: Key,
- /// Number of documents in the bucket.
- pub doc_count: u64,
- #[serde(flatten)]
- /// Sub-aggregations in this bucket.
- pub sub_aggregation: AggregationResults,
- /// The from range of the bucket. Equals `f64::MIN` when `None`.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub from: Option<f64>,
- /// The to range of the bucket. Equals `f64::MAX` when `None`.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub to: Option<f64>,
- /// The optional string representation for the `from` range.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub from_as_string: Option<String>,
- /// The optional string representation for the `to` range.
- #[serde(skip_serializing_if = "Option::is_none")]
- pub to_as_string: Option<String>,
-}
-impl RangeBucketEntry {
- pub(crate) fn get_bucket_count(&self) -> u64 {
- 1 + self.sub_aggregation.get_bucket_count()
- }
-}
diff --git a/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rs b/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rs
deleted file mode 100644
index e1ca3426..00000000
--- a/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rs
+++ /dev/null
@@ -1,675 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use super::{HistogramAggregation, HistogramBounds};
-use crate::aggregation::*;
-
-/// DateHistogramAggregation is similar to `HistogramAggregation`, but it can only be used with date
-/// type.
-///
-/// Currently only **fixed time** intervals are supported. Calendar-aware time intervals are not
-/// supported.
-///
-/// Like the histogram, values are rounded down into the closest bucket.
-///
-/// For this calculation all fastfield values are converted to f64.
-///
-/// # Limitations/Compatibility
-/// Only fixed time intervals are supported.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "prices": {
-/// "date_histogram": {
-/// "field": "price",
-/// "fixed_interval": "30d"
-/// }
-/// }
-/// }
-/// ```
-///
-/// Response
-/// See [`BucketEntry`](crate::aggregation::agg_result::BucketEntry)
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct DateHistogramAggregationReq {
- #[doc(hidden)]
- /// Only for validation
- interval: Option<String>,
- #[doc(hidden)]
- /// Only for validation
- calendar_interval: Option<String>,
- /// The field to aggregate on.
- pub field: String,
- /// The format to format dates. Unsupported currently.
- pub format: Option<String>,
- /// The interval to chunk your data range. Each bucket spans a value range of
- /// [0..fixed_interval). Accepted values
- ///
- /// Fixed intervals are configured with the `fixed_interval` parameter.
- /// In contrast to calendar-aware intervals, fixed intervals are a fixed number of SI units and
- /// never deviate, regardless of where they fall on the calendar. One second is always
- /// composed of 1000ms. This allows fixed intervals to be specified in any multiple of the
- /// supported units. However, it means fixed intervals cannot express other units such as
- /// months, since the duration of a month is not a fixed quantity. Attempting to specify a
- /// calendar interval like month or quarter will return an Error.
- ///
- /// The accepted units for fixed intervals are:
- /// * `ms`: milliseconds
- /// * `s`: seconds. Defined as 1000 milliseconds each.
- /// * `m`: minutes. Defined as 60 seconds each (60_000 milliseconds).
- /// * `h`: hours. Defined as 60 minutes each (3_600_000 milliseconds).
- /// * `d`: days. Defined as 24 hours (86_400_000 milliseconds).
- ///
- /// Fractional time values are not supported, but you can address this by shifting to another
- /// time unit (e.g., `1.5h` could instead be specified as `90m`).
- ///
- /// `Option` for validation, the parameter is not optional
- pub fixed_interval: Option<String>,
- /// Intervals implicitly defines an absolute grid of buckets `[interval * k, interval * (k +
- /// 1))`.
- ///
- /// Offset makes it possible to shift this grid into
- /// `[offset + interval * k, offset + interval * (k + 1))`. Offset has to be in the range [0,
- /// interval).
- ///
- /// The `offset` parameter is has the same syntax as the `fixed_interval` parameter, but
- /// also allows for negative values.
- pub offset: Option<String>,
- /// The minimum number of documents in a bucket to be returned. Defaults to 0.
- pub min_doc_count: Option<u64>,
- /// Limits the data range to `[min, max]` closed interval.
- ///
- /// This can be used to filter values if they are not in the data range.
- ///
- /// hard_bounds only limits the buckets, to force a range set both extended_bounds and
- /// hard_bounds to the same range.
- ///
- /// Needs to be provided as timestamp in millisecond precision.
- ///
- /// ## Example
- /// ```json
- /// {
- /// "sales_over_time": {
- /// "date_histogram": {
- /// "field": "dates",
- /// "interval": "1d",
- /// "hard_bounds": {
- /// "min": 0,
- /// "max": 1420502400000
- /// }
- /// }
- /// }
- /// }
- /// ```
- pub hard_bounds: Option<HistogramBounds>,
- /// Can be set to extend your bounds. The range of the buckets is by default defined by the
- /// data range of the values of the documents. As the name suggests, this can only be used to
- /// extend the value range. If the bounds for min or max are not extending the range, the value
- /// has no effect on the returned buckets.
- ///
- /// Cannot be set in conjunction with min_doc_count > 0, since the empty buckets from extended
- /// bounds would not be returned.
- pub extended_bounds: Option<HistogramBounds>,
-
- /// Whether to return the buckets as a hash map
- #[serde(default)]
- pub keyed: bool,
-}
-
-impl DateHistogramAggregationReq {
- pub(crate) fn to_histogram_req(&self) -> crate::Result<HistogramAggregation> {
- self.validate()?;
- Ok(HistogramAggregation {
- field: self.field.to_string(),
- interval: parse_into_milliseconds(self.fixed_interval.as_ref().unwrap())? as f64,
- offset: self
- .offset
- .as_ref()
- .map(|offset| parse_offset_into_milliseconds(offset))
- .transpose()?
- .map(|el| el as f64),
- min_doc_count: self.min_doc_count,
- hard_bounds: self.hard_bounds,
- extended_bounds: self.extended_bounds,
- keyed: self.keyed,
- is_normalized_to_ns: false,
- })
- }
-
- fn validate(&self) -> crate::Result<()> {
- if let Some(interval) = self.interval.as_ref() {
- return Err(crate::TantivyError::InvalidArgument(format!(
- "`interval` parameter {interval:?} in date histogram is unsupported, only \
- `fixed_interval` is supported"
- )));
- }
- if let Some(interval) = self.calendar_interval.as_ref() {
- return Err(crate::TantivyError::InvalidArgument(format!(
- "`calendar_interval` parameter {interval:?} in date histogram is unsupported, \
- only `fixed_interval` is supported"
- )));
- }
- if self.format.is_some() {
- return Err(crate::TantivyError::InvalidArgument(
- "format parameter on date_histogram is unsupported".to_string(),
- ));
- }
-
- if self.fixed_interval.is_none() {
- return Err(crate::TantivyError::InvalidArgument(
- "fixed_interval in date histogram is missing".to_string(),
- ));
- }
-
- parse_into_milliseconds(self.fixed_interval.as_ref().unwrap())?;
-
- Ok(())
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Error)]
-/// Errors when parsing the fixed interval for `DateHistogramAggregationReq`.
-pub enum DateHistogramParseError {
- /// Unit not recognized in passed String
- #[error("Unit not recognized in passed String {0:?}")]
- UnitNotRecognized(String),
- /// Number not found in passed String
- #[error("Number not found in passed String {0:?}")]
- NumberMissing(String),
- /// Unit not found in passed String
- #[error("Unit not found in passed String {0:?}")]
- UnitMissing(String),
- /// Offset invalid
- #[error("passed offset is invalid {0:?}")]
- InvalidOffset(String),
- /// Value out of bounds
- #[error("passed value is out of bounds: {0:?}")]
- OutOfBounds(String),
-}
-
-fn parse_offset_into_milliseconds(input: &str) -> Result<i64, AggregationError> {
- let is_sign = |byte| &[byte] == b"-" || &[byte] == b"+";
- if input.is_empty() {
- return Err(DateHistogramParseError::InvalidOffset(input.to_string()).into());
- }
-
- let has_sign = is_sign(input.as_bytes()[0]);
- if has_sign {
- let (sign, input) = input.split_at(1);
- let val = parse_into_milliseconds(input)?;
- if sign == "-" {
- Ok(-val)
- } else {
- Ok(val)
- }
- } else {
- parse_into_milliseconds(input)
- }
-}
-
-fn parse_into_milliseconds(input: &str) -> Result<i64, AggregationError> {
- let split_boundary = input
- .as_bytes()
- .iter()
- .take_while(|byte| byte.is_ascii_digit())
- .count();
- let (number, unit) = input.split_at(split_boundary);
- if number.is_empty() {
- return Err(DateHistogramParseError::NumberMissing(input.to_string()).into());
- }
- if unit.is_empty() {
- return Err(DateHistogramParseError::UnitMissing(input.to_string()).into());
- }
- let number: i64 = number
- .parse()
- // Technically this should never happen, but there was a bug
- // here and being defensive does not hurt.
- .map_err(|_err| DateHistogramParseError::NumberMissing(input.to_string()))?;
-
- let unit_in_ms = match unit {
- "ms" | "milliseconds" => 1,
- "s" | "seconds" => 1000,
- "m" | "minutes" => 60 * 1000,
- "h" | "hours" => 60 * 60 * 1000,
- "d" | "days" => 24 * 60 * 60 * 1000,
- _ => return Err(DateHistogramParseError::UnitNotRecognized(unit.to_string()).into()),
- };
-
- let val = number * unit_in_ms;
- // The field type is in nanoseconds precision, so validate the value to fit the range
- val.checked_mul(1_000_000)
- .ok_or_else(|| DateHistogramParseError::OutOfBounds(input.to_string()))?;
-
- Ok(val)
-}
-
-#[cfg(test)]
-pub mod tests {
- use pretty_assertions::assert_eq;
-
- use super::*;
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::tests::exec_request;
- use crate::indexer::NoMergePolicy;
- use crate::schema::{Schema, FAST, STRING};
- use crate::{Index, IndexWriter, TantivyDocument};
-
- #[test]
- fn test_parse_into_millisecs() {
- assert_eq!(parse_into_milliseconds("1m").unwrap(), 60_000);
- assert_eq!(parse_into_milliseconds("2m").unwrap(), 120_000);
- assert_eq!(parse_into_milliseconds("2minutes").unwrap(), 120_000);
- assert_eq!(
- parse_into_milliseconds("2y").unwrap_err(),
- DateHistogramParseError::UnitNotRecognized("y".to_string()).into()
- );
- assert_eq!(
- parse_into_milliseconds("2000").unwrap_err(),
- DateHistogramParseError::UnitMissing("2000".to_string()).into()
- );
- assert_eq!(
- parse_into_milliseconds("ms").unwrap_err(),
- DateHistogramParseError::NumberMissing("ms".to_string()).into()
- );
- }
-
- #[test]
- fn test_parse_offset_into_milliseconds() {
- assert_eq!(parse_offset_into_milliseconds("1m").unwrap(), 60_000);
- assert_eq!(parse_offset_into_milliseconds("+1m").unwrap(), 60_000);
- assert_eq!(parse_offset_into_milliseconds("-1m").unwrap(), -60_000);
- assert_eq!(parse_offset_into_milliseconds("2m").unwrap(), 120_000);
- assert_eq!(parse_offset_into_milliseconds("+2m").unwrap(), 120_000);
- assert_eq!(parse_offset_into_milliseconds("-2m").unwrap(), -120_000);
- assert_eq!(parse_offset_into_milliseconds("-2ms").unwrap(), -2);
- assert_eq!(
- parse_offset_into_milliseconds("2y").unwrap_err(),
- DateHistogramParseError::UnitNotRecognized("y".to_string()).into()
- );
- assert_eq!(
- parse_offset_into_milliseconds("2000").unwrap_err(),
- DateHistogramParseError::UnitMissing("2000".to_string()).into()
- );
- assert_eq!(
- parse_offset_into_milliseconds("ms").unwrap_err(),
- DateHistogramParseError::NumberMissing("ms".to_string()).into()
- );
- }
-
- #[test]
- fn test_parse_into_milliseconds_do_not_accept_non_ascii() {
- assert!(parse_into_milliseconds("1m").is_err());
- }
-
- pub fn get_test_index_from_docs(
- merge_segments: bool,
- segment_and_docs: &[Vec<&str>],
- ) -> crate::Result<Index> {
- let mut schema_builder = Schema::builder();
- schema_builder.add_date_field("date", FAST);
- schema_builder.add_json_field("mixed", FAST);
- schema_builder.add_text_field("text", FAST | STRING);
- schema_builder.add_text_field("text2", FAST | STRING);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema.clone());
- {
- let mut index_writer = index.writer_with_num_threads(1, 30_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- for values in segment_and_docs {
- for doc_str in values {
- let doc = TantivyDocument::parse_json(&schema, doc_str)?;
- index_writer.add_document(doc)?;
- }
- // writing the segment
- index_writer.commit()?;
- }
- }
- if merge_segments {
- let segment_ids = index
- .searchable_segment_ids()
- .expect("Searchable segments failed.");
- if segment_ids.len() > 1 {
- let mut index_writer: IndexWriter = index.writer_for_tests()?;
- index_writer.merge(&segment_ids).wait()?;
- index_writer.wait_merging_threads()?;
- }
- }
-
- Ok(index)
- }
-
- #[test]
- fn histogram_test_date_force_merge_segments() {
- histogram_test_date_merge_segments(true)
- }
-
- #[test]
- fn histogram_test_date() {
- histogram_test_date_merge_segments(false)
- }
-
- fn histogram_test_date_merge_segments(merge_segments: bool) {
- let docs = vec![
- vec![r#"{ "date": "2015-01-01T12:10:30Z", "text": "aaa" }"#],
- vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],
- vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],
- vec![r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb" }"#],
- vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],
- vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],
- ];
- let index = get_test_index_from_docs(merge_segments, &docs).unwrap();
-
- {
- // 30day + offset
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "30d",
- "offset": "-4d"
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!({
- "sales_over_time" : {
- "buckets" : [
- {
- "key_as_string" : "2015-01-01T00:00:00Z",
- "key" : 1420070400000.0,
- "doc_count" : 6
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
-
- {
- // 30day + offset + sub_agg
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "30d",
- "offset": "-4d"
- },
- "aggs": {
- "texts": {
- "terms": {"field": "text"}
- }
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!({
- "sales_over_time" : {
- "buckets" : [
- {
- "key_as_string" : "2015-01-01T00:00:00Z",
- "key" : 1420070400000.0,
- "doc_count" : 6,
- "texts": {
- "buckets": [
- {
- "doc_count": 3,
- "key": "bbb"
- },
- {
- "doc_count": 2,
- "key": "ccc"
- },
- {
- "doc_count": 1,
- "key": "aaa"
- }
- ],
- "doc_count_error_upper_bound": 0,
- "sum_other_doc_count": 0
- }
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
- {
- // 1day
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "1d"
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!( {
- "sales_over_time": {
- "buckets": [
- {
- "doc_count": 3,
- "key": 1420070400000.0,
- "key_as_string": "2015-01-01T00:00:00Z"
- },
- {
- "doc_count": 1,
- "key": 1420156800000.0,
- "key_as_string": "2015-01-02T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420243200000.0,
- "key_as_string": "2015-01-03T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420329600000.0,
- "key_as_string": "2015-01-04T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420416000000.0,
- "key_as_string": "2015-01-05T00:00:00Z"
- },
- {
- "doc_count": 2,
- "key": 1420502400000.0,
- "key_as_string": "2015-01-06T00:00:00Z"
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
-
- {
- // 1day + extended_bounds
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "1d",
- "extended_bounds": {
- "min": 1419984000000.0,
- "max": 1420588800000.0
- }
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!({
- "sales_over_time" : {
- "buckets": [
- {
- "doc_count": 0,
- "key": 1419984000000.0,
- "key_as_string": "2014-12-31T00:00:00Z"
- },
- {
- "doc_count": 3,
- "key": 1420070400000.0,
- "key_as_string": "2015-01-01T00:00:00Z"
- },
- {
- "doc_count": 1,
- "key": 1420156800000.0,
- "key_as_string": "2015-01-02T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420243200000.0,
- "key_as_string": "2015-01-03T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420329600000.0,
- "key_as_string": "2015-01-04T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420416000000.0,
- "key_as_string": "2015-01-05T00:00:00Z"
- },
- {
- "doc_count": 2,
- "key": 1420502400000.0,
- "key_as_string": "2015-01-06T00:00:00Z"
- },
- {
- "doc_count": 0,
- "key": 1420588800000.0,
- "key_as_string": "2015-01-07T00:00:00Z"
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
- {
- // 1day + hard_bounds + extended_bounds
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "1d",
- "hard_bounds": {
- "min": 1420156800000.0,
- "max": 1420243200000.0
- }
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!({
- "sales_over_time" : {
- "buckets": [
- {
- "doc_count": 1,
- "key": 1420156800000.0,
- "key_as_string": "2015-01-02T00:00:00Z"
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
-
- {
- // 1day + hard_bounds as Rfc3339
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "fixed_interval": "1d",
- "hard_bounds": {
- "min": "2015-01-02T00:00:00Z",
- "max": "2015-01-02T12:00:00Z"
- }
- }
- }
- }
- );
-
- let agg_req: Aggregations = serde_json::from_str(
- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),
- )
- .unwrap();
- let res = exec_request(agg_req, &index).unwrap();
- let expected_res = json!({
- "sales_over_time" : {
- "buckets": [
- {
- "doc_count": 1,
- "key": 1420156800000.0,
- "key_as_string": "2015-01-02T00:00:00Z"
- }
- ]
- }
- });
- assert_eq!(res, expected_res);
- }
- }
- #[test]
- fn histogram_test_invalid_req() {
- let docs = vec![];
-
- let index = get_test_index_from_docs(false, &docs).unwrap();
- let elasticsearch_compatible_json = json!(
- {
- "sales_over_time": {
- "date_histogram": {
- "field": "date",
- "interval": "30d",
- "offset": "-4d"
- }
- }
- }
- );
-
- let agg_req: Aggregations =
- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())
- .unwrap();
- let err = exec_request(agg_req, &index).unwrap_err();
- assert_eq!(
- err.to_string(),
- r#"An invalid argument was passed: '`interval` parameter "30d" in date histogram is unsupported, only `fixed_interval` is supported'"#
- );
- }
-}
diff --git a/crates/tantivy/src/aggregation/bucket/histogram/histogram.rs b/crates/tantivy/src/aggregation/bucket/histogram/histogram.rs
deleted file mode 100644
index cce91c14..00000000
--- a/crates/tantivy/src/aggregation/bucket/histogram/histogram.rs
+++ /dev/null
@@ -1,595 +0,0 @@
-use std::cmp::Ordering;
-
-use crate::bitpacker::minmax;
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-
-use crate::aggregation::agg_limits::MemoryConsumption;
-use crate::aggregation::agg_req::Aggregations;
-use crate::aggregation::agg_req_with_accessor::{
- AggregationWithAccessor, AggregationsWithAccessor,
-};
-use crate::aggregation::agg_result::BucketEntry;
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
- IntermediateHistogramBucketEntry,
-};
-use crate::aggregation::segment_agg_result::{
- build_segment_agg_collector, SegmentAggregationCollector,
-};
-use crate::aggregation::*;
-use crate::TantivyError;
-
-/// Histogram is a bucket aggregation, where buckets are created dynamically for given `interval`.
-/// Each document value is rounded down to its bucket.
-///
-/// E.g. if we have a price 18 and an interval of 5, the document will fall into the bucket with
-/// the key 15. The formula used for this is:
-/// `((val - offset) / interval).floor() * interval + offset`
-///
-/// For this calculation all fastfield values are converted to f64.
-///
-/// # Returned Buckets
-/// By default buckets are returned between the min and max value of the documents, including empty
-/// buckets.
-/// Setting min_doc_count to != 0 will filter empty buckets.
-///
-/// The value range of the buckets can bet extended via
-/// [extended_bounds](HistogramAggregation::extended_bounds) or limit the range via
-/// [hard_bounds](HistogramAggregation::hard_bounds).
-///
-/// # Result
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`BucketEntry`](crate::aggregation::agg_result::BucketEntry) on the
-/// `AggregationCollector`.
-///
-/// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateHistogramBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateHistogramBucketEntry) on the
-/// `DistributedAggregationCollector`.
-///
-/// # Limitations/Compatibility
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "prices": {
-/// "histogram": {
-/// "field": "price",
-/// "interval": 10
-/// }
-/// }
-/// }
-/// ```
-///
-/// Response
-/// See [`BucketEntry`](crate::aggregation::agg_result::BucketEntry)
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct HistogramAggregation {
- /// The field to aggregate on.
- pub field: String,
- /// The interval to chunk your data range. Each bucket spans a value range of [0..interval).
- /// Must be a positive value.
- #[serde(deserialize_with = "deserialize_f64")]
- pub interval: f64,
- /// Intervals implicitly defines an absolute grid of buckets `[interval * k, interval * (k +
- /// 1))`.
- ///
- /// Offset makes it possible to shift this grid into
- /// `[offset + interval * k, offset + interval * (k + 1))`. Offset has to be in the range [0,
- /// interval).
- ///
- /// As an example, if there are two documents with value 9 and 12 and interval 10.0, they would
- /// fall into the buckets with the key 0 and 10.
- /// With offset 5 and interval 10, they would both fall into the bucket with they key 5 and the
- /// range [5..15)
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub offset: Option<f64>,
- /// The minimum number of documents in a bucket to be returned. Defaults to 0.
- pub min_doc_count: Option<u64>,
- /// Limits the data range to `[min, max]` closed interval.
- ///
- /// This can be used to filter values if they are not in the data range.
- ///
- /// hard_bounds only limits the buckets, to force a range set both extended_bounds and
- /// hard_bounds to the same range.
- ///
- /// ## Example
- /// ```json
- /// {
- /// "prices": {
- /// "histogram": {
- /// "field": "price",
- /// "interval": 10,
- /// "hard_bounds": {
- /// "min": 0,
- /// "max": 100
- /// }
- /// }
- /// }
- /// }
- /// ```
- pub hard_bounds: Option<HistogramBounds>,
- /// Can be set to extend your bounds. The range of the buckets is by default defined by the
- /// data range of the values of the documents. As the name suggests, this can only be used to
- /// extend the value range. If the bounds for min or max are not extending the range, the value
- /// has no effect on the returned buckets.
- ///
- /// Cannot be set in conjunction with min_doc_count > 0, since the empty buckets from extended
- /// bounds would not be returned.
- pub extended_bounds: Option<HistogramBounds>,
- /// Whether to return the buckets as a hash map
- #[serde(default)]
- pub keyed: bool,
- /// Whether the values are normalized to ns for date time values. Defaults to false.
- #[serde(default)]
- pub is_normalized_to_ns: bool,
-}
-
-impl HistogramAggregation {
- pub(crate) fn normalize_date_time(&mut self) {
- if !self.is_normalized_to_ns {
- // values are provided in ms, but the fastfield is in nano seconds
- self.interval *= 1_000_000.0;
- self.offset = self.offset.map(|off| off * 1_000_000.0);
- self.hard_bounds = self.hard_bounds.map(|bounds| HistogramBounds {
- min: bounds.min * 1_000_000.0,
- max: bounds.max * 1_000_000.0,
- });
- self.extended_bounds = self.extended_bounds.map(|bounds| HistogramBounds {
- min: bounds.min * 1_000_000.0,
- max: bounds.max * 1_000_000.0,
- });
- self.is_normalized_to_ns = true;
- }
- }
-
- fn validate(&self) -> crate::Result<()> {
- if self.interval <= 0.0f64 {
- return Err(TantivyError::InvalidArgument(
- "interval must be a positive value".to_string(),
- ));
- }
-
- if self.min_doc_count.unwrap_or(0) > 0 && self.extended_bounds.is_some() {
- return Err(TantivyError::InvalidArgument(
- "Cannot set min_doc_count and extended_bounds at the same time".to_string(),
- ));
- }
-
- if let (Some(hard_bounds), Some(extended_bounds)) = (self.hard_bounds, self.extended_bounds)
- {
- if extended_bounds.min < hard_bounds.min || extended_bounds.max > hard_bounds.max {
- return Err(TantivyError::InvalidArgument(format!(
- "extended_bounds have to be inside hard_bounds, extended_bounds: \
- {extended_bounds}, hard_bounds {hard_bounds}"
- )));
- }
- }
-
- Ok(())
- }
-
- /// Returns the minimum number of documents required for a bucket to be returned.
- pub fn min_doc_count(&self) -> u64 {
- self.min_doc_count.unwrap_or(0)
- }
-}
-
-/// Used to set extended or hard bounds on the histogram.
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
-pub struct HistogramBounds {
- /// The lower bounds.
- #[serde(deserialize_with = "deserialize_date_or_num")]
- pub min: f64,
- /// The upper bounds.
- #[serde(deserialize_with = "deserialize_date_or_num")]
- pub max: f64,
-}
-
-fn deserialize_date_or_num<'de, D>(deserializer: D) -> Result<f64, D::Error>
-where
- D: serde::Deserializer<'de>,
-{
- let value: serde_json::Value = Deserialize::deserialize(deserializer)?;
-
- // Check if the value is a string representing an Rfc3339 formatted date
- if let serde_json::Value::String(date_str) = value {
- // Parse the Rfc3339 formatted date string into a DateTime<Utc>
- let date =
- time::OffsetDateTime::parse(&date_str, &time::format_description::well_known::Rfc3339)
- .map_err(|_| serde::de::Error::custom("Invalid Rfc3339 formatted date"))?;
-
- let milliseconds: i64 = (date.unix_timestamp_nanos() / 1_000_000)
- .try_into()
- .map_err(|_| serde::de::Error::custom("{date_str} out of allowed range"))?;
-
- // Return the milliseconds as f64
- Ok(milliseconds as f64)
- } else {
- // The value is not a string, so assume it's a regular f64 number
- value
- .as_f64()
- .ok_or_else(|| serde::de::Error::custom("Invalid number format"))
- }
-}
-
-impl Display for HistogramBounds {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.write_fmt(format_args!("[{},{}]", self.min, self.max))
- }
-}
-
-impl HistogramBounds {
- fn contains(&self, val: f64) -> bool {
- val >= self.min && val <= self.max
- }
-}
-
-#[derive(Default, Clone, Debug, PartialEq)]
-pub(crate) struct SegmentHistogramBucketEntry {
- pub key: f64,
- pub doc_count: u64,
-}
-
-impl SegmentHistogramBucketEntry {
- pub(crate) fn into_intermediate_bucket_entry(
- self,
- sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,
- agg_with_accessor: &AggregationsWithAccessor,
- ) -> crate::Result<IntermediateHistogramBucketEntry> {
- let mut sub_aggregation_res = IntermediateAggregationResults::default();
- if let Some(sub_aggregation) = sub_aggregation {
- sub_aggregation
- .add_intermediate_aggregation_result(agg_with_accessor, &mut sub_aggregation_res)?;
- }
- Ok(IntermediateHistogramBucketEntry {
- key: self.key,
- doc_count: self.doc_count,
- sub_aggregation: sub_aggregation_res,
- })
- }
-}
-
-/// The collector puts values from the fast field into the correct buckets and does a conversion to
-/// the correct datatype.
-#[derive(Clone, Debug)]
-pub struct SegmentHistogramCollector {
- /// The buckets containing the aggregation data.
- buckets: FxHashMap<i64, SegmentHistogramBucketEntry>,
- sub_aggregations: FxHashMap<i64, Box<dyn SegmentAggregationCollector>>,
- sub_aggregation_blueprint: Option<Box<dyn SegmentAggregationCollector>>,
- column_type: ColumnType,
- interval: f64,
- offset: f64,
- bounds: HistogramBounds,
- accessor_idx: usize,
-}
-
-impl SegmentAggregationCollector for SegmentHistogramCollector {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];
-
- let bucket = self.into_intermediate_bucket_result(agg_with_accessor)?;
- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.collect_block(&[doc], agg_with_accessor)
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];
-
- let mem_pre = self.get_memory_consumption();
-
- let bounds = self.bounds;
- let interval = self.interval;
- let offset = self.offset;
- let get_bucket_pos = |val| (get_bucket_pos_f64(val, interval, offset) as i64);
-
- bucket_agg_accessor
- .column_block_accessor
- .fetch_block(docs, &bucket_agg_accessor.accessor);
-
- for (doc, val) in bucket_agg_accessor
- .column_block_accessor
- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)
- {
- let val = self.f64_from_fastfield_u64(val);
-
- let bucket_pos = get_bucket_pos(val);
-
- if bounds.contains(val) {
- let bucket = self.buckets.entry(bucket_pos).or_insert_with(|| {
- let key = get_bucket_key_from_pos(bucket_pos as f64, interval, offset);
- SegmentHistogramBucketEntry { key, doc_count: 0 }
- });
- bucket.doc_count += 1;
- if let Some(sub_aggregation_blueprint) = self.sub_aggregation_blueprint.as_mut() {
- self.sub_aggregations
- .entry(bucket_pos)
- .or_insert_with(|| sub_aggregation_blueprint.clone())
- .collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;
- }
- }
- }
-
- let mem_delta = self.get_memory_consumption() - mem_pre;
- if mem_delta > 0 {
- bucket_agg_accessor
- .limits
- .add_memory_consumed(mem_delta as u64)?;
- }
-
- Ok(())
- }
-
- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- let sub_aggregation_accessor =
- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;
-
- for sub_aggregation in self.sub_aggregations.values_mut() {
- sub_aggregation.flush(sub_aggregation_accessor)?;
- }
-
- Ok(())
- }
-}
-
-impl SegmentHistogramCollector {
- fn get_memory_consumption(&self) -> usize {
- let self_mem = std::mem::size_of::<Self>();
- let sub_aggs_mem = self.sub_aggregations.memory_consumption();
- let buckets_mem = self.buckets.memory_consumption();
- self_mem + sub_aggs_mem + buckets_mem
- }
- /// Converts the collector result into a intermediate bucket result.
- pub fn into_intermediate_bucket_result(
- self,
- agg_with_accessor: &AggregationWithAccessor,
- ) -> crate::Result<IntermediateBucketResult> {
- let mut buckets = Vec::with_capacity(self.buckets.len());
-
- for (bucket_pos, bucket) in self.buckets {
- let bucket_res = bucket.into_intermediate_bucket_entry(
- self.sub_aggregations.get(&bucket_pos).cloned(),
- &agg_with_accessor.sub_aggregation,
- );
-
- buckets.push(bucket_res?);
- }
- buckets.sort_unstable_by(|b1, b2| b1.key.total_cmp(&b2.key));
-
- Ok(IntermediateBucketResult::Histogram {
- buckets,
- is_date_agg: self.column_type == ColumnType::DateTime,
- })
- }
-
- pub(crate) fn from_req_and_validate(
- mut req: HistogramAggregation,
- sub_aggregation: &mut AggregationsWithAccessor,
- field_type: ColumnType,
- accessor_idx: usize,
- ) -> crate::Result<Self> {
- req.validate()?;
- if field_type == ColumnType::DateTime {
- req.normalize_date_time();
- }
-
- let sub_aggregation_blueprint = if sub_aggregation.is_empty() {
- None
- } else {
- let sub_aggregation = build_segment_agg_collector(sub_aggregation)?;
- Some(sub_aggregation)
- };
-
- let bounds = req.hard_bounds.unwrap_or(HistogramBounds {
- min: f64::MIN,
- max: f64::MAX,
- });
-
- Ok(Self {
- buckets: Default::default(),
- column_type: field_type,
- interval: req.interval,
- offset: req.offset.unwrap_or(0.0),
- bounds,
- sub_aggregations: Default::default(),
- sub_aggregation_blueprint,
- accessor_idx,
- })
- }
-
- #[inline]
- fn f64_from_fastfield_u64(&self, val: u64) -> f64 {
- f64_from_fastfield_u64(val, &self.column_type)
- }
-}
-
-#[inline]
-fn get_bucket_pos_f64(val: f64, interval: f64, offset: f64) -> f64 {
- ((val - offset) / interval).floor()
-}
-
-#[inline]
-fn get_bucket_key_from_pos(bucket_pos: f64, interval: f64, offset: f64) -> f64 {
- bucket_pos * interval + offset
-}
-
-// Convert to BucketEntry and fill gaps
-fn intermediate_buckets_to_final_buckets_fill_gaps(
- buckets: Vec<IntermediateHistogramBucketEntry>,
- histogram_req: &HistogramAggregation,
- sub_aggregation: &Aggregations,
- limits: &AggregationLimits,
-) -> crate::Result<Vec<BucketEntry>> {
- // Generate the full list of buckets without gaps.
- //
- // The bounds are the min max from the current buckets, optionally extended by
- // extended_bounds from the request
- let min_max = minmax(buckets.iter().map(|bucket| bucket.key));
-
- // memory check upfront
- let (_, first_bucket_num, last_bucket_num) =
- generate_bucket_pos_with_opt_minmax(histogram_req, min_max);
-
- // It's based on user input, so we need to account for overflows
- let added_buckets = ((last_bucket_num.saturating_sub(first_bucket_num)).max(0) as u64)
- .saturating_sub(buckets.len() as u64);
- limits.add_memory_consumed(
- added_buckets * std::mem::size_of::<IntermediateHistogramBucketEntry>() as u64,
- )?;
- // create buckets
- let fill_gaps_buckets = generate_buckets_with_opt_minmax(histogram_req, min_max);
-
- let empty_sub_aggregation = IntermediateAggregationResults::empty_from_req(sub_aggregation);
-
- // Use merge_join_by to fill in gaps, since buckets are sorted
-
- let final_buckets: Vec<BucketEntry> = buckets
- .into_iter()
- .merge_join_by(fill_gaps_buckets, |existing_bucket, fill_gaps_bucket| {
- existing_bucket
- .key
- .partial_cmp(fill_gaps_bucket)
- .unwrap_or(Ordering::Equal)
- })
- .map(|either| match either {
- // Ignore the generated bucket
- itertools::EitherOrBoth::Both(existing, _) => existing,
- itertools::EitherOrBoth::Left(existing) => existing,
- // Add missing bucket
- itertools::EitherOrBoth::Right(missing_bucket) => IntermediateHistogramBucketEntry {
- key: missing_bucket,
- doc_count: 0,
- sub_aggregation: empty_sub_aggregation.clone(),
- },
- })
- .map(|intermediate_bucket| {
- intermediate_bucket.into_final_bucket_entry(sub_aggregation, limits)
- })
- .collect::<crate::Result<Vec<_>>>()?;
-
- Ok(final_buckets)
-}
-
-// Convert to BucketEntry
-pub(crate) fn intermediate_histogram_buckets_to_final_buckets(
- buckets: Vec<IntermediateHistogramBucketEntry>,
- is_date_agg: bool,
- histogram_req: &HistogramAggregation,
- sub_aggregation: &Aggregations,
- limits: &AggregationLimits,
-) -> crate::Result<Vec<BucketEntry>> {
- // Normalization is column type dependent.
- // The request used in the the call to final is not yet be normalized.
- // Normalization is changing the precision from milliseconds to nanoseconds.
- let mut histogram_req = histogram_req.clone();
- if is_date_agg {
- histogram_req.normalize_date_time();
- }
- let mut buckets = if histogram_req.min_doc_count() == 0 {
- // With min_doc_count != 0, we may need to add buckets, so that there are no
- // gaps, since intermediate result does not contain empty buckets (filtered to
- // reduce serialization size).
- intermediate_buckets_to_final_buckets_fill_gaps(
- buckets,
- &histogram_req,
- sub_aggregation,
- limits,
- )?
- } else {
- buckets
- .into_iter()
- .filter(|histogram_bucket| histogram_bucket.doc_count >= histogram_req.min_doc_count())
- .map(|histogram_bucket| {
- histogram_bucket.into_final_bucket_entry(sub_aggregation, limits)
- })
- .collect::<crate::Result<Vec<_>>>()?
- };
-
- // If we have a date type on the histogram buckets, we add the `key_as_string` field as rfc339
- // and normalize from nanoseconds to milliseconds
- if is_date_agg {
- for bucket in buckets.iter_mut() {
- if let crate::aggregation::Key::F64(ref mut val) = bucket.key {
- let key_as_string = format_date(*val as i64)?;
- *val /= 1_000_000.0;
- bucket.key_as_string = Some(key_as_string);
- }
- }
- }
-
- Ok(buckets)
-}
-
-/// Applies req extended_bounds/hard_bounds on the min_max value
-///
-/// May return `(f64::MAX, f64::MIN)`, if there is no range.
-fn get_req_min_max(req: &HistogramAggregation, min_max: Option<(f64, f64)>) -> (f64, f64) {
- let (mut min, mut max) = min_max.unwrap_or((f64::MAX, f64::MIN));
-
- if let Some(extended_bounds) = &req.extended_bounds {
- min = min.min(extended_bounds.min);
- max = max.max(extended_bounds.max);
- }
-
- if let Some(hard_bounds) = &req.hard_bounds {
- min = min.max(hard_bounds.min);
- max = max.min(hard_bounds.max);
- }
-
- (min, max)
-}
-
-/// Generates buckets with req.interval
-/// Range is computed for provided min_max and request extended_bounds/hard_bounds
-/// returns empty vec when there is no range to span
-pub(crate) fn generate_bucket_pos_with_opt_minmax(
- req: &HistogramAggregation,
- min_max: Option<(f64, f64)>,
-) -> (f64, i64, i64) {
- let (min, max) = get_req_min_max(req, min_max);
-
- let offset = req.offset.unwrap_or(0.0);
- let first_bucket_num = get_bucket_pos_f64(min, req.interval, offset) as i64;
- let last_bucket_num = get_bucket_pos_f64(max, req.interval, offset) as i64;
- (offset, first_bucket_num, last_bucket_num)
-}
-
-/// Generates buckets with req.interval
-/// Range is computed for provided min_max and request extended_bounds/hard_bounds
-/// returns empty vec when there is no range to span
-pub(crate) fn generate_buckets_with_opt_minmax(
- req: &HistogramAggregation,
- min_max: Option<(f64, f64)>,
-) -> Vec<f64> {
- let (offset, first_bucket_num, last_bucket_num) =
- generate_bucket_pos_with_opt_minmax(req, min_max);
- let mut buckets = Vec::with_capacity((first_bucket_num..=last_bucket_num).count());
- for bucket_pos in first_bucket_num..=last_bucket_num {
- let bucket_key = bucket_pos as f64 * req.interval + offset;
- buckets.push(bucket_key);
- }
-
- buckets
-}
diff --git a/crates/tantivy/src/aggregation/bucket/histogram/mod.rs b/crates/tantivy/src/aggregation/bucket/histogram/mod.rs
deleted file mode 100644
index 82f695ca..00000000
--- a/crates/tantivy/src/aggregation/bucket/histogram/mod.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-mod date_histogram;
-mod histogram;
-pub use date_histogram::*;
-pub use histogram::*;
diff --git a/crates/tantivy/src/aggregation/bucket/mod.rs b/crates/tantivy/src/aggregation/bucket/mod.rs
deleted file mode 100644
index f1eaa975..00000000
--- a/crates/tantivy/src/aggregation/bucket/mod.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-//! Module for all bucket aggregations.
-//!
-//! BucketAggregations create buckets of documents.
-//! Each bucket is associated with a rule which
-//! determines whether or not a document in the falls into it. In other words, the buckets
-//! effectively define document sets. Buckets are not necessarily disjunct, therefore a document can
-//! fall into multiple buckets. In addition to the buckets themselves, the bucket aggregations also
-//! compute and return the number of documents for each bucket. Bucket aggregations, as opposed to
-//! metric aggregations, can hold sub-aggregations. These sub-aggregations will be aggregated for
-//! the buckets created by their "parent" bucket aggregation. There are different bucket
-//! aggregators, each with a different "bucketing" strategy. Some define a single bucket, some
-//! define fixed number of multiple buckets, and others dynamically create the buckets during the
-//! aggregation process.
-//!
-//! Results of final buckets are [`BucketResult`](super::agg_result::BucketResult).
-//! Results of intermediate buckets are
-//! [`IntermediateBucketResult`](super::intermediate_agg_result::IntermediateBucketResult)
-//!
-//! ## Supported Bucket Aggregations
-//! - [Histogram](HistogramAggregation)
-//! - [DateHistogram](DateHistogramAggregationReq)
-//! - [Range](RangeAggregation)
-//! - [Terms](TermsAggregation)
-
-mod histogram;
-mod range;
-mod term_agg;
-mod term_missing_agg;
-
-use std::collections::HashMap;
-use std::fmt;
-
-pub use histogram::*;
-pub use range::*;
-use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
-pub use term_agg::*;
-pub use term_missing_agg::*;
-
-/// Order for buckets in a bucket aggregation.
-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Default)]
-pub enum Order {
- /// Asc order
- #[serde(rename = "asc")]
- Asc,
- /// Desc order
- #[serde(rename = "desc")]
- #[default]
- Desc,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-/// Order property by which to apply the order
-#[derive(Default)]
-pub enum OrderTarget {
- /// The key of the bucket
- Key,
- /// The doc count of the bucket
- #[default]
- Count,
- /// Order by value of the sub aggregation metric with identified by given `String`.
- ///
- /// Only single value metrics are supported currently
- SubAggregation(String),
-}
-
-impl From<&str> for OrderTarget {
- fn from(val: &str) -> Self {
- match val {
- "_key" => OrderTarget::Key,
- "_count" => OrderTarget::Count,
- _ => OrderTarget::SubAggregation(val.to_string()),
- }
- }
-}
-
-impl fmt::Display for OrderTarget {
- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- match self {
- OrderTarget::Key => f.write_str("_key"),
- OrderTarget::Count => f.write_str("_count"),
- OrderTarget::SubAggregation(agg) => agg.fmt(f),
- }
- }
-}
-
-/// Set the order. target is either "_count", "_key", or the name of
-/// a metric sub_aggregation.
-///
-/// De/Serializes to elasticsearch compatible JSON.
-///
-/// Examples in JSON format:
-/// { "_count": "asc" }
-/// { "_key": "asc" }
-/// { "average_price": "asc" }
-#[derive(Clone, Default, Debug, PartialEq)]
-pub struct CustomOrder {
- /// The target property by which to sort by
- pub target: OrderTarget,
- /// The order asc or desc
- pub order: Order,
-}
-
-impl Serialize for CustomOrder {
- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
- where S: Serializer {
- let map: HashMap<String, Order> =
- std::iter::once((self.target.to_string(), self.order)).collect();
- map.serialize(serializer)
- }
-}
-
-impl<'de> Deserialize<'de> for CustomOrder {
- fn deserialize<D>(deserializer: D) -> Result<CustomOrder, D::Error>
- where D: Deserializer<'de> {
- HashMap::<String, Order>::deserialize(deserializer).and_then(|map| {
- if let Some((key, value)) = map.into_iter().next() {
- Ok(CustomOrder {
- target: key.as_str().into(),
- order: value,
- })
- } else {
- Err(de::Error::custom(
- "unexpected empty map in order".to_string(),
- ))
- }
- })
- }
-}
-
-#[test]
-fn custom_order_serde_test() {
- let order = CustomOrder {
- target: OrderTarget::Key,
- order: Order::Desc,
- };
-
- let order_str = serde_json::to_string(&order).unwrap();
- assert_eq!(order_str, "{\"_key\":\"desc\"}");
- let order_deser = serde_json::from_str(&order_str).unwrap();
-
- assert_eq!(order, order_deser);
-
- let order_deser: serde_json::Result<CustomOrder> = serde_json::from_str("{}");
- assert!(order_deser.is_err());
-
- let order_deser: serde_json::Result<CustomOrder> = serde_json::from_str("[]");
- assert!(order_deser.is_err());
-}
diff --git a/crates/tantivy/src/aggregation/bucket/range.rs b/crates/tantivy/src/aggregation/bucket/range.rs
deleted file mode 100644
index af86dde0..00000000
--- a/crates/tantivy/src/aggregation/bucket/range.rs
+++ /dev/null
@@ -1,837 +0,0 @@
-use std::fmt::Debug;
-use std::ops::Range;
-
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-
-use crate::aggregation::agg_limits::ResourceLimitGuard;
-use crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor;
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
- IntermediateRangeBucketEntry, IntermediateRangeBucketResult,
-};
-use crate::aggregation::segment_agg_result::{
- build_segment_agg_collector, SegmentAggregationCollector,
-};
-use crate::aggregation::*;
-use crate::TantivyError;
-
-/// Provide user-defined buckets to aggregate on.
-/// Two special buckets will automatically be created to cover the whole range of values.
-/// The provided buckets have to be continuous.
-/// During the aggregation, the values extracted from the fast_field `field` will be checked
-/// against each bucket range. Note that this aggregation includes the from value and excludes the
-/// to value for each range.
-///
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`RangeBucketEntry`](crate::aggregation::agg_result::RangeBucketEntry) on the
-/// `AggregationCollector`.
-///
-/// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateRangeBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateRangeBucketEntry) on the
-/// `DistributedAggregationCollector`.
-///
-/// # Limitations/Compatibility
-/// Overlapping ranges are not yet supported.
-///
-/// # Request JSON Format
-/// ```json
-/// {
-/// "my_ranges": {
-/// "field": "score",
-/// "ranges": [
-/// { "to": 3.0 },
-/// { "from": 3.0, "to": 7.0 },
-/// { "from": 7.0, "to": 20.0 },
-/// { "from": 20.0 }
-/// ]
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct RangeAggregation {
- /// The field to aggregate on.
- pub field: String,
- /// Note that this aggregation includes the from value and excludes the to value for each
- /// range. Extra buckets will be created until the first to, and last from, if necessary.
- pub ranges: Vec<RangeAggregationRange>,
- /// Whether to return the buckets as a hash map
- #[serde(default)]
- pub keyed: bool,
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// The range for one range bucket.
-pub struct RangeAggregationRange {
- /// Custom key for the range bucket
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub key: Option<String>,
- /// The from range value, which is inclusive in the range.
- /// `None` equals to an open ended interval.
- #[serde(
- skip_serializing_if = "Option::is_none",
- default,
- deserialize_with = "deserialize_option_f64"
- )]
- pub from: Option<f64>,
- /// The to range value, which is not inclusive in the range.
- /// `None` equals to an open ended interval.
- #[serde(
- skip_serializing_if = "Option::is_none",
- default,
- deserialize_with = "deserialize_option_f64"
- )]
- pub to: Option<f64>,
-}
-
-impl From<Range<f64>> for RangeAggregationRange {
- fn from(range: Range<f64>) -> Self {
- let from = if range.start == f64::MIN {
- None
- } else {
- Some(range.start)
- };
- let to = if range.end == f64::MAX {
- None
- } else {
- Some(range.end)
- };
- RangeAggregationRange {
- key: None,
- from,
- to,
- }
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-/// Internally used u64 range for one range bucket.
-pub(crate) struct InternalRangeAggregationRange {
- /// Custom key for the range bucket
- key: Option<String>,
- /// `u64` range value
- range: Range<u64>,
-}
-
-impl From<Range<u64>> for InternalRangeAggregationRange {
- fn from(range: Range<u64>) -> Self {
- InternalRangeAggregationRange { key: None, range }
- }
-}
-
-#[derive(Clone, Debug)]
-pub(crate) struct SegmentRangeAndBucketEntry {
- range: Range<u64>,
- bucket: SegmentRangeBucketEntry,
-}
-
-/// The collector puts values from the fast field into the correct buckets and does a conversion to
-/// the correct datatype.
-#[derive(Clone, Debug)]
-pub struct SegmentRangeCollector {
- /// The buckets containing the aggregation data.
- buckets: Vec<SegmentRangeAndBucketEntry>,
- column_type: ColumnType,
- pub(crate) accessor_idx: usize,
-}
-
-#[derive(Clone)]
-pub(crate) struct SegmentRangeBucketEntry {
- pub key: Key,
- pub doc_count: u64,
- pub sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,
- /// The from range of the bucket. Equals `f64::MIN` when `None`.
- pub from: Option<f64>,
- /// The to range of the bucket. Equals `f64::MAX` when `None`. Open interval, `to` is not
- /// inclusive.
- pub to: Option<f64>,
-}
-
-impl Debug for SegmentRangeBucketEntry {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("SegmentRangeBucketEntry")
- .field("key", &self.key)
- .field("doc_count", &self.doc_count)
- .field("from", &self.from)
- .field("to", &self.to)
- .finish()
- }
-}
-impl SegmentRangeBucketEntry {
- pub(crate) fn into_intermediate_bucket_entry(
- self,
- agg_with_accessor: &AggregationsWithAccessor,
- ) -> crate::Result<IntermediateRangeBucketEntry> {
- let mut sub_aggregation_res = IntermediateAggregationResults::default();
- if let Some(sub_aggregation) = self.sub_aggregation {
- sub_aggregation
- .add_intermediate_aggregation_result(agg_with_accessor, &mut sub_aggregation_res)?
- } else {
- Default::default()
- };
-
- Ok(IntermediateRangeBucketEntry {
- key: self.key.into(),
- doc_count: self.doc_count,
- sub_aggregation: sub_aggregation_res,
- from: self.from,
- to: self.to,
- })
- }
-}
-
-impl SegmentAggregationCollector for SegmentRangeCollector {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let field_type = self.column_type;
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- let sub_agg = &agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;
-
- let buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry> = self
- .buckets
- .into_iter()
- .map(move |range_bucket| {
- Ok((
- range_to_string(&range_bucket.range, &field_type)?,
- range_bucket
- .bucket
- .into_intermediate_bucket_entry(sub_agg)?,
- ))
- })
- .collect::<crate::Result<_>>()?;
-
- let bucket = IntermediateBucketResult::Range(IntermediateRangeBucketResult {
- buckets,
- column_type: Some(self.column_type),
- });
-
- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.collect_block(&[doc], agg_with_accessor)
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];
-
- bucket_agg_accessor
- .column_block_accessor
- .fetch_block(docs, &bucket_agg_accessor.accessor);
-
- for (doc, val) in bucket_agg_accessor
- .column_block_accessor
- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)
- {
- let bucket_pos = self.get_bucket_pos(val);
-
- let bucket = &mut self.buckets[bucket_pos];
-
- bucket.bucket.doc_count += 1;
- if let Some(sub_aggregation) = &mut bucket.bucket.sub_aggregation {
- sub_aggregation.collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;
- }
- }
-
- Ok(())
- }
-
- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- let sub_aggregation_accessor =
- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;
-
- for bucket in self.buckets.iter_mut() {
- if let Some(sub_agg) = bucket.bucket.sub_aggregation.as_mut() {
- sub_agg.flush(sub_aggregation_accessor)?;
- }
- }
-
- Ok(())
- }
-}
-
-impl SegmentRangeCollector {
- pub(crate) fn from_req_and_validate(
- req: &RangeAggregation,
- sub_aggregation: &mut AggregationsWithAccessor,
- limits: &ResourceLimitGuard,
- field_type: ColumnType,
- accessor_idx: usize,
- ) -> crate::Result<Self> {
- // The range input on the request is f64.
- // We need to convert to u64 ranges, because we read the values as u64.
- // The mapping from the conversion is monotonic so ordering is preserved.
- let buckets: Vec<_> = extend_validate_ranges(&req.ranges, &field_type)?
- .iter()
- .map(|range| {
- let key = range
- .key
- .clone()
- .map(|key| Ok(Key::Str(key)))
- .unwrap_or_else(|| range_to_key(&range.range, &field_type))?;
- let to = if range.range.end == u64::MAX {
- None
- } else {
- Some(f64_from_fastfield_u64(range.range.end, &field_type))
- };
- let from = if range.range.start == u64::MIN {
- None
- } else {
- Some(f64_from_fastfield_u64(range.range.start, &field_type))
- };
- let sub_aggregation = if sub_aggregation.is_empty() {
- None
- } else {
- Some(build_segment_agg_collector(sub_aggregation)?)
- };
-
- Ok(SegmentRangeAndBucketEntry {
- range: range.range.clone(),
- bucket: SegmentRangeBucketEntry {
- doc_count: 0,
- sub_aggregation,
- key,
- from,
- to,
- },
- })
- })
- .collect::<crate::Result<_>>()?;
-
- limits.add_memory_consumed(
- buckets.len() as u64 * std::mem::size_of::<SegmentRangeAndBucketEntry>() as u64,
- )?;
-
- Ok(SegmentRangeCollector {
- buckets,
- column_type: field_type,
- accessor_idx,
- })
- }
-
- #[inline]
- fn get_bucket_pos(&self, val: u64) -> usize {
- let pos = self
- .buckets
- .binary_search_by_key(&val, |probe| probe.range.start)
- .unwrap_or_else(|pos| pos - 1);
- debug_assert!(self.buckets[pos].range.contains(&val));
- pos
- }
-}
-
-/// Converts the user provided f64 range value to fast field value space.
-///
-/// Internally fast field values are always stored as u64.
-/// If the fast field has u64 `[1, 2, 5]`, these values are stored as is in the fast field.
-/// A fast field with f64 `[1.0, 2.0, 5.0]` is converted to u64 space, using a
-/// monotonic mapping function, so the order is preserved.
-///
-/// Consequently, a f64 user range 1.0..3.0 needs to be converted to fast field value space using
-/// the same monotonic mapping function, so that the provided ranges contain the u64 values in the
-/// fast field.
-/// The alternative would be that every value read would be converted to the f64 range, but that is
-/// more computational expensive when many documents are hit.
-fn to_u64_range(
- range: &RangeAggregationRange,
- field_type: &ColumnType,
-) -> crate::Result<InternalRangeAggregationRange> {
- let start = if let Some(from) = range.from {
- f64_to_fastfield_u64(from, field_type)
- .ok_or_else(|| TantivyError::InvalidArgument("invalid field type".to_string()))?
- } else {
- u64::MIN
- };
-
- let end = if let Some(to) = range.to {
- f64_to_fastfield_u64(to, field_type)
- .ok_or_else(|| TantivyError::InvalidArgument("invalid field type".to_string()))?
- } else {
- u64::MAX
- };
-
- Ok(InternalRangeAggregationRange {
- key: range.key.clone(),
- range: start..end,
- })
-}
-
-/// Extends the provided buckets to contain the whole value range, by inserting buckets at the
-/// beginning and end and filling gaps.
-fn extend_validate_ranges(
- buckets: &[RangeAggregationRange],
- field_type: &ColumnType,
-) -> crate::Result<Vec<InternalRangeAggregationRange>> {
- let mut converted_buckets = buckets
- .iter()
- .map(|range| to_u64_range(range, field_type))
- .collect::<crate::Result<Vec<_>>>()?;
-
- converted_buckets.sort_by_key(|bucket| bucket.range.start);
- if converted_buckets[0].range.start != u64::MIN {
- converted_buckets.insert(0, (u64::MIN..converted_buckets[0].range.start).into());
- }
-
- if converted_buckets[converted_buckets.len() - 1].range.end != u64::MAX {
- converted_buckets
- .push((converted_buckets[converted_buckets.len() - 1].range.end..u64::MAX).into());
- }
-
- // fill up holes in the ranges
- let find_hole = |converted_buckets: &[InternalRangeAggregationRange]| {
- for (pos, ranges) in converted_buckets.windows(2).enumerate() {
- if ranges[0].range.end > ranges[1].range.start {
- return Err(TantivyError::InvalidArgument(format!(
- "Overlapping ranges not supported range {:?}, range+1 {:?}",
- ranges[0], ranges[1]
- )));
- }
- if ranges[0].range.end != ranges[1].range.start {
- return Ok(Some(pos));
- }
- }
- Ok(None)
- };
-
- while let Some(hole_pos) = find_hole(&converted_buckets)? {
- let new_range =
- converted_buckets[hole_pos].range.end..converted_buckets[hole_pos + 1].range.start;
- converted_buckets.insert(hole_pos + 1, new_range.into());
- }
-
- Ok(converted_buckets)
-}
-
-pub(crate) fn range_to_string(
- range: &Range<u64>,
- field_type: &ColumnType,
-) -> crate::Result<String> {
- // is_start is there for malformed requests, e.g. ig the user passes the range u64::MIN..0.0,
- // it should be rendered as "*-0" and not "*-*"
- let to_str = |val: u64, is_start: bool| {
- if (is_start && val == u64::MIN) || (!is_start && val == u64::MAX) {
- Ok("*".to_string())
- } else if *field_type == ColumnType::DateTime {
- let val = i64::from_u64(val);
- format_date(val)
- } else {
- Ok(f64_from_fastfield_u64(val, field_type).to_string())
- }
- };
-
- Ok(format!(
- "{}-{}",
- to_str(range.start, true)?,
- to_str(range.end, false)?
- ))
-}
-
-pub(crate) fn range_to_key(range: &Range<u64>, field_type: &ColumnType) -> crate::Result<Key> {
- Ok(Key::Str(range_to_string(range, field_type)?))
-}
-
-#[cfg(test)]
-mod tests {
-
- use serde_json::Value;
-
- use super::*;
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::tests::{
- exec_request, exec_request_with_query, get_test_index_2_segments,
- get_test_index_with_num_docs,
- };
-
- pub fn get_collector_from_ranges(
- ranges: Vec<RangeAggregationRange>,
- field_type: ColumnType,
- ) -> SegmentRangeCollector {
- let req = RangeAggregation {
- field: "dummy".to_string(),
- ranges,
- ..Default::default()
- };
-
- SegmentRangeCollector::from_req_and_validate(
- &req,
- &mut Default::default(),
- &AggregationLimits::default().new_guard(),
- field_type,
- 0,
- )
- .expect("unexpected error")
- }
-
- #[test]
- fn range_fraction_test() -> crate::Result<()> {
- let index = get_test_index_with_num_docs(false, 100)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "range": {
- "range": {
- "field": "fraction_f64",
- "ranges": [
- {"from": 0.0, "to": 0.1},
- {"from": 0.1, "to": 0.2},
- ]
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(res["range"]["buckets"][0]["key"], "*-0");
- assert_eq!(res["range"]["buckets"][0]["doc_count"], 0);
- assert_eq!(res["range"]["buckets"][1]["key"], "0-0.1");
- assert_eq!(res["range"]["buckets"][1]["doc_count"], 10);
- assert_eq!(res["range"]["buckets"][2]["key"], "0.1-0.2");
- assert_eq!(res["range"]["buckets"][2]["doc_count"], 10);
- assert_eq!(res["range"]["buckets"][3]["key"], "0.2-*");
- assert_eq!(res["range"]["buckets"][3]["doc_count"], 80);
-
- Ok(())
- }
-
- #[test]
- fn range_fraction_test_with_sub_agg() -> crate::Result<()> {
- let index = get_test_index_with_num_docs(false, 100)?;
-
- let sub_agg_req: Aggregations = serde_json::from_value(json!({
- "avg": { "avg": { "field": "score_f64", } }
-
- }))
- .unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "range": {
- "range": {
- "field": "fraction_f64",
- "ranges": [
- {"from": 0.0, "to": 0.1},
- {"from": 0.1, "to": 0.2},
- ]
- },
- "aggs": sub_agg_req
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(res["range"]["buckets"][0]["key"], "*-0");
- assert_eq!(res["range"]["buckets"][0]["doc_count"], 0);
- assert_eq!(res["range"]["buckets"][1]["key"], "0-0.1");
- assert_eq!(res["range"]["buckets"][1]["doc_count"], 10);
- assert_eq!(res["range"]["buckets"][2]["key"], "0.1-0.2");
- assert_eq!(res["range"]["buckets"][2]["doc_count"], 10);
- assert_eq!(res["range"]["buckets"][3]["key"], "0.2-*");
- assert_eq!(res["range"]["buckets"][3]["doc_count"], 80);
-
- Ok(())
- }
-
- #[test]
- fn range_keyed_buckets_test() -> crate::Result<()> {
- let index = get_test_index_with_num_docs(false, 100)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "range": {
- "range": {
- "field": "fraction_f64",
- "ranges": [
- {"from": 0.0, "to": 0.1},
- {"from": 0.1, "to": 0.2},
- ],
- "keyed": true
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res,
- json!({
- "range": {
- "buckets": {
- "*-0": { "key": "*-0", "doc_count": 0, "to": 0.0},
- "0-0.1": {"key": "0-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},
- "0.1-0.2": {"key": "0.1-0.2", "doc_count": 10, "from": 0.1, "to": 0.2},
- "0.2-*": {"key": "0.2-*", "doc_count": 80, "from": 0.2},
- }
- }
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn range_custom_key_test() -> crate::Result<()> {
- let index = get_test_index_with_num_docs(false, 100)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "range": {
- "range": {
- "field": "fraction_f64",
- "ranges": [
- {"key": "custom-key-0-to-0.1", "from": 0.0, "to": 0.1},
- {"from": 0.1, "to": 0.2},
- ],
- "keyed": false
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res,
- json!({
- "range": {
- "buckets": [
- {"key": "*-0", "doc_count": 0, "to": 0.0},
- {"key": "custom-key-0-to-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},
- {"key": "0.1-0.2", "doc_count": 10, "from": 0.1, "to": 0.2},
- {"key": "0.2-*", "doc_count": 80, "from": 0.2}
- ]
- }
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn range_date_test_single_segment() -> crate::Result<()> {
- range_date_test_with_opt(true)
- }
-
- #[test]
- fn range_date_test_multi_segment() -> crate::Result<()> {
- range_date_test_with_opt(false)
- }
-
- fn range_date_test_with_opt(merge_segments: bool) -> crate::Result<()> {
- let index = get_test_index_2_segments(merge_segments)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "date_ranges": {
- "range": {
- "field": "date",
- "ranges": [
- {"to": 1546300800000000000i64},
- {"from": 1546300800000000000i64, "to": 1546387200000000000i64},
- ],
- "keyed": false
- },
- }
- }))
- .unwrap();
-
- let agg_res = exec_request(agg_req, &index)?;
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
-
- assert_eq!(
- res["date_ranges"]["buckets"][0]["from_as_string"],
- Value::Null
- );
- assert_eq!(
- res["date_ranges"]["buckets"][0]["key"],
- "*-2019-01-01T00:00:00Z"
- );
- assert_eq!(
- res["date_ranges"]["buckets"][1]["from_as_string"],
- "2019-01-01T00:00:00Z"
- );
- assert_eq!(
- res["date_ranges"]["buckets"][1]["to_as_string"],
- "2019-01-02T00:00:00Z"
- );
-
- assert_eq!(
- res["date_ranges"]["buckets"][2]["from_as_string"],
- "2019-01-02T00:00:00Z"
- );
- assert_eq!(
- res["date_ranges"]["buckets"][2]["to_as_string"],
- Value::Null
- );
-
- Ok(())
- }
-
- #[test]
- fn range_custom_key_keyed_buckets_test() -> crate::Result<()> {
- let index = get_test_index_with_num_docs(false, 100)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "range": {
- "range": {
- "field": "fraction_f64",
- "ranges": [
- {"key": "custom-key-0-to-0.1", "from": 0.0, "to": 0.1},
- ],
- "keyed": true
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res,
- json!({
- "range": {
- "buckets": {
- "*-0": { "key": "*-0", "doc_count": 0, "to": 0.0},
- "custom-key-0-to-0.1": {"key": "custom-key-0-to-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},
- "0.1-*": {"key": "0.1-*", "doc_count": 90, "from": 0.1},
- }
- }
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn bucket_test_extend_range_hole() {
- let buckets = vec![(10f64..20f64).into(), (30f64..40f64).into()];
- let collector = get_collector_from_ranges(buckets, ColumnType::F64);
-
- let buckets = collector.buckets;
- assert_eq!(buckets[0].range.start, u64::MIN);
- assert_eq!(buckets[0].range.end, 10f64.to_u64());
- assert_eq!(buckets[1].range.start, 10f64.to_u64());
- assert_eq!(buckets[1].range.end, 20f64.to_u64());
- // Added bucket to fill hole
- assert_eq!(buckets[2].range.start, 20f64.to_u64());
- assert_eq!(buckets[2].range.end, 30f64.to_u64());
- assert_eq!(buckets[3].range.start, 30f64.to_u64());
- assert_eq!(buckets[3].range.end, 40f64.to_u64());
- }
-
- #[test]
- fn bucket_test_range_conversion_special_case() {
- // the monotonic conversion between f64 and u64, does not map f64::MIN.to_u64() ==
- // u64::MIN, but the into trait converts f64::MIN/MAX to None
- let buckets = vec![
- (f64::MIN..10f64).into(),
- (10f64..20f64).into(),
- (20f64..f64::MAX).into(),
- ];
- let collector = get_collector_from_ranges(buckets, ColumnType::F64);
-
- let buckets = collector.buckets;
- assert_eq!(buckets[0].range.start, u64::MIN);
- assert_eq!(buckets[0].range.end, 10f64.to_u64());
- assert_eq!(buckets[1].range.start, 10f64.to_u64());
- assert_eq!(buckets[1].range.end, 20f64.to_u64());
- assert_eq!(buckets[2].range.start, 20f64.to_u64());
- assert_eq!(buckets[2].range.end, u64::MAX);
- assert_eq!(buckets.len(), 3);
- }
-
- #[test]
- fn bucket_range_test_negative_vals() {
- let buckets = vec![(-10f64..-1f64).into()];
- let collector = get_collector_from_ranges(buckets, ColumnType::F64);
-
- let buckets = collector.buckets;
- assert_eq!(&buckets[0].bucket.key.to_string(), "*--10");
- assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "-1-*");
- }
- #[test]
- fn bucket_range_test_positive_vals() {
- let buckets = vec![(0f64..10f64).into()];
- let collector = get_collector_from_ranges(buckets, ColumnType::F64);
-
- let buckets = collector.buckets;
- assert_eq!(&buckets[0].bucket.key.to_string(), "*-0");
- assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "10-*");
- }
-
- #[test]
- fn range_binary_search_test_u64() {
- let check_ranges = |ranges: Vec<RangeAggregationRange>| {
- let collector = get_collector_from_ranges(ranges, ColumnType::U64);
- let search = |val: u64| collector.get_bucket_pos(val);
-
- assert_eq!(search(u64::MIN), 0);
- assert_eq!(search(9), 0);
- assert_eq!(search(10), 1);
- assert_eq!(search(11), 1);
- assert_eq!(search(99), 1);
- assert_eq!(search(100), 2);
- assert_eq!(search(u64::MAX - 1), 2); // Since the end range is never included, the max
- // value
- };
-
- let ranges = vec![(10.0..100.0).into()];
- check_ranges(ranges);
-
- let ranges = vec![
- RangeAggregationRange {
- key: None,
- to: Some(10.0),
- from: None,
- },
- (10.0..100.0).into(),
- ];
- check_ranges(ranges);
-
- let ranges = vec![
- RangeAggregationRange {
- key: None,
- to: Some(10.0),
- from: None,
- },
- (10.0..100.0).into(),
- RangeAggregationRange {
- key: None,
- to: None,
- from: Some(100.0),
- },
- ];
- check_ranges(ranges);
- }
-
- #[test]
- fn range_binary_search_test_f64() {
- let ranges = vec![(10.0..100.0).into()];
-
- let collector = get_collector_from_ranges(ranges, ColumnType::F64);
- let search = |val: u64| collector.get_bucket_pos(val);
-
- assert_eq!(search(u64::MIN), 0);
- assert_eq!(search(9f64.to_u64()), 0);
- assert_eq!(search(10f64.to_u64()), 1);
- assert_eq!(search(11f64.to_u64()), 1);
- assert_eq!(search(99f64.to_u64()), 1);
- assert_eq!(search(100f64.to_u64()), 2);
- assert_eq!(search(u64::MAX - 1), 2); // Since the end range is never included,
- // the max value
- }
-}
diff --git a/crates/tantivy/src/aggregation/bucket/term_agg.rs b/crates/tantivy/src/aggregation/bucket/term_agg.rs
deleted file mode 100644
index 7c8000b7..00000000
--- a/crates/tantivy/src/aggregation/bucket/term_agg.rs
+++ /dev/null
@@ -1,2002 +0,0 @@
-use std::fmt::Debug;
-use std::net::Ipv6Addr;
-
-use crate::columnar::column_values::CompactSpaceU64Accessor;
-use crate::columnar::{
- BytesColumn, ColumnType, MonotonicallyMappableToU128, MonotonicallyMappableToU64, StrColumn,
-};
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-
-use super::{CustomOrder, Order, OrderTarget};
-use crate::aggregation::agg_limits::MemoryConsumption;
-use crate::aggregation::agg_req_with_accessor::{
- AggregationWithAccessor, AggregationsWithAccessor,
-};
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
- IntermediateKey, IntermediateTermBucketEntry, IntermediateTermBucketResult,
-};
-use crate::aggregation::segment_agg_result::{
- build_segment_agg_collector, SegmentAggregationCollector,
-};
-use crate::aggregation::{f64_from_fastfield_u64, format_date, Key};
-use crate::error::DataCorruption;
-use crate::TantivyError;
-
-/// Creates a bucket for every unique term and counts the number of occurences.
-/// Note that doc_count in the response buckets equals term count here.
-///
-/// If the text is untokenized and single value, that means one term per document and therefore it
-/// is in fact doc count.
-///
-/// ## Prerequisite
-/// Term aggregations work only on [fast fields](`crate::fastfield`) of type `u64`, `f64`, `i64` and
-/// text.
-///
-/// ## Document count error
-/// To improve performance, results from one segment are cut off at `segment_size`. On a index with
-/// a single segment this is fine. When combining results of multiple segments, terms that
-/// don't make it in the top n of a shard increase the theoretical upper bound error by lowest
-/// term-count.
-///
-/// Even with a larger `segment_size` value, doc_count values for a terms aggregation may be
-/// approximate. As a result, any sub-aggregations on the terms aggregation may also be approximate.
-/// `sum_other_doc_count` is the number of documents that didn’t make it into the top size
-/// terms. If this is greater than 0, you can be sure that the terms agg had to throw away some
-/// buckets, either because they didn’t fit into size on the root node or they didn’t fit into
-/// `segment_size` on the segment node.
-///
-/// ## Per bucket document count error
-/// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will include
-/// doc_count_error_upper_bound, which is an upper bound to the error on the doc_count returned by
-/// each segment. It’s the sum of the size of the largest bucket on each segment that didn’t fit
-/// into segment_size.
-///
-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with
-/// [`BucketEntry`](crate::aggregation::agg_result::BucketEntry) on the
-/// `AggregationCollector`.
-///
-/// Result type is
-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with
-/// [`IntermediateTermBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateTermBucketEntry) on the
-/// `DistributedAggregationCollector`.
-///
-/// # Limitations/Compatibility
-///
-/// Each segment returns up to [segment_size](TermsAggregation::segment_size) results. This
-/// differences to elasticsearch behaviour.
-///
-/// # Request JSON Format
-/// ```json
-/// {
-/// "genres": {
-/// "terms":{ "field": "genre" }
-/// }
-/// }
-/// ```
-///
-/// /// # Response JSON Format
-/// ```json
-/// {
-/// ...
-/// "aggregations": {
-/// "genres": {
-/// "doc_count_error_upper_bound": 0,
-/// "sum_other_doc_count": 0,
-/// "buckets": [
-/// { "key": "drumnbass", "doc_count": 6 },
-/// { "key": "raggae", "doc_count": 4 },
-/// { "key": "jazz", "doc_count": 2 }
-/// ]
-/// }
-/// }
-/// }
-/// ```
-
-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
-pub struct TermsAggregation {
- /// The field to aggregate on.
- pub field: String,
- /// By default, the top 10 terms with the most documents are returned.
- /// Larger values for size are more expensive.
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub size: Option<u32>,
-
- /// To get more accurate results, we fetch more than `size` from each segment.
- ///
- /// Increasing this value is will increase the cost for more accuracy.
- ///
- /// Defaults to 10 * size.
- #[serde(skip_serializing_if = "Option::is_none", default)]
- #[serde(alias = "shard_size")]
- #[serde(alias = "split_size")]
- pub segment_size: Option<u32>,
-
- /// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will
- /// include doc_count_error_upper_bound, which is an upper bound to the error on the
- /// doc_count returned by each shard. It’s the sum of the size of the largest bucket on
- /// each segment that didn’t fit into `shard_size`.
- ///
- /// Defaults to true when ordering by count desc.
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub show_term_doc_count_error: Option<bool>,
-
- /// Filter all terms that are lower than `min_doc_count`. Defaults to 1.
- ///
- /// **Expensive**: When set to 0, this will return all terms in the field.
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub min_doc_count: Option<u64>,
-
- /// Set the order. `String` is here a target, which is either "_count", "_key", or the name of
- /// a metric sub_aggregation.
- ///
- /// Single value metrics like average can be addressed by its name.
- /// Multi value metrics like stats are required to address their field by name e.g.
- /// "stats.avg"
- ///
- /// Examples in JSON format:
- /// { "_count": "asc" }
- /// { "_key": "asc" }
- /// { "average_price": "asc" }
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub order: Option<CustomOrder>,
-
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "missing": "NO_DATA" }
- ///
- /// # Internal
- ///
- /// Internally, `missing` requires some specialized handling in some scenarios.
- ///
- /// Simple Case:
- /// In the simplest case, we can just put the missing value in the termmap use that. In case of
- /// text we put a special u64::MAX and replace it at the end with the actual missing value,
- /// when loading the text.
- /// Special Case 1:
- /// If we have multiple columns on one field, we need to have a union on the indices on both
- /// columns, to find docids without a value. That requires a special missing aggreggation.
- /// Special Case 2: if the key is of type text and the column is numerical, we also need to use
- /// the special missing aggregation, since there is no mechanism in the numerical column to
- /// add text.
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub missing: Option<Key>,
-}
-
-/// Same as TermsAggregation, but with populated defaults.
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct TermsAggregationInternal {
- /// The field to aggregate on.
- pub field: String,
- /// By default, the top 10 terms with the most documents are returned.
- /// Larger values for size are more expensive.
- ///
- /// Defaults to 10.
- pub size: u32,
-
- /// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will
- /// include doc_count_error_upper_bound, which is an upper bound to the error on the
- /// doc_count returned by each shard. It’s the sum of the size of the largest bucket on
- /// each segment that didn’t fit into `segment_size`.
- pub show_term_doc_count_error: bool,
-
- /// The get more accurate results, we fetch more than `size` from each segment.
- ///
- /// Increasing this value is will increase the cost for more accuracy.
- pub segment_size: u32,
-
- /// Filter all terms that are lower than `min_doc_count`. Defaults to 1.
- ///
- /// *Expensive*: When set to 0, this will return all terms in the field.
- pub min_doc_count: u64,
-
- pub order: CustomOrder,
- pub missing: Option<Key>,
-}
-
-impl TermsAggregationInternal {
- pub(crate) fn from_req(req: &TermsAggregation) -> Self {
- let size = req.size.unwrap_or(10);
-
- let mut segment_size = req.segment_size.unwrap_or(size * 10);
-
- let order = req.order.clone().unwrap_or_default();
- segment_size = segment_size.max(size);
- TermsAggregationInternal {
- field: req.field.to_string(),
- size,
- segment_size,
- show_term_doc_count_error: req
- .show_term_doc_count_error
- .unwrap_or_else(|| order == CustomOrder::default()),
- min_doc_count: req.min_doc_count.unwrap_or(1),
- order,
- missing: req.missing.clone(),
- }
- }
-}
-
-#[derive(Clone, Debug, Default)]
-/// Container to store term_ids/or u64 values and their buckets.
-struct TermBuckets {
- pub(crate) entries: FxHashMap<u64, u32>,
- pub(crate) sub_aggs: FxHashMap<u64, Box<dyn SegmentAggregationCollector>>,
-}
-
-impl TermBuckets {
- fn get_memory_consumption(&self) -> usize {
- let sub_aggs_mem = self.sub_aggs.memory_consumption();
- let buckets_mem = self.entries.memory_consumption();
- sub_aggs_mem + buckets_mem
- }
-
- fn force_flush(
- &mut self,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- for sub_aggregations in &mut self.sub_aggs.values_mut() {
- sub_aggregations.as_mut().flush(agg_with_accessor)?;
- }
- Ok(())
- }
-}
-
-/// The collector puts values from the fast field into the correct buckets and does a conversion to
-/// the correct datatype.
-#[derive(Clone, Debug)]
-pub struct SegmentTermCollector {
- /// The buckets containing the aggregation data.
- term_buckets: TermBuckets,
- req: TermsAggregationInternal,
- blueprint: Option<Box<dyn SegmentAggregationCollector>>,
- column_type: ColumnType,
- accessor_idx: usize,
-}
-
-pub(crate) fn get_agg_name_and_property(name: &str) -> (&str, &str) {
- let (agg_name, agg_property) = name.split_once('.').unwrap_or((name, ""));
- (agg_name, agg_property)
-}
-
-impl SegmentAggregationCollector for SegmentTermCollector {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];
-
- let bucket = self.into_intermediate_bucket_result(agg_with_accessor)?;
- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.collect_block(&[doc], agg_with_accessor)
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];
-
- let mem_pre = self.get_memory_consumption();
-
- if let Some(missing) = bucket_agg_accessor.missing_value_for_accessor {
- bucket_agg_accessor
- .column_block_accessor
- .fetch_block_with_missing(docs, &bucket_agg_accessor.accessor, missing);
- } else {
- bucket_agg_accessor
- .column_block_accessor
- .fetch_block(docs, &bucket_agg_accessor.accessor);
- }
-
- for term_id in bucket_agg_accessor.column_block_accessor.iter_vals() {
- let entry = self.term_buckets.entries.entry(term_id).or_default();
- *entry += 1;
- }
- // has subagg
- if let Some(blueprint) = self.blueprint.as_ref() {
- for (doc, term_id) in bucket_agg_accessor
- .column_block_accessor
- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)
- {
- let sub_aggregations = self
- .term_buckets
- .sub_aggs
- .entry(term_id)
- .or_insert_with(|| blueprint.clone());
- sub_aggregations.collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;
- }
- }
-
- let mem_delta = self.get_memory_consumption() - mem_pre;
- if mem_delta > 0 {
- bucket_agg_accessor
- .limits
- .add_memory_consumed(mem_delta as u64)?;
- }
-
- Ok(())
- }
-
- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- let sub_aggregation_accessor =
- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;
-
- self.term_buckets.force_flush(sub_aggregation_accessor)?;
- Ok(())
- }
-}
-
-impl SegmentTermCollector {
- fn get_memory_consumption(&self) -> usize {
- let self_mem = std::mem::size_of::<Self>();
- let term_buckets_mem = self.term_buckets.get_memory_consumption();
- self_mem + term_buckets_mem
- }
-
- pub(crate) fn from_req_and_validate(
- req: &TermsAggregation,
- sub_aggregations: &mut AggregationsWithAccessor,
- field_type: ColumnType,
- accessor_idx: usize,
- ) -> crate::Result<Self> {
- if field_type == ColumnType::Bytes {
- return Err(TantivyError::InvalidArgument(format!(
- "terms aggregation is not supported for column type {field_type:?}"
- )));
- }
- let term_buckets = TermBuckets::default();
-
- if let Some(custom_order) = req.order.as_ref() {
- // Validate sub aggregtion exists
- if let OrderTarget::SubAggregation(sub_agg_name) = &custom_order.target {
- let (agg_name, _agg_property) = get_agg_name_and_property(sub_agg_name);
-
- sub_aggregations.aggs.get(agg_name).ok_or_else(|| {
- TantivyError::InvalidArgument(format!(
- "could not find aggregation with name {agg_name} in metric \
- sub_aggregations"
- ))
- })?;
- }
- }
-
- let has_sub_aggregations = !sub_aggregations.is_empty();
- let blueprint = if has_sub_aggregations {
- let sub_aggregation = build_segment_agg_collector(sub_aggregations)?;
- Some(sub_aggregation)
- } else {
- None
- };
-
- Ok(SegmentTermCollector {
- req: TermsAggregationInternal::from_req(req),
- term_buckets,
- blueprint,
- column_type: field_type,
- accessor_idx,
- })
- }
-
- #[inline]
- pub(crate) fn into_intermediate_bucket_result(
- mut self,
- agg_with_accessor: &AggregationWithAccessor,
- ) -> crate::Result<IntermediateBucketResult> {
- let mut entries: Vec<(u64, u32)> = self.term_buckets.entries.into_iter().collect();
-
- let order_by_sub_aggregation =
- matches!(self.req.order.target, OrderTarget::SubAggregation(_));
-
- match self.req.order.target {
- OrderTarget::Key => {
- // We rely on the fact, that term ordinals match the order of the strings
- // TODO: We could have a special collector, that keeps only TOP n results at any
- // time.
- if self.req.order.order == Order::Desc {
- entries.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.0));
- } else {
- entries.sort_unstable_by_key(|bucket| bucket.0);
- }
- }
- OrderTarget::SubAggregation(_name) => {
- // don't sort and cut off since it's hard to make assumptions on the quality of the
- // results when cutting off du to unknown nature of the sub_aggregation (possible
- // to check).
- }
- OrderTarget::Count => {
- if self.req.order.order == Order::Desc {
- entries.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.1));
- } else {
- entries.sort_unstable_by_key(|bucket| bucket.1);
- }
- }
- }
-
- let (term_doc_count_before_cutoff, sum_other_doc_count) = if order_by_sub_aggregation {
- (0, 0)
- } else {
- cut_off_buckets(&mut entries, self.req.segment_size as usize)
- };
-
- let mut dict: FxHashMap<IntermediateKey, IntermediateTermBucketEntry> = Default::default();
- dict.reserve(entries.len());
-
- let mut into_intermediate_bucket_entry =
- |id, doc_count| -> crate::Result<IntermediateTermBucketEntry> {
- let intermediate_entry = if self.blueprint.as_ref().is_some() {
- let mut sub_aggregation_res = IntermediateAggregationResults::default();
- self.term_buckets
- .sub_aggs
- .remove(&id)
- .unwrap_or_else(|| {
- panic!("Internal Error: could not find subaggregation for id {id}")
- })
- .add_intermediate_aggregation_result(
- &agg_with_accessor.sub_aggregation,
- &mut sub_aggregation_res,
- )?;
-
- IntermediateTermBucketEntry {
- doc_count,
- sub_aggregation: sub_aggregation_res,
- }
- } else {
- IntermediateTermBucketEntry {
- doc_count,
- sub_aggregation: Default::default(),
- }
- };
- Ok(intermediate_entry)
- };
-
- if self.column_type == ColumnType::Str {
- let term_dict = agg_with_accessor
- .str_dict_column
- .as_ref()
- .cloned()
- .unwrap_or_else(|| {
- StrColumn::wrap(BytesColumn::empty(agg_with_accessor.accessor.num_docs()))
- });
- let mut buffer = String::new();
- for (term_id, doc_count) in entries {
- let intermediate_entry = into_intermediate_bucket_entry(term_id, doc_count)?;
- // Special case for missing key
- if term_id == u64::MAX {
- let missing_key = self
- .req
- .missing
- .as_ref()
- .expect("Found placeholder term_id but `missing` is None");
- match missing_key {
- Key::Str(missing) => {
- buffer.clear();
- buffer.push_str(missing);
- dict.insert(
- IntermediateKey::Str(buffer.to_string()),
- intermediate_entry,
- );
- }
- Key::F64(val) => {
- buffer.push_str(&val.to_string());
- dict.insert(IntermediateKey::F64(*val), intermediate_entry);
- }
- }
- } else {
- if !term_dict.ord_to_str(term_id, &mut buffer)? {
- return Err(TantivyError::InternalError(format!(
- "Couldn't find term_id {term_id} in dict"
- )));
- }
- dict.insert(IntermediateKey::Str(buffer.to_string()), intermediate_entry);
- }
- }
- if self.req.min_doc_count == 0 {
- // TODO: Handle rev streaming for descending sorting by keys
- let mut stream = term_dict.dictionary().stream()?;
- let empty_sub_aggregation = IntermediateAggregationResults::empty_from_req(
- agg_with_accessor.agg.sub_aggregation(),
- );
- while let Some((key, _ord)) = stream.next() {
- if dict.len() >= self.req.segment_size as usize {
- break;
- }
-
- let key = IntermediateKey::Str(
- std::str::from_utf8(key)
- .map_err(|utf8_err| DataCorruption::comment_only(utf8_err.to_string()))?
- .to_string(),
- );
-
- dict.entry(key.clone())
- .or_insert_with(|| IntermediateTermBucketEntry {
- doc_count: 0,
- sub_aggregation: empty_sub_aggregation.clone(),
- });
- }
- }
- } else if self.column_type == ColumnType::DateTime {
- for (val, doc_count) in entries {
- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
- let val = i64::from_u64(val);
- let date = format_date(val)?;
- dict.insert(IntermediateKey::Str(date), intermediate_entry);
- }
- } else if self.column_type == ColumnType::Bool {
- for (val, doc_count) in entries {
- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
- let val = bool::from_u64(val);
- dict.insert(IntermediateKey::Bool(val), intermediate_entry);
- }
- } else if self.column_type == ColumnType::IpAddr {
- let compact_space_accessor = agg_with_accessor
- .accessor
- .values
- .clone()
- .downcast_arc::<CompactSpaceU64Accessor>()
- .map_err(|_| {
- TantivyError::AggregationError(
- crate::aggregation::AggregationError::InternalError(
- "Type mismatch: Could not downcast to CompactSpaceU64Accessor"
- .to_string(),
- ),
- )
- })?;
-
- for (val, doc_count) in entries {
- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
- let val: u128 = compact_space_accessor.compact_to_u128(val as u32);
- let val = Ipv6Addr::from_u128(val);
- dict.insert(IntermediateKey::IpAddr(val), intermediate_entry);
- }
- } else {
- for (val, doc_count) in entries {
- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;
- let val = f64_from_fastfield_u64(val, &self.column_type);
- dict.insert(IntermediateKey::F64(val), intermediate_entry);
- }
- };
-
- Ok(IntermediateBucketResult::Terms {
- buckets: IntermediateTermBucketResult {
- entries: dict,
- sum_other_doc_count,
- doc_count_error_upper_bound: term_doc_count_before_cutoff,
- },
- })
- }
-}
-
-pub(crate) trait GetDocCount {
- fn doc_count(&self) -> u64;
-}
-impl GetDocCount for (u64, u32) {
- fn doc_count(&self) -> u64 {
- self.1 as u64
- }
-}
-impl GetDocCount for (String, IntermediateTermBucketEntry) {
- fn doc_count(&self) -> u64 {
- self.1.doc_count as u64
- }
-}
-
-pub(crate) fn cut_off_buckets<T: GetDocCount + Debug>(
- entries: &mut Vec<T>,
- num_elem: usize,
-) -> (u64, u64) {
- let term_doc_count_before_cutoff = entries
- .get(num_elem)
- .map(|entry| entry.doc_count())
- .unwrap_or(0);
-
- let sum_other_doc_count = entries
- .get(num_elem..)
- .map(|cut_off_range| cut_off_range.iter().map(|entry| entry.doc_count()).sum())
- .unwrap_or(0);
-
- entries.truncate(num_elem);
- (term_doc_count_before_cutoff, sum_other_doc_count)
-}
-
-#[cfg(test)]
-mod tests {
- use std::net::IpAddr;
- use std::str::FromStr;
-
- use crate::common::DateTime;
- use time::{Date, Month};
-
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::tests::{
- exec_request, exec_request_with_query, exec_request_with_query_and_memory_limit,
- get_test_index_from_terms, get_test_index_from_values_and_terms,
- };
- use crate::aggregation::AggregationLimits;
- use crate::indexer::NoMergePolicy;
- use crate::schema::{IntoIpv6Addr, Schema, FAST, STRING};
- use crate::{Index, IndexWriter};
-
- #[test]
- fn terms_aggregation_test_single_segment() -> crate::Result<()> {
- terms_aggregation_test_merge_segment(true)
- }
- #[test]
- fn terms_aggregation_test() -> crate::Result<()> {
- terms_aggregation_test_merge_segment(false)
- }
- fn terms_aggregation_test_merge_segment(merge_segments: bool) -> crate::Result<()> {
- let segment_and_terms = vec![
- vec!["terma"],
- vec!["termb"],
- vec!["termc"],
- vec!["terma"],
- vec!["terma"],
- vec!["terma"],
- vec!["termb"],
- vec!["terma"],
- ];
- let index = get_test_index_from_terms(merge_segments, &segment_and_terms)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 1);
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "size": 2,
- "segment_size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 1);
-
- // test min_doc_count
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "size": 2,
- "min_doc_count": 3,
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(
- res["my_texts"]["buckets"][1]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0); // TODO sum_other_doc_count with min_doc_count
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_test_order_count_single_segment() -> crate::Result<()> {
- terms_aggregation_test_order_count_merge_segment(true)
- }
- #[test]
- fn terms_aggregation_test_count_order() -> crate::Result<()> {
- terms_aggregation_test_order_count_merge_segment(false)
- }
- fn terms_aggregation_test_order_count_merge_segment(merge_segments: bool) -> crate::Result<()> {
- let segment_and_terms = vec![
- vec![(5.0, "terma".to_string())],
- vec![(2.0, "termb".to_string())],
- vec![(2.0, "terma".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(8.0, "termb".to_string())],
- vec![(5.0, "terma".to_string())],
- ];
- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;
-
- let sub_agg: Aggregations = serde_json::from_value(json!({
- "avg_score": {
- "avg": {
- "field": "score",
- }
- },
- "stats_score": {
- "stats": {
- "field": "score",
- }
- }
- }))
- .unwrap();
-
- // sub agg desc
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_count": "asc",
- },
- },
- "aggs": sub_agg,
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 5.0);
-
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 6);
- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 4.5);
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // Agg on non string
- //
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_scores1": {
- "terms": {
- "field": "score",
- "order": {
- "_count": "asc",
- },
- },
- "aggs": sub_agg,
- },
- "my_scores2": {
- "terms": {
- "field": "score_f64",
- "order": {
- "_count": "asc",
- },
- },
- "aggs": sub_agg,
- },
- "my_scores3": {
- "terms": {
- "field": "score_i64",
- "order": {
- "_count": "asc",
- },
- },
- "aggs": sub_agg,
- }
-
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_scores1"]["buckets"][0]["key"], 8.0);
- assert_eq!(res["my_scores1"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_scores1"]["buckets"][0]["avg_score"]["value"], 8.0);
-
- assert_eq!(res["my_scores1"]["buckets"][1]["key"], 2.0);
- assert_eq!(res["my_scores1"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_scores1"]["buckets"][1]["avg_score"]["value"], 2.0);
-
- assert_eq!(res["my_scores1"]["buckets"][2]["key"], 1.0);
- assert_eq!(res["my_scores1"]["buckets"][2]["doc_count"], 3);
- assert_eq!(res["my_scores1"]["buckets"][2]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_scores1"]["buckets"][3]["key"], 5.0);
- assert_eq!(res["my_scores1"]["buckets"][3]["doc_count"], 5);
- assert_eq!(res["my_scores1"]["buckets"][3]["avg_score"]["value"], 5.0);
-
- assert_eq!(res["my_scores1"]["sum_other_doc_count"], 0);
-
- assert_eq!(res["my_scores2"]["buckets"][0]["key"], 8.0);
- assert_eq!(res["my_scores2"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_scores2"]["buckets"][0]["avg_score"]["value"], 8.0);
-
- assert_eq!(res["my_scores2"]["buckets"][1]["key"], 2.0);
- assert_eq!(res["my_scores2"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_scores2"]["buckets"][1]["avg_score"]["value"], 2.0);
-
- assert_eq!(res["my_scores2"]["buckets"][2]["key"], 1.0);
- assert_eq!(res["my_scores2"]["buckets"][2]["doc_count"], 3);
- assert_eq!(res["my_scores2"]["buckets"][2]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_scores2"]["sum_other_doc_count"], 0);
-
- assert_eq!(res["my_scores3"]["buckets"][0]["key"], 8.0);
- assert_eq!(res["my_scores3"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_scores3"]["buckets"][0]["avg_score"]["value"], 8.0);
-
- assert_eq!(res["my_scores3"]["buckets"][1]["key"], 2.0);
- assert_eq!(res["my_scores3"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_scores3"]["buckets"][1]["avg_score"]["value"], 2.0);
-
- assert_eq!(res["my_scores3"]["buckets"][2]["key"], 1.0);
- assert_eq!(res["my_scores3"]["buckets"][2]["doc_count"], 3);
- assert_eq!(res["my_scores3"]["buckets"][2]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_scores3"]["sum_other_doc_count"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_test_order_sub_agg_single_segment() -> crate::Result<()> {
- terms_aggregation_test_order_sub_agg_merge_segment(true)
- }
- #[test]
- fn terms_aggregation_test_sub_agg_order() -> crate::Result<()> {
- terms_aggregation_test_order_sub_agg_merge_segment(false)
- }
- fn terms_aggregation_test_order_sub_agg_merge_segment(
- merge_segments: bool,
- ) -> crate::Result<()> {
- let segment_and_terms = vec![
- vec![(5.0, "terma".to_string())],
- vec![(4.0, "termb".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(8.0, "termb".to_string())],
- vec![(5.0, "terma".to_string())],
- ];
- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;
-
- let sub_agg: Aggregations = serde_json::from_value(json!({
- "avg_score": {
- "avg": {
- "field": "score",
- }
- },
- "stats_score": {
- "stats": {
- "field": "score",
- }
- }
- }))
- .unwrap();
-
- // sub agg desc
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "avg_score": "desc"
- }
- },
- "aggs": sub_agg,
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 6.0);
-
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);
-
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // sub agg asc
- //
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "avg_score": "asc"
- }
- },
- "aggs": sub_agg,
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);
-
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 6.0);
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // sub agg multi value asc
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "stats_score.avg": "asc"
- }
- },
- "aggs": sub_agg,
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 1.0);
-
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);
-
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 6.0);
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // sub agg invalid request
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "doesnotexist": "asc"
- }
- },
- "aggs": sub_agg,
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index);
- assert!(res.is_err());
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_test_order_key_single_segment() -> crate::Result<()> {
- terms_aggregation_test_order_key_merge_segment(true)
- }
- #[test]
- fn terms_aggregation_test_key_order() -> crate::Result<()> {
- terms_aggregation_test_order_key_merge_segment(false)
- }
- fn terms_aggregation_test_order_key_merge_segment(merge_segments: bool) -> crate::Result<()> {
- let segment_and_terms = vec![
- vec![(5.0, "terma".to_string())],
- vec![(4.0, "termb".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(1.0, "termc".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(5.0, "terma".to_string())],
- vec![(8.0, "termb".to_string())],
- vec![(5.0, "terma".to_string())],
- ];
- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;
-
- // key asc
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "asc"
- }
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 3);
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // key desc and size cut_off
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "asc"
- },
- "size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["doc_count"],
- serde_json::Value::Null
- );
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 3);
-
- // key asc and segment_size cut_off
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "asc"
- },
- "size": 2,
- "segment_size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["doc_count"],
- serde_json::Value::Null
- );
-
- // key desc
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "desc"
- },
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 5);
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
-
- // key desc, size cut_off
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "desc"
- },
- "size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["doc_count"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 5);
-
- // key desc, segment_size cut_off
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "order": {
- "_key": "desc"
- },
- "size": 2,
- "segment_size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["doc_count"],
- serde_json::Value::Null
- );
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_min_doc_count_special_case() -> crate::Result<()> {
- let terms_per_segment = vec![
- vec!["terma", "terma", "termb", "termb", "termb", "termc"],
- vec!["terma", "terma", "termb", "termc", "termc"],
- ];
-
- let index = get_test_index_from_terms(false, &terms_per_segment)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "min_doc_count": 0,
- },
- }
- }))
- .unwrap();
-
- // searching for terma, but min_doc_count will return all terms
- let res = exec_request_with_query(agg_req, &index, Some(("string_id", "terma")))?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 0);
- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");
- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 0);
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_min_doc_count_special_case_with_sub_agg_empty_merge() -> crate::Result<()>
- {
- let mut schema_builder = Schema::builder();
- let string_field_1 = schema_builder.add_text_field("string1", STRING | FAST);
- let string_field_2 = schema_builder.add_text_field("string2", STRING | FAST);
- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();
- let score_field = schema_builder.add_u64_field("score", score_fieldtype);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- // writing the segment
- index_writer.add_document(doc!(
- string_field_1 => "A".to_string(),
- string_field_2 => "hit".to_string(),
- score_field => 1u64,
- ))?;
- index_writer.add_document(doc!(
- string_field_1 => "B".to_string(),
- string_field_2 => "nohit".to_string(), // this doc gets filtered in this segment,
- // but the term will still be loaded because
- // min_doc_count == 0
- score_field => 2u64,
- ))?;
- index_writer.commit()?;
-
- index_writer.add_document(doc!(
- string_field_1 => "A".to_string(),
- string_field_2 => "hit".to_string(),
- score_field => 2u64,
- ))?;
- index_writer.add_document(doc!(
- string_field_1 => "B".to_string(),
- string_field_2 => "hit".to_string(),
- score_field => 4u64,
- ))?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string1",
- "min_doc_count": 0,
- },
- "aggs":{
- "elhistogram": {
- "histogram": {
- "field": "score",
- "interval": 1
- }
- }
- }
- }
- }))
- .unwrap();
-
- // searching for terma, but min_doc_count will return all terms
- let res = exec_request_with_query(agg_req, &index, Some(("string2", "hit")))?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "A");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][0]["elhistogram"]["buckets"],
- json!([{ "doc_count": 1, "key": 1.0 }, { "doc_count": 1, "key": 2.0 } ])
- );
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "B");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);
- assert_eq!(
- res["my_texts"]["buckets"][1]["elhistogram"]["buckets"],
- json!([ { "doc_count": 1, "key": 4.0 } ])
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_error_count_test() -> crate::Result<()> {
- let terms_per_segment = vec![
- vec!["terma", "terma", "termb", "termb", "termb", "termc"], /* termc doesn't make it
- * from this segment */
- vec!["terma", "terma", "termb", "termc", "termc"], /* termb doesn't make it from
- * this segment */
- ];
-
- let index = get_test_index_from_terms(false, &terms_per_segment)?;
- assert_eq!(index.searchable_segments().unwrap().len(), 2);
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "size": 2,
- "segment_size": 2
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 3);
- assert_eq!(
- res["my_texts"]["buckets"][2]["doc_count"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 4);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 2);
-
- // disable doc_count_error_upper_bound
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "size": 2,
- "segment_size": 2,
- "show_term_doc_count_error": false
- },
- }
- }))
- .unwrap();
-
- let res = exec_request(agg_req, &index)?;
-
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 4);
- assert_eq!(
- res["my_texts"]["doc_count_error_upper_bound"],
- serde_json::Value::Null
- );
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_term_bucket_limit() -> crate::Result<()> {
- let terms: Vec<String> = (0..20_000).map(|el| el.to_string()).collect();
- let terms_per_segment = vec![terms.iter().map(|el| el.as_str()).collect()];
-
- let index = get_test_index_from_terms(true, &terms_per_segment)?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "string_id",
- "min_doc_count": 0,
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query_and_memory_limit(
- agg_req,
- &index,
- None,
- AggregationLimits::new(Some(50_000), None),
- )
- .unwrap_err();
- assert!(res
- .to_string()
- .contains("Aborting aggregation because memory limit was exceeded. Limit: 50.00 KB"));
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_different_tokenizer_on_ff_test() -> crate::Result<()> {
- let terms = vec!["Hello Hello", "Hallo Hallo", "Hallo Hallo"];
-
- let index = get_test_index_from_terms(true, &[terms])?;
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "text_id",
- "min_doc_count": 0,
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None).unwrap();
- println!("{}", serde_json::to_string_pretty(&res).unwrap());
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hallo Hallo");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);
-
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Hello Hello");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);
-
- Ok(())
- }
-
- #[test]
- fn test_json_format() -> crate::Result<()> {
- let agg_req: Aggregations = serde_json::from_value(json!({
- "term_agg_test": {
- "terms": {
- "field": "string_id",
- "size": 2,
- "segment_size": 2,
- "order": {
- "_key": "desc"
- }
- },
- }
- }))
- .unwrap();
-
- let elasticsearch_compatible_json = json!(
- {
- "term_agg_test":{
- "terms": {
- "field": "string_id",
- "size": 2u64,
- "segment_size": 2u64,
- "order": {"_key": "desc"}
- }
- }
- });
-
- let agg_req_deser: Aggregations =
- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())
- .unwrap();
- assert_eq!(agg_req, agg_req_deser);
-
- let elasticsearch_compatible_json = json!(
- {
- "term_agg_test":{
- "terms": {
- "field": "string_id",
- "split_size": 2u64,
- }
- }
- });
-
- // test alias shard_size, split_size
- let agg_req: Aggregations = serde_json::from_value(json!({
- "term_agg_test": {
- "terms": {
- "field": "string_id",
- "split_size": 2,
- },
- }
- }))
- .unwrap();
-
- let agg_req_deser: Aggregations =
- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())
- .unwrap();
- assert_eq!(agg_req, agg_req_deser);
-
- let elasticsearch_compatible_json = json!(
- {
- "term_agg_test":{
- "terms": {
- "field": "string_id",
- "shard_size": 2u64,
- }
- }
- });
-
- let agg_req_deser: Aggregations =
- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())
- .unwrap();
- assert_eq!(agg_req, agg_req_deser);
-
- Ok(())
- }
- #[test]
- fn terms_empty_json() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with empty json
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
- // => Segment with json, but no field partially_empty
- index_writer
- .add_document(doc!(json => json!({"different_field": "blue"})))
- .unwrap();
- index_writer.commit().unwrap();
- //// => Segment with field partially_empty
- index_writer
- .add_document(doc!(json => json!({"partially_empty": "blue"})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "json.partially_empty"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "blue");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_texts"]["buckets"][1], serde_json::Value::Null);
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_bytes() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let bytes_field = schema_builder.add_bytes_field("bytes", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer.add_document(doc!(
- bytes_field => vec![1,2,3],
- ))?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "bytes"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // TODO: Returning an error would be better instead of an empty result, since this is not a
- // JSON field
- assert_eq!(
- res["my_texts"]["buckets"][0]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_multi_value() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", FAST);
- let id_field = schema_builder.add_u64_field("id", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- text_field => "Hello Hello",
- id_field => 1u64,
- id_field => 1u64,
- ))?;
- // Missing
- index_writer.add_document(doc!())?;
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- ))?;
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- ))?;
- index_writer.commit()?;
- // Empty segment special case
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
- // Full segment special case
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- id_field => 1u64,
- ))?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "text",
- "missing": "Empty"
- },
- },
- "my_texts2": {
- "terms": {
- "field": "text",
- "missing": 1337
- },
- },
- "my_ids": {
- "terms": {
- "field": "id",
- "missing": 1337
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hello Hello");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Empty");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["key"],
- serde_json::Value::Null
- );
- // text field with numner as missing fallback
- assert_eq!(res["my_texts2"]["buckets"][0]["key"], "Hello Hello");
- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 5);
- assert_eq!(res["my_texts2"]["buckets"][1]["key"], 1337.0);
- assert_eq!(res["my_texts2"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts2"]["buckets"][2]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- // id field
- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);
- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 3);
- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
- #[test]
- fn terms_aggregation_missing_simple_id() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let id_field = schema_builder.add_u64_field("id", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer.add_document(doc!(
- id_field => 1u64,
- ))?;
- // Missing
- index_writer.add_document(doc!())?;
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_ids": {
- "terms": {
- "field": "id",
- "missing": 1337
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // id field
- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);
- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 1);
- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing1() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("text", FAST);
- let id_field = schema_builder.add_u64_field("id", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- id_field => 1u64,
- ))?;
- // Missing
- index_writer.add_document(doc!())?;
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- ))?;
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- ))?;
- index_writer.commit()?;
- // Empty segment special case
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
- // Full segment special case
- index_writer.add_document(doc!(
- text_field => "Hello Hello",
- id_field => 1u64,
- ))?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "text",
- "missing": "Empty"
- },
- },
- "my_texts2": {
- "terms": {
- "field": "text",
- "missing": 1337
- },
- },
- "my_ids": {
- "terms": {
- "field": "id",
- "missing": 1337
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hello Hello");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Empty");
- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts"]["buckets"][2]["key"],
- serde_json::Value::Null
- );
- // text field with numner as missing fallback
- assert_eq!(res["my_texts2"]["buckets"][0]["key"], "Hello Hello");
- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_texts2"]["buckets"][1]["key"], 1337.0);
- assert_eq!(res["my_texts2"]["buckets"][1]["doc_count"], 2);
- assert_eq!(
- res["my_texts2"]["buckets"][2]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- // id field
- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 4);
- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);
- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 2);
- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
- #[test]
- fn terms_aggregation_missing_empty() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- schema_builder.add_text_field("text", FAST);
- schema_builder.add_u64_field("id", FAST);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- // Empty segment special case
- index_writer.add_document(doc!())?;
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_texts": {
- "terms": {
- "field": "text",
- "missing": "Empty"
- },
- },
- "my_texts2": {
- "terms": {
- "field": "text",
- "missing": 1337
- },
- },
- "my_ids": {
- "terms": {
- "field": "id",
- "missing": 1337
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Empty");
- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);
- assert_eq!(
- res["my_texts"]["buckets"][1]["key"],
- serde_json::Value::Null
- );
- // text field with number as missing fallback
- assert_eq!(res["my_texts2"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 1);
- assert_eq!(
- res["my_texts2"]["buckets"][1]["key"],
- serde_json::Value::Null
- );
- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);
- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);
-
- // id field
- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["my_ids"]["buckets"][1]["key"], serde_json::Value::Null);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_date() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let date_field = schema_builder.add_date_field("date_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- {
- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;
- writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_date": {
- "terms": {
- "field": "date_field"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // date_field field
- assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");
- assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");
- assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);
- assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
- #[test]
- fn terms_aggregation_date_missing() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let date_field = schema_builder.add_date_field("date_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- {
- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;
- writer.add_document(doc!())?;
- writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_date": {
- "terms": {
- "field": "date_field",
- "missing": "1982-09-17T00:00:00Z"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // date_field field
- assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");
- assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");
- assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);
- assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_bool() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let field = schema_builder.add_bool_field("bool_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- {
- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
- writer.add_document(doc!(field=>true))?;
- writer.add_document(doc!(field=>false))?;
- writer.add_document(doc!(field=>true))?;
- writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_bool": {
- "terms": {
- "field": "bool_field"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(res["my_bool"]["buckets"][0]["key"], 1.0);
- assert_eq!(res["my_bool"]["buckets"][0]["key_as_string"], "true");
- assert_eq!(res["my_bool"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_bool"]["buckets"][1]["key"], 0.0);
- assert_eq!(res["my_bool"]["buckets"][1]["key_as_string"], "false");
- assert_eq!(res["my_bool"]["buckets"][1]["doc_count"], 1);
- assert_eq!(res["my_bool"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_ip_addr() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let field = schema_builder.add_ip_addr_field("ip_field", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- {
- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;
- // IpV6 loopback
- writer.add_document(doc!(field=>IpAddr::from_str("::1").unwrap().into_ipv6_addr()))?;
- writer.add_document(doc!(field=>IpAddr::from_str("::1").unwrap().into_ipv6_addr()))?;
- // IpV4
- writer.add_document(
- doc!(field=>IpAddr::from_str("127.0.0.1").unwrap().into_ipv6_addr()),
- )?;
- writer.commit()?;
- }
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_bool": {
- "terms": {
- "field": "ip_field"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
- // print as json
- // println!("{}", serde_json::to_string_pretty(&res).unwrap());
-
- assert_eq!(res["my_bool"]["buckets"][0]["key"], "::1");
- assert_eq!(res["my_bool"]["buckets"][0]["doc_count"], 2);
- assert_eq!(res["my_bool"]["buckets"][1]["key"], "127.0.0.1");
- assert_eq!(res["my_bool"]["buckets"][1]["doc_count"], 1);
- assert_eq!(res["my_bool"]["buckets"][2]["key"], serde_json::Value::Null);
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/bucket/term_missing_agg.rs b/crates/tantivy/src/aggregation/bucket/term_missing_agg.rs
deleted file mode 100644
index bb8b295b..00000000
--- a/crates/tantivy/src/aggregation/bucket/term_missing_agg.rs
+++ /dev/null
@@ -1,481 +0,0 @@
-use rustc_hash::FxHashMap;
-
-use crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor;
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,
- IntermediateKey, IntermediateTermBucketEntry, IntermediateTermBucketResult,
-};
-use crate::aggregation::segment_agg_result::{
- build_segment_agg_collector, SegmentAggregationCollector,
-};
-
-/// The specialized missing term aggregation.
-#[derive(Default, Debug, Clone)]
-pub struct TermMissingAgg {
- missing_count: u32,
- accessor_idx: usize,
- sub_agg: Option<Box<dyn SegmentAggregationCollector>>,
-}
-impl TermMissingAgg {
- pub(crate) fn new(
- accessor_idx: usize,
- sub_aggregations: &mut AggregationsWithAccessor,
- ) -> crate::Result<Self> {
- let has_sub_aggregations = !sub_aggregations.is_empty();
- let sub_agg = if has_sub_aggregations {
- let sub_aggregation = build_segment_agg_collector(sub_aggregations)?;
- Some(sub_aggregation)
- } else {
- None
- };
-
- Ok(Self {
- accessor_idx,
- sub_agg,
- ..Default::default()
- })
- }
-}
-
-impl SegmentAggregationCollector for TermMissingAgg {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];
- let term_agg = agg_with_accessor
- .agg
- .agg
- .as_term()
- .expect("TermMissingAgg collector must be term agg req");
- let missing = term_agg
- .missing
- .as_ref()
- .expect("TermMissingAgg collector, but no missing found in agg req")
- .clone();
- let mut entries: FxHashMap<IntermediateKey, IntermediateTermBucketEntry> =
- Default::default();
-
- let mut missing_entry = IntermediateTermBucketEntry {
- doc_count: self.missing_count,
- sub_aggregation: Default::default(),
- };
- if let Some(sub_agg) = self.sub_agg {
- let mut res = IntermediateAggregationResults::default();
- sub_agg.add_intermediate_aggregation_result(
- &agg_with_accessor.sub_aggregation,
- &mut res,
- )?;
- missing_entry.sub_aggregation = res;
- }
-
- entries.insert(missing.into(), missing_entry);
-
- let bucket = IntermediateBucketResult::Terms {
- buckets: IntermediateTermBucketResult {
- entries,
- sum_other_doc_count: 0,
- doc_count_error_upper_bound: 0,
- },
- };
-
- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;
-
- Ok(())
- }
-
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let agg = &mut agg_with_accessor.aggs.values[self.accessor_idx];
- let has_value = agg
- .accessors
- .iter()
- .any(|(acc, _)| acc.index.has_value(doc));
- if !has_value {
- self.missing_count += 1;
- if let Some(sub_agg) = self.sub_agg.as_mut() {
- sub_agg.collect(doc, &mut agg.sub_aggregation)?;
- }
- }
- Ok(())
- }
-
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- for doc in docs {
- self.collect(*doc, agg_with_accessor)?;
- }
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::tests::exec_request_with_query;
- use crate::schema::{Schema, FAST};
- use crate::{Index, IndexWriter};
-
- #[test]
- fn terms_aggregation_missing_mixed_type_mult_seg_sub_agg() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let score = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with all values numeric
- index_writer
- .add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))
- .unwrap();
- index_writer.add_document(doc!(score => 5.0))?;
- // index_writer.commit().unwrap();
- //// => Segment with all values text
- index_writer
- .add_document(doc!(score => 1.0, json => json!({"mixed_type": "blue"})))
- .unwrap();
- index_writer.add_document(doc!(score => 5.0))?;
- // index_writer.commit().unwrap();
-
- // => Segment with mixed values
- index_writer.add_document(doc!(json => json!({"mixed_type": "red"})))?;
- index_writer.add_document(doc!(json => json!({"mixed_type": -20.5})))?;
- index_writer.add_document(doc!(json => json!({"mixed_type": true})))?;
- index_writer.add_document(doc!(score => 5.0))?;
-
- index_writer.commit().unwrap();
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- "aggs": {
- "sum_score": {
- "sum": {
- "field": "score"
- }
- }
- }
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(
- res["replace_null"]["buckets"][0]["sum_score"]["value"],
- 15.0
- );
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_mixed_type_sub_agg_reg1() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let score = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with all values numeric
- index_writer.add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))?;
- index_writer.add_document(doc!(score => 5.0))?;
- index_writer.add_document(doc!(score => 5.0))?;
-
- index_writer.commit().unwrap();
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- "aggs": {
- "sum_score": {
- "sum": {
- "field": "score"
- }
- }
- }
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 2);
- assert_eq!(
- res["replace_null"]["buckets"][0]["sum_score"]["value"],
- 10.0
- );
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_mult_seg_empty() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let score = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-
- index_writer.add_document(doc!(score => 5.0))?;
- index_writer.commit().unwrap();
- index_writer.add_document(doc!(score => 5.0))?;
- index_writer.commit().unwrap();
- index_writer.add_document(doc!(score => 5.0))?;
-
- index_writer.commit().unwrap();
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- "aggs": {
- "sum_score": {
- "sum": {
- "field": "score"
- }
- }
- }
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(
- res["replace_null"]["buckets"][0]["sum_score"]["value"],
- 15.0
- );
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_single_seg_empty() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let score = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-
- index_writer.add_document(doc!(score => 5.0))?;
- index_writer.add_document(doc!(score => 5.0))?;
- index_writer.add_document(doc!(score => 5.0))?;
-
- index_writer.commit().unwrap();
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- "aggs": {
- "sum_score": {
- "sum": {
- "field": "score"
- }
- }
- }
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(
- res["replace_null"]["buckets"][0]["sum_score"]["value"],
- 15.0
- );
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_mixed_type_mult_seg() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with all values numeric
- index_writer
- .add_document(doc!(json => json!({"mixed_type": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
- //// => Segment with all values text
- index_writer
- .add_document(doc!(json => json!({"mixed_type": "blue"})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
-
- // => Segment with mixed values
- index_writer
- .add_document(doc!(json => json!({"mixed_type": "red"})))
- .unwrap();
- index_writer
- .add_document(doc!(json => json!({"mixed_type": -20.5})))
- .unwrap();
- index_writer
- .add_document(doc!(json => json!({"mixed_type": true})))
- .unwrap();
- index_writer.add_document(doc!())?;
-
- index_writer.commit().unwrap();
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- },
- "replace_num": {
- "terms": {
- "field": "json.mixed_type",
- "missing": 1337
- },
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["replace_num"]["buckets"][0]["key"], 1337.0);
- assert_eq!(res["replace_num"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_str_on_numeric_field() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with all values numeric
- index_writer
- .add_document(doc!(json => json!({"mixed_type": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.add_document(doc!())?;
-
- index_writer
- .add_document(doc!(json => json!({"mixed_type": -20.5})))
- .unwrap();
- index_writer.add_document(doc!())?;
-
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-
- #[test]
- fn terms_aggregation_missing_mixed_type_one_seg() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with all values numeric
- index_writer
- .add_document(doc!(json => json!({"mixed_type": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- //// => Segment with all values text
- index_writer
- .add_document(doc!(json => json!({"mixed_type": "blue"})))
- .unwrap();
- index_writer.add_document(doc!())?;
-
- // => Segment with mixed values
- index_writer
- .add_document(doc!(json => json!({"mixed_type": "red"})))
- .unwrap();
- index_writer
- .add_document(doc!(json => json!({"mixed_type": -20.5})))
- .unwrap();
- index_writer
- .add_document(doc!(json => json!({"mixed_type": true})))
- .unwrap();
- index_writer.add_document(doc!())?;
-
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "replace_null": {
- "terms": {
- "field": "json.mixed_type",
- "missing": "NULL"
- },
- },
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- // text field
- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");
- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);
- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);
- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/buf_collector.rs b/crates/tantivy/src/aggregation/buf_collector.rs
deleted file mode 100644
index 15be6281..00000000
--- a/crates/tantivy/src/aggregation/buf_collector.rs
+++ /dev/null
@@ -1,83 +0,0 @@
-use super::agg_req_with_accessor::AggregationsWithAccessor;
-use super::intermediate_agg_result::IntermediateAggregationResults;
-use super::segment_agg_result::SegmentAggregationCollector;
-use crate::DocId;
-
-pub(crate) const DOC_BLOCK_SIZE: usize = 64;
-pub(crate) type DocBlock = [DocId; DOC_BLOCK_SIZE];
-
-/// BufAggregationCollector buffers documents before calling collect_block().
-#[derive(Clone)]
-pub(crate) struct BufAggregationCollector {
- pub(crate) collector: Box<dyn SegmentAggregationCollector>,
- staged_docs: DocBlock,
- num_staged_docs: usize,
-}
-
-impl std::fmt::Debug for BufAggregationCollector {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("SegmentAggregationResultsCollector")
- .field("staged_docs", &&self.staged_docs[..self.num_staged_docs])
- .field("num_staged_docs", &self.num_staged_docs)
- .finish()
- }
-}
-
-impl BufAggregationCollector {
- pub fn new(collector: Box<dyn SegmentAggregationCollector>) -> Self {
- Self {
- collector,
- num_staged_docs: 0,
- staged_docs: [0; DOC_BLOCK_SIZE],
- }
- }
-}
-
-impl SegmentAggregationCollector for BufAggregationCollector {
- #[inline]
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- Box::new(self.collector).add_intermediate_aggregation_result(agg_with_accessor, results)
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.staged_docs[self.num_staged_docs] = doc;
- self.num_staged_docs += 1;
- if self.num_staged_docs == self.staged_docs.len() {
- self.collector
- .collect_block(&self.staged_docs[..self.num_staged_docs], agg_with_accessor)?;
- self.num_staged_docs = 0;
- }
- Ok(())
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.collector.collect_block(docs, agg_with_accessor)?;
-
- Ok(())
- }
-
- #[inline]
- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- self.collector
- .collect_block(&self.staged_docs[..self.num_staged_docs], agg_with_accessor)?;
- self.num_staged_docs = 0;
-
- self.collector.flush(agg_with_accessor)?;
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/collector.rs b/crates/tantivy/src/aggregation/collector.rs
deleted file mode 100644
index 2b9ee6f6..00000000
--- a/crates/tantivy/src/aggregation/collector.rs
+++ /dev/null
@@ -1,209 +0,0 @@
-use super::agg_req::Aggregations;
-use super::agg_req_with_accessor::AggregationsWithAccessor;
-use super::agg_result::AggregationResults;
-use super::buf_collector::BufAggregationCollector;
-use super::intermediate_agg_result::IntermediateAggregationResults;
-use super::segment_agg_result::{
- build_segment_agg_collector, AggregationLimits, SegmentAggregationCollector,
-};
-use crate::aggregation::agg_req_with_accessor::get_aggs_with_segment_accessor_and_validate;
-use crate::collector::{Collector, SegmentCollector};
-use crate::index::SegmentReader;
-use crate::{DocId, SegmentOrdinal, TantivyError};
-
-/// The default max bucket count, before the aggregation fails.
-pub const DEFAULT_BUCKET_LIMIT: u32 = 65000;
-
-/// The default memory limit in bytes before the aggregation fails. 500MB
-pub const DEFAULT_MEMORY_LIMIT: u64 = 500_000_000;
-
-/// Collector for aggregations.
-///
-/// The collector collects all aggregations by the underlying aggregation request.
-pub struct AggregationCollector {
- agg: Aggregations,
- limits: AggregationLimits,
-}
-
-impl AggregationCollector {
- /// Create collector from aggregation request.
- ///
- /// Aggregation fails when the limits in `AggregationLimits` is exceeded. (memory limit and
- /// bucket limit)
- pub fn from_aggs(agg: Aggregations, limits: AggregationLimits) -> Self {
- Self { agg, limits }
- }
-}
-
-/// Collector for distributed aggregations.
-///
-/// The collector collects all aggregations by the underlying aggregation request.
-///
-/// # Purpose
-/// AggregationCollector returns `IntermediateAggregationResults` and not the final
-/// `AggregationResults`, so that results from different indices can be merged and then converted
-/// into the final `AggregationResults` via the `into_final_result()` method.
-pub struct DistributedAggregationCollector {
- agg: Aggregations,
- limits: AggregationLimits,
-}
-
-impl DistributedAggregationCollector {
- /// Create collector from aggregation request.
- ///
- /// Aggregation fails when the limits in `AggregationLimits` is exceeded. (memory limit and
- /// bucket limit)
- pub fn from_aggs(agg: Aggregations, limits: AggregationLimits) -> Self {
- Self { agg, limits }
- }
-}
-
-impl Collector for DistributedAggregationCollector {
- type Fruit = IntermediateAggregationResults;
-
- type Child = AggregationSegmentCollector;
-
- fn for_segment(
- &self,
- segment_local_id: crate::SegmentOrdinal,
- reader: &crate::SegmentReader,
- ) -> crate::Result<Self::Child> {
- AggregationSegmentCollector::from_agg_req_and_reader(
- &self.agg,
- reader,
- segment_local_id,
- &self.limits,
- )
- }
-
- fn requires_scoring(&self) -> bool {
- false
- }
-
- fn merge_fruits(
- &self,
- segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,
- ) -> crate::Result<Self::Fruit> {
- merge_fruits(segment_fruits)
- }
-}
-
-impl Collector for AggregationCollector {
- type Fruit = AggregationResults;
-
- type Child = AggregationSegmentCollector;
-
- fn for_segment(
- &self,
- segment_local_id: crate::SegmentOrdinal,
- reader: &crate::SegmentReader,
- ) -> crate::Result<Self::Child> {
- AggregationSegmentCollector::from_agg_req_and_reader(
- &self.agg,
- reader,
- segment_local_id,
- &self.limits,
- )
- }
-
- fn requires_scoring(&self) -> bool {
- false
- }
-
- fn merge_fruits(
- &self,
- segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,
- ) -> crate::Result<Self::Fruit> {
- let res = merge_fruits(segment_fruits)?;
- res.into_final_result(self.agg.clone(), &self.limits)
- }
-}
-
-fn merge_fruits(
- mut segment_fruits: Vec<crate::Result<IntermediateAggregationResults>>,
-) -> crate::Result<IntermediateAggregationResults> {
- if let Some(fruit) = segment_fruits.pop() {
- let mut fruit = fruit?;
- for next_fruit in segment_fruits {
- fruit.merge_fruits(next_fruit?)?;
- }
- Ok(fruit)
- } else {
- Ok(IntermediateAggregationResults::default())
- }
-}
-
-/// `AggregationSegmentCollector` does the aggregation collection on a segment.
-pub struct AggregationSegmentCollector {
- aggs_with_accessor: AggregationsWithAccessor,
- agg_collector: BufAggregationCollector,
- error: Option<TantivyError>,
-}
-
-impl AggregationSegmentCollector {
- /// Creates an `AggregationSegmentCollector from` an [`Aggregations`] request and a segment
- /// reader. Also includes validation, e.g. checking field types and existence.
- pub fn from_agg_req_and_reader(
- agg: &Aggregations,
- reader: &SegmentReader,
- segment_ordinal: SegmentOrdinal,
- limits: &AggregationLimits,
- ) -> crate::Result<Self> {
- let mut aggs_with_accessor =
- get_aggs_with_segment_accessor_and_validate(agg, reader, segment_ordinal, limits)?;
- let result =
- BufAggregationCollector::new(build_segment_agg_collector(&mut aggs_with_accessor)?);
- Ok(AggregationSegmentCollector {
- aggs_with_accessor,
- agg_collector: result,
- error: None,
- })
- }
-}
-
-impl SegmentCollector for AggregationSegmentCollector {
- type Fruit = crate::Result<IntermediateAggregationResults>;
-
- #[inline]
- fn collect(&mut self, doc: DocId, _score: crate::Score) {
- if self.error.is_some() {
- return;
- }
- if let Err(err) = self
- .agg_collector
- .collect(doc, &mut self.aggs_with_accessor)
- {
- self.error = Some(err);
- }
- }
-
- /// The query pushes the documents to the collector via this method.
- ///
- /// Only valid for Collectors that ignore docs
- fn collect_block(&mut self, docs: &[DocId]) {
- if self.error.is_some() {
- return;
- }
- if let Err(err) = self
- .agg_collector
- .collect_block(docs, &mut self.aggs_with_accessor)
- {
- self.error = Some(err);
- }
- }
-
- fn harvest(mut self) -> Self::Fruit {
- if let Some(err) = self.error {
- return Err(err);
- }
- self.agg_collector.flush(&mut self.aggs_with_accessor)?;
-
- let mut sub_aggregation_res = IntermediateAggregationResults::default();
- Box::new(self.agg_collector).add_intermediate_aggregation_result(
- &self.aggs_with_accessor,
- &mut sub_aggregation_res,
- )?;
-
- Ok(sub_aggregation_res)
- }
-}
diff --git a/crates/tantivy/src/aggregation/date.rs b/crates/tantivy/src/aggregation/date.rs
deleted file mode 100644
index 97befe7b..00000000
--- a/crates/tantivy/src/aggregation/date.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-use time::format_description::well_known::Rfc3339;
-use time::OffsetDateTime;
-
-use crate::TantivyError;
-
-pub(crate) fn format_date(val: i64) -> crate::Result<String> {
- let datetime = OffsetDateTime::from_unix_timestamp_nanos(val as i128).map_err(|err| {
- TantivyError::InvalidArgument(format!(
- "Could not convert {val:?} to OffsetDateTime, err {err:?}"
- ))
- })?;
- let key_as_string = datetime
- .format(&Rfc3339)
- .map_err(|_err| TantivyError::InvalidArgument("Could not serialize date".to_string()))?;
- Ok(key_as_string)
-}
diff --git a/crates/tantivy/src/aggregation/error.rs b/crates/tantivy/src/aggregation/error.rs
deleted file mode 100644
index 2034d5de..00000000
--- a/crates/tantivy/src/aggregation/error.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use crate::common::ByteCount;
-
-use super::bucket::DateHistogramParseError;
-
-/// Error that may occur when opening a directory
-#[derive(Debug, Clone, PartialEq, Eq, Error)]
-pub enum AggregationError {
- /// InternalError Aggregation Request
- #[error("InternalError: {0:?}")]
- InternalError(String),
- /// Invalid Aggregation Request
- #[error("InvalidRequest: {0:?}")]
- InvalidRequest(String),
- /// Date histogram parse error
- #[error("Date histogram parse error: {0:?}")]
- DateHistogramParseError(#[from] DateHistogramParseError),
- /// Memory limit exceeded
- #[error(
- "Aborting aggregation because memory limit was exceeded. Limit: {limit:?}, Current: \
- {current:?}"
- )]
- MemoryExceeded {
- /// Memory consumption limit
- limit: ByteCount,
- /// Current memory consumption
- current: ByteCount,
- },
- /// Bucket limit exceeded
- #[error(
- "Aborting aggregation because bucket limit was exceeded. Limit: {limit:?}, Current: \
- {current:?}"
- )]
- BucketLimitExceeded {
- /// Bucket limit
- limit: u32,
- /// Current num buckets
- current: u32,
- },
-}
diff --git a/crates/tantivy/src/aggregation/intermediate_agg_result.rs b/crates/tantivy/src/aggregation/intermediate_agg_result.rs
deleted file mode 100644
index b62852de..00000000
--- a/crates/tantivy/src/aggregation/intermediate_agg_result.rs
+++ /dev/null
@@ -1,930 +0,0 @@
-//! Contains the intermediate aggregation tree, that can be merged.
-//! Intermediate aggregation results can be used to merge results between segments or between
-//! indices.
-
-use std::cmp::Ordering;
-use std::collections::hash_map::Entry;
-use std::hash::Hash;
-use std::net::Ipv6Addr;
-
-use crate::columnar::ColumnType;
-use itertools::Itertools;
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-
-use super::agg_req::{Aggregation, AggregationVariants, Aggregations};
-use super::agg_result::{AggregationResult, BucketResult, MetricResult, RangeBucketEntry};
-use super::bucket::{
- cut_off_buckets, get_agg_name_and_property, intermediate_histogram_buckets_to_final_buckets,
- GetDocCount, Order, OrderTarget, RangeAggregation, TermsAggregation,
-};
-use super::metric::{
- IntermediateAverage, IntermediateCount, IntermediateExtendedStats, IntermediateMax,
- IntermediateMin, IntermediateStats, IntermediateSum, PercentilesCollector, TopHitsTopNComputer,
-};
-use super::segment_agg_result::AggregationLimits;
-use super::{format_date, AggregationError, Key, SerializedKey};
-use crate::aggregation::agg_result::{AggregationResults, BucketEntries, BucketEntry};
-use crate::aggregation::bucket::TermsAggregationInternal;
-use crate::TantivyError;
-
-/// Contains the intermediate aggregation result, which is optimized to be merged with other
-/// intermediate results.
-///
-/// Notice: This struct should not be de/serialized via JSON format.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateAggregationResults {
- pub(crate) aggs_res: FxHashMap<String, IntermediateAggregationResult>,
-}
-
-#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)]
-/// The key to identify a bucket.
-/// This might seem redundant with `Key`, but the point is to have a different
-/// Serialize implementation.
-pub enum IntermediateKey {
- /// Ip Addr key
- IpAddr(Ipv6Addr),
- /// Bool key
- Bool(bool),
- /// String key
- Str(String),
- /// `f64` key
- F64(f64),
-}
-impl From<Key> for IntermediateKey {
- fn from(value: Key) -> Self {
- match value {
- Key::Str(s) => Self::Str(s),
- Key::F64(f) => Self::F64(f),
- }
- }
-}
-impl From<IntermediateKey> for Key {
- fn from(value: IntermediateKey) -> Self {
- match value {
- IntermediateKey::Str(s) => Self::Str(s),
- IntermediateKey::IpAddr(s) => {
- // Prefer to use the IPv4 representation if possible
- if let Some(ip) = s.to_ipv4_mapped() {
- Self::Str(ip.to_string())
- } else {
- Self::Str(s.to_string())
- }
- }
- IntermediateKey::F64(f) => Self::F64(f),
- IntermediateKey::Bool(f) => Self::F64(f as u64 as f64),
- }
- }
-}
-
-impl Eq for IntermediateKey {}
-
-impl std::hash::Hash for IntermediateKey {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- core::mem::discriminant(self).hash(state);
- match self {
- IntermediateKey::Str(text) => text.hash(state),
- IntermediateKey::F64(val) => val.to_bits().hash(state),
- IntermediateKey::Bool(val) => val.hash(state),
- IntermediateKey::IpAddr(val) => val.hash(state),
- }
- }
-}
-
-impl IntermediateAggregationResults {
- /// Add a result
- pub fn push(&mut self, key: String, value: IntermediateAggregationResult) -> crate::Result<()> {
- let entry = self.aggs_res.entry(key);
- match entry {
- Entry::Occupied(mut e) => {
- // In case of term aggregation over different types, we need to merge the results.
- e.get_mut().merge_fruits(value)?;
- }
- Entry::Vacant(e) => {
- e.insert(value);
- }
- }
- Ok(())
- }
-
- /// Convert intermediate result and its aggregation request to the final result.
- pub fn into_final_result(
- self,
- req: Aggregations,
- limits: &AggregationLimits,
- ) -> crate::Result<AggregationResults> {
- let res = self.into_final_result_internal(&req, limits)?;
- let bucket_count = res.get_bucket_count() as u32;
- if bucket_count > limits.get_bucket_limit() {
- return Err(TantivyError::AggregationError(
- AggregationError::BucketLimitExceeded {
- limit: limits.get_bucket_limit(),
- current: bucket_count,
- },
- ));
- }
- Ok(res)
- }
-
- /// Convert intermediate result and its aggregation request to the final result.
- pub(crate) fn into_final_result_internal(
- self,
- req: &Aggregations,
- limits: &AggregationLimits,
- ) -> crate::Result<AggregationResults> {
- let mut results: FxHashMap<String, AggregationResult> = FxHashMap::default();
- for (key, agg_res) in self.aggs_res.into_iter() {
- let req = req.get(key.as_str()).unwrap_or_else(|| {
- panic!(
- "Could not find key {:?} in request keys {:?}. This probably means that \
- add_intermediate_aggregation_result passed the wrong agg object.",
- key,
- req.keys().collect::<Vec<_>>()
- )
- });
- results.insert(key, agg_res.into_final_result(req, limits)?);
- }
- // Handle empty results
- if results.len() != req.len() {
- for (key, req) in req.iter() {
- if !results.contains_key(key) {
- let empty_res = empty_from_req(req);
- results.insert(key.to_string(), empty_res.into_final_result(req, limits)?);
- }
- }
- }
-
- Ok(AggregationResults(results))
- }
-
- pub(crate) fn empty_from_req(req: &Aggregations) -> Self {
- let mut aggs_res: FxHashMap<String, IntermediateAggregationResult> = FxHashMap::default();
- for (key, req) in req.iter() {
- let empty_res = empty_from_req(req);
- aggs_res.insert(key.to_string(), empty_res);
- }
-
- Self { aggs_res }
- }
-
- /// Merge another intermediate aggregation result into this result.
- ///
- /// The order of the values need to be the same on both results. This is ensured when the same
- /// (key values) are present on the underlying `VecWithNames` struct.
- pub fn merge_fruits(&mut self, other: IntermediateAggregationResults) -> crate::Result<()> {
- for (left, right) in self.aggs_res.values_mut().zip(other.aggs_res.into_values()) {
- left.merge_fruits(right)?;
- }
- Ok(())
- }
-}
-
-pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult {
- use AggregationVariants::*;
- match req.agg {
- Terms(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Terms {
- buckets: Default::default(),
- }),
- Range(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(
- Default::default(),
- )),
- Histogram(_) => {
- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {
- buckets: Vec::new(),
- is_date_agg: false,
- })
- }
- DateHistogram(_) => {
- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {
- buckets: Vec::new(),
- is_date_agg: true,
- })
- }
- Average(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Average(
- IntermediateAverage::default(),
- )),
- Count(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Count(
- IntermediateCount::default(),
- )),
- Max(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Max(
- IntermediateMax::default(),
- )),
- Min(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Min(
- IntermediateMin::default(),
- )),
- Stats(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Stats(
- IntermediateStats::default(),
- )),
- ExtendedStats(_) => IntermediateAggregationResult::Metric(
- IntermediateMetricResult::ExtendedStats(IntermediateExtendedStats::default()),
- ),
- Sum(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Sum(
- IntermediateSum::default(),
- )),
- Percentiles(_) => IntermediateAggregationResult::Metric(
- IntermediateMetricResult::Percentiles(PercentilesCollector::default()),
- ),
- TopHits(ref req) => IntermediateAggregationResult::Metric(
- IntermediateMetricResult::TopHits(TopHitsTopNComputer::new(req)),
- ),
- }
-}
-
-/// An aggregation is either a bucket or a metric.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum IntermediateAggregationResult {
- /// Bucket variant
- Bucket(IntermediateBucketResult),
- /// Metric variant
- Metric(IntermediateMetricResult),
-}
-
-impl IntermediateAggregationResult {
- pub(crate) fn into_final_result(
- self,
- req: &Aggregation,
- limits: &AggregationLimits,
- ) -> crate::Result<AggregationResult> {
- let res = match self {
- IntermediateAggregationResult::Bucket(bucket) => {
- AggregationResult::BucketResult(bucket.into_final_bucket_result(req, limits)?)
- }
- IntermediateAggregationResult::Metric(metric) => {
- AggregationResult::MetricResult(metric.into_final_metric_result(req))
- }
- };
- Ok(res)
- }
- fn merge_fruits(&mut self, other: IntermediateAggregationResult) -> crate::Result<()> {
- match (self, other) {
- (
- IntermediateAggregationResult::Bucket(b1),
- IntermediateAggregationResult::Bucket(b2),
- ) => b1.merge_fruits(b2),
- (
- IntermediateAggregationResult::Metric(m1),
- IntermediateAggregationResult::Metric(m2),
- ) => m1.merge_fruits(m2),
- _ => panic!("aggregation result type mismatch (mixed metric and buckets)"),
- }
- }
-}
-
-/// Holds the intermediate data for metric results
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum IntermediateMetricResult {
- /// Intermediate average result.
- Percentiles(PercentilesCollector),
- /// Intermediate average result.
- Average(IntermediateAverage),
- /// Intermediate count result.
- Count(IntermediateCount),
- /// Intermediate max result.
- Max(IntermediateMax),
- /// Intermediate min result.
- Min(IntermediateMin),
- /// Intermediate stats result.
- Stats(IntermediateStats),
- /// Intermediate stats result.
- ExtendedStats(IntermediateExtendedStats),
- /// Intermediate sum result.
- Sum(IntermediateSum),
- /// Intermediate top_hits result
- TopHits(TopHitsTopNComputer),
-}
-
-impl IntermediateMetricResult {
- fn into_final_metric_result(self, req: &Aggregation) -> MetricResult {
- match self {
- IntermediateMetricResult::Average(intermediate_avg) => {
- MetricResult::Average(intermediate_avg.finalize().into())
- }
- IntermediateMetricResult::Count(intermediate_count) => {
- MetricResult::Count(intermediate_count.finalize().into())
- }
- IntermediateMetricResult::Max(intermediate_max) => {
- MetricResult::Max(intermediate_max.finalize().into())
- }
- IntermediateMetricResult::Min(intermediate_min) => {
- MetricResult::Min(intermediate_min.finalize().into())
- }
- IntermediateMetricResult::Stats(intermediate_stats) => {
- MetricResult::Stats(intermediate_stats.finalize())
- }
- IntermediateMetricResult::ExtendedStats(intermediate_stats) => {
- MetricResult::ExtendedStats(intermediate_stats.finalize())
- }
- IntermediateMetricResult::Sum(intermediate_sum) => {
- MetricResult::Sum(intermediate_sum.finalize().into())
- }
- IntermediateMetricResult::Percentiles(percentiles) => MetricResult::Percentiles(
- percentiles
- .into_final_result(req.agg.as_percentile().expect("unexpected metric type")),
- ),
- IntermediateMetricResult::TopHits(top_hits) => {
- MetricResult::TopHits(top_hits.into_final_result())
- }
- }
- }
-
- // TODO: this is our top-of-the-chain fruit merge mech
- fn merge_fruits(&mut self, other: IntermediateMetricResult) -> crate::Result<()> {
- match (self, other) {
- (
- IntermediateMetricResult::Average(avg_left),
- IntermediateMetricResult::Average(avg_right),
- ) => {
- avg_left.merge_fruits(avg_right);
- }
- (
- IntermediateMetricResult::Count(count_left),
- IntermediateMetricResult::Count(count_right),
- ) => {
- count_left.merge_fruits(count_right);
- }
- (IntermediateMetricResult::Max(max_left), IntermediateMetricResult::Max(max_right)) => {
- max_left.merge_fruits(max_right);
- }
- (IntermediateMetricResult::Min(min_left), IntermediateMetricResult::Min(min_right)) => {
- min_left.merge_fruits(min_right);
- }
- (
- IntermediateMetricResult::Stats(stats_left),
- IntermediateMetricResult::Stats(stats_right),
- ) => {
- stats_left.merge_fruits(stats_right);
- }
- (
- IntermediateMetricResult::ExtendedStats(extended_stats_left),
- IntermediateMetricResult::ExtendedStats(extended_stats_right),
- ) => {
- extended_stats_left.merge_fruits(extended_stats_right);
- }
- (IntermediateMetricResult::Sum(sum_left), IntermediateMetricResult::Sum(sum_right)) => {
- sum_left.merge_fruits(sum_right);
- }
- (
- IntermediateMetricResult::Percentiles(left),
- IntermediateMetricResult::Percentiles(right),
- ) => {
- left.merge_fruits(right)?;
- }
- (IntermediateMetricResult::TopHits(left), IntermediateMetricResult::TopHits(right)) => {
- left.merge_fruits(right)?;
- }
- _ => {
- panic!("incompatible fruit types in tree or missing merge_fruits handler");
- }
- }
-
- Ok(())
- }
-}
-
-/// The intermediate bucket results. Internally they can be easily merged via the keys of the
-/// buckets.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub enum IntermediateBucketResult {
- /// This is the range entry for a bucket, which contains a key, count, from, to, and optionally
- /// sub_aggregations.
- Range(IntermediateRangeBucketResult),
- /// This is the histogram entry for a bucket, which contains a key, count, and optionally
- /// sub_aggregations.
- Histogram {
- /// The column_type of the underlying `Column` is DateTime
- is_date_agg: bool,
- /// The histogram buckets
- buckets: Vec<IntermediateHistogramBucketEntry>,
- },
- /// Term aggregation
- Terms {
- /// The term buckets
- buckets: IntermediateTermBucketResult,
- },
-}
-
-impl IntermediateBucketResult {
- pub(crate) fn into_final_bucket_result(
- self,
- req: &Aggregation,
- limits: &AggregationLimits,
- ) -> crate::Result<BucketResult> {
- match self {
- IntermediateBucketResult::Range(range_res) => {
- let mut buckets: Vec<RangeBucketEntry> = range_res
- .buckets
- .into_values()
- .map(|bucket| {
- bucket.into_final_bucket_entry(
- req.sub_aggregation(),
- req.agg
- .as_range()
- .expect("unexpected aggregation, expected histogram aggregation"),
- range_res.column_type,
- limits,
- )
- })
- .collect::<crate::Result<Vec<_>>>()?;
-
- buckets.sort_by(|left, right| {
- left.from
- .unwrap_or(f64::MIN)
- .total_cmp(&right.from.unwrap_or(f64::MIN))
- });
-
- let is_keyed = req
- .agg
- .as_range()
- .expect("unexpected aggregation, expected range aggregation")
- .keyed;
- let buckets = if is_keyed {
- let mut bucket_map =
- FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
- for bucket in buckets {
- bucket_map.insert(bucket.key.to_string(), bucket);
- }
- BucketEntries::HashMap(bucket_map)
- } else {
- BucketEntries::Vec(buckets)
- };
- Ok(BucketResult::Range { buckets })
- }
- IntermediateBucketResult::Histogram {
- is_date_agg,
- buckets,
- } => {
- let histogram_req = &req
- .agg
- .as_histogram()?
- .expect("unexpected aggregation, expected histogram aggregation");
- let buckets = intermediate_histogram_buckets_to_final_buckets(
- buckets,
- is_date_agg,
- histogram_req,
- req.sub_aggregation(),
- limits,
- )?;
-
- let buckets = if histogram_req.keyed {
- let mut bucket_map =
- FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());
- for bucket in buckets {
- bucket_map.insert(bucket.key.to_string(), bucket);
- }
- BucketEntries::HashMap(bucket_map)
- } else {
- BucketEntries::Vec(buckets)
- };
- Ok(BucketResult::Histogram { buckets })
- }
- IntermediateBucketResult::Terms { buckets: terms } => terms.into_final_result(
- req.agg
- .as_term()
- .expect("unexpected aggregation, expected term aggregation"),
- req.sub_aggregation(),
- limits,
- ),
- }
- }
-
- fn merge_fruits(&mut self, other: IntermediateBucketResult) -> crate::Result<()> {
- match (self, other) {
- (
- IntermediateBucketResult::Terms {
- buckets: term_res_left,
- },
- IntermediateBucketResult::Terms {
- buckets: term_res_right,
- },
- ) => {
- merge_maps(&mut term_res_left.entries, term_res_right.entries)?;
- term_res_left.sum_other_doc_count += term_res_right.sum_other_doc_count;
- term_res_left.doc_count_error_upper_bound +=
- term_res_right.doc_count_error_upper_bound;
- }
-
- (
- IntermediateBucketResult::Range(range_res_left),
- IntermediateBucketResult::Range(range_res_right),
- ) => {
- merge_maps(&mut range_res_left.buckets, range_res_right.buckets)?;
- }
- (
- IntermediateBucketResult::Histogram {
- buckets: buckets_left,
- is_date_agg: _,
- },
- IntermediateBucketResult::Histogram {
- buckets: buckets_right,
- is_date_agg: _,
- },
- ) => {
- let buckets: Result<Vec<IntermediateHistogramBucketEntry>, TantivyError> =
- buckets_left
- .drain(..)
- .merge_join_by(buckets_right, |left, right| {
- left.key.partial_cmp(&right.key).unwrap_or(Ordering::Equal)
- })
- .map(|either| match either {
- itertools::EitherOrBoth::Both(mut left, right) => {
- left.merge_fruits(right)?;
- Ok(left)
- }
- itertools::EitherOrBoth::Left(left) => Ok(left),
- itertools::EitherOrBoth::Right(right) => Ok(right),
- })
- .collect::<Result<_, _>>();
-
- *buckets_left = buckets?;
- }
- (IntermediateBucketResult::Range(_), _) => {
- panic!("try merge on different types")
- }
- (IntermediateBucketResult::Histogram { .. }, _) => {
- panic!("try merge on different types")
- }
- (IntermediateBucketResult::Terms { .. }, _) => {
- panic!("try merge on different types")
- }
- }
- Ok(())
- }
-}
-
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// Range aggregation including error counts
-pub struct IntermediateRangeBucketResult {
- pub(crate) buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry>,
- pub(crate) column_type: Option<ColumnType>,
-}
-
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// Term aggregation including error counts
-pub struct IntermediateTermBucketResult {
- pub(crate) entries: FxHashMap<IntermediateKey, IntermediateTermBucketEntry>,
- pub(crate) sum_other_doc_count: u64,
- pub(crate) doc_count_error_upper_bound: u64,
-}
-
-impl IntermediateTermBucketResult {
- pub(crate) fn into_final_result(
- self,
- req: &TermsAggregation,
- sub_aggregation_req: &Aggregations,
- limits: &AggregationLimits,
- ) -> crate::Result<BucketResult> {
- let req = TermsAggregationInternal::from_req(req);
- let mut buckets: Vec<BucketEntry> = self
- .entries
- .into_iter()
- .filter(|bucket| bucket.1.doc_count as u64 >= req.min_doc_count)
- .map(|(key, entry)| {
- let key_as_string = match key {
- IntermediateKey::Bool(key) => {
- let val = if key { "true" } else { "false" };
- Some(val.to_string())
- }
- _ => None,
- };
- Ok(BucketEntry {
- key_as_string,
- key: key.into(),
- doc_count: entry.doc_count as u64,
- sub_aggregation: entry
- .sub_aggregation
- .into_final_result_internal(sub_aggregation_req, limits)?,
- })
- })
- .collect::<crate::Result<_>>()?;
-
- let order = req.order.order;
- match req.order.target {
- OrderTarget::Key => {
- buckets.sort_by(|left, right| {
- if req.order.order == Order::Asc {
- left.key.partial_cmp(&right.key)
- } else {
- right.key.partial_cmp(&left.key)
- }
- .expect("expected type string, which is always sortable")
- });
- }
- OrderTarget::Count => {
- if req.order.order == Order::Desc {
- buckets.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.doc_count()));
- } else {
- buckets.sort_unstable_by_key(|bucket| bucket.doc_count());
- }
- }
- OrderTarget::SubAggregation(name) => {
- let (agg_name, agg_property) = get_agg_name_and_property(&name);
- let mut buckets_with_val = buckets
- .into_iter()
- .map(|bucket| {
- let val = bucket
- .sub_aggregation
- .get_value_from_aggregation(agg_name, agg_property)?
- .unwrap_or(f64::MIN);
- Ok((bucket, val))
- })
- .collect::<crate::Result<Vec<_>>>()?;
-
- buckets_with_val.sort_by(|(_, val1), (_, val2)| match &order {
- Order::Desc => val2.total_cmp(val1),
- Order::Asc => val1.total_cmp(val2),
- });
- buckets = buckets_with_val
- .into_iter()
- .map(|(bucket, _val)| bucket)
- .collect_vec();
- }
- }
-
- // We ignore _term_doc_count_before_cutoff here, because it increases the upperbound error
- // only for terms that didn't make it into the top N.
- //
- // This can be interesting, as a value of quality of the results, but not good to check the
- // actual error count for the returned terms.
- let (_term_doc_count_before_cutoff, sum_other_doc_count) =
- cut_off_buckets(&mut buckets, req.size as usize);
-
- let doc_count_error_upper_bound = if req.show_term_doc_count_error {
- Some(self.doc_count_error_upper_bound)
- } else {
- None
- };
-
- Ok(BucketResult::Terms {
- buckets,
- sum_other_doc_count: self.sum_other_doc_count + sum_other_doc_count,
- doc_count_error_upper_bound,
- })
- }
-}
-
-trait MergeFruits {
- fn merge_fruits(&mut self, other: Self) -> crate::Result<()>;
-}
-
-fn merge_maps<V: MergeFruits + Clone, T: Eq + PartialEq + Hash>(
- entries_left: &mut FxHashMap<T, V>,
- mut entries_right: FxHashMap<T, V>,
-) -> crate::Result<()> {
- for (name, entry_left) in entries_left.iter_mut() {
- if let Some(entry_right) = entries_right.remove(name) {
- entry_left.merge_fruits(entry_right)?;
- }
- }
-
- for (key, res) in entries_right.into_iter() {
- entries_left.entry(key).or_insert(res);
- }
- Ok(())
-}
-
-/// This is the histogram entry for a bucket, which contains a key, count, and optionally
-/// sub_aggregations.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateHistogramBucketEntry {
- /// The unique the bucket is identified.
- pub key: f64,
- /// The number of documents in the bucket.
- pub doc_count: u64,
- /// The sub_aggregation in this bucket.
- pub sub_aggregation: IntermediateAggregationResults,
-}
-
-impl IntermediateHistogramBucketEntry {
- pub(crate) fn into_final_bucket_entry(
- self,
- req: &Aggregations,
- limits: &AggregationLimits,
- ) -> crate::Result<BucketEntry> {
- Ok(BucketEntry {
- key_as_string: None,
- key: Key::F64(self.key),
- doc_count: self.doc_count,
- sub_aggregation: self
- .sub_aggregation
- .into_final_result_internal(req, limits)?,
- })
- }
-}
-
-/// This is the range entry for a bucket, which contains a key, count, and optionally
-/// sub_aggregations.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateRangeBucketEntry {
- /// The unique key the bucket is identified with.
- pub key: IntermediateKey,
- /// The number of documents in the bucket.
- pub doc_count: u64,
- /// The sub_aggregation in this bucket.
- pub sub_aggregation: IntermediateAggregationResults,
- /// The from range of the bucket. Equals `f64::MIN` when `None`.
- pub from: Option<f64>,
- /// The to range of the bucket. Equals `f64::MAX` when `None`.
- pub to: Option<f64>,
-}
-
-impl IntermediateRangeBucketEntry {
- pub(crate) fn into_final_bucket_entry(
- self,
- req: &Aggregations,
- _range_req: &RangeAggregation,
- column_type: Option<ColumnType>,
- limits: &AggregationLimits,
- ) -> crate::Result<RangeBucketEntry> {
- let mut range_bucket_entry = RangeBucketEntry {
- key: self.key.into(),
- doc_count: self.doc_count,
- sub_aggregation: self
- .sub_aggregation
- .into_final_result_internal(req, limits)?,
- to: self.to,
- from: self.from,
- to_as_string: None,
- from_as_string: None,
- };
-
- // If we have a date type on the histogram buckets, we add the `key_as_string` field as
- // rfc339
- if column_type == Some(ColumnType::DateTime) {
- if let Some(val) = range_bucket_entry.to {
- let key_as_string = format_date(val as i64)?;
- range_bucket_entry.to_as_string = Some(key_as_string);
- }
- if let Some(val) = range_bucket_entry.from {
- let key_as_string = format_date(val as i64)?;
- range_bucket_entry.from_as_string = Some(key_as_string);
- }
- }
-
- Ok(range_bucket_entry)
- }
-}
-
-/// This is the term entry for a bucket, which contains a count, and optionally
-/// sub_aggregations.
-#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateTermBucketEntry {
- /// The number of documents in the bucket.
- pub doc_count: u32,
- /// The sub_aggregation in this bucket.
- pub sub_aggregation: IntermediateAggregationResults,
-}
-
-impl MergeFruits for IntermediateTermBucketEntry {
- fn merge_fruits(&mut self, other: IntermediateTermBucketEntry) -> crate::Result<()> {
- self.doc_count += other.doc_count;
- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;
- Ok(())
- }
-}
-
-impl MergeFruits for IntermediateRangeBucketEntry {
- fn merge_fruits(&mut self, other: IntermediateRangeBucketEntry) -> crate::Result<()> {
- self.doc_count += other.doc_count;
- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;
- Ok(())
- }
-}
-
-impl MergeFruits for IntermediateHistogramBucketEntry {
- fn merge_fruits(&mut self, other: IntermediateHistogramBucketEntry) -> crate::Result<()> {
- self.doc_count += other.doc_count;
- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::collections::HashMap;
-
- use pretty_assertions::assert_eq;
-
- use super::*;
-
- fn get_sub_test_tree(data: &[(String, u64)]) -> IntermediateAggregationResults {
- let mut map = HashMap::new();
- let mut buckets = FxHashMap::default();
- for (key, doc_count) in data {
- buckets.insert(
- key.to_string(),
- IntermediateRangeBucketEntry {
- key: IntermediateKey::Str(key.to_string()),
- doc_count: *doc_count,
- sub_aggregation: Default::default(),
- from: None,
- to: None,
- },
- );
- }
- map.insert(
- "my_agg_level2".to_string(),
- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(
- IntermediateRangeBucketResult {
- buckets,
- column_type: None,
- },
- )),
- );
- IntermediateAggregationResults {
- aggs_res: map.into_iter().collect(),
- }
- }
-
- fn get_intermediat_tree_with_ranges(
- data: &[(String, u64, String, u64)],
- ) -> IntermediateAggregationResults {
- let mut map = HashMap::new();
- let mut buckets: FxHashMap<_, _> = Default::default();
- for (key, doc_count, sub_aggregation_key, sub_aggregation_count) in data {
- buckets.insert(
- key.to_string(),
- IntermediateRangeBucketEntry {
- key: IntermediateKey::Str(key.to_string()),
- doc_count: *doc_count,
- from: None,
- to: None,
- sub_aggregation: get_sub_test_tree(&[(
- sub_aggregation_key.to_string(),
- *sub_aggregation_count,
- )]),
- },
- );
- }
- map.insert(
- "my_agg_level1".to_string(),
- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(
- IntermediateRangeBucketResult {
- buckets,
- column_type: None,
- },
- )),
- );
- IntermediateAggregationResults {
- aggs_res: map.into_iter().collect(),
- }
- }
-
- #[test]
- fn test_merge_fruits_tree_1() {
- let mut tree_left = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 50, "1900".to_string(), 25),
- ("blue".to_string(), 30, "1900".to_string(), 30),
- ]);
- let tree_right = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 60, "1900".to_string(), 30),
- ("blue".to_string(), 25, "1900".to_string(), 50),
- ]);
-
- tree_left.merge_fruits(tree_right).unwrap();
-
- let tree_expected = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 110, "1900".to_string(), 55),
- ("blue".to_string(), 55, "1900".to_string(), 80),
- ]);
-
- assert_eq!(tree_left, tree_expected);
- }
-
- #[test]
- fn test_merge_fruits_tree_2() {
- let mut tree_left = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 50, "1900".to_string(), 25),
- ("blue".to_string(), 30, "1900".to_string(), 30),
- ]);
- let tree_right = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 60, "1900".to_string(), 30),
- ("green".to_string(), 25, "1900".to_string(), 50),
- ]);
-
- tree_left.merge_fruits(tree_right).unwrap();
-
- let tree_expected = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 110, "1900".to_string(), 55),
- ("blue".to_string(), 30, "1900".to_string(), 30),
- ("green".to_string(), 25, "1900".to_string(), 50),
- ]);
-
- assert_eq!(tree_left, tree_expected);
- }
-
- #[test]
- fn test_merge_fruits_tree_empty() {
- let mut tree_left = get_intermediat_tree_with_ranges(&[
- ("red".to_string(), 50, "1900".to_string(), 25),
- ("blue".to_string(), 30, "1900".to_string(), 30),
- ]);
-
- let orig = tree_left.clone();
-
- tree_left
- .merge_fruits(IntermediateAggregationResults::default())
- .unwrap();
-
- assert_eq!(tree_left, orig);
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/average.rs b/crates/tantivy/src/aggregation/metric/average.rs
deleted file mode 100644
index e707f2b0..00000000
--- a/crates/tantivy/src/aggregation/metric/average.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::*;
-
-/// A single-value metric aggregation that computes the average of numeric values that are
-/// extracted from the aggregated documents.
-/// See [super::SingleMetricResult] for return value.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "avg": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct AverageAggregation {
- /// The field name to compute the average on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl AverageAggregation {
- /// Creates a new [`AverageAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- Self {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Intermediate result of the average aggregation that can be combined with other intermediate
-/// results.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateAverage {
- stats: IntermediateStats,
-}
-
-impl IntermediateAverage {
- /// Creates a new [`IntermediateAverage`] instance from a [`SegmentStatsCollector`].
- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
- Self {
- stats: collector.stats,
- }
- }
- /// Merges the other intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateAverage) {
- self.stats.merge_fruits(other.stats);
- }
- /// Computes the final average value.
- pub fn finalize(&self) -> Option<f64> {
- self.stats.finalize().avg
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn deserialization_with_missing_test1() {
- let json = r#"{
- "field": "score",
- "missing": "10.0"
- }"#;
- let avg: AverageAggregation = serde_json::from_str(json).unwrap();
- assert_eq!(avg.field, "score");
- assert_eq!(avg.missing, Some(10.0));
- // no dot
- let json = r#"{
- "field": "score",
- "missing": "10"
- }"#;
- let avg: AverageAggregation = serde_json::from_str(json).unwrap();
- assert_eq!(avg.field, "score");
- assert_eq!(avg.missing, Some(10.0));
-
- // from value
- let avg: AverageAggregation = serde_json::from_value(json!({
- "field": "score_f64",
- "missing": 10u64,
- }))
- .unwrap();
- assert_eq!(avg.missing, Some(10.0));
- // from value
- let avg: AverageAggregation = serde_json::from_value(json!({
- "field": "score_f64",
- "missing": 10u32,
- }))
- .unwrap();
- assert_eq!(avg.missing, Some(10.0));
- let avg: AverageAggregation = serde_json::from_value(json!({
- "field": "score_f64",
- "missing": 10i8,
- }))
- .unwrap();
- assert_eq!(avg.missing, Some(10.0));
- }
-
- #[test]
- fn deserialization_with_missing_test_fail() {
- let json = r#"{
- "field": "score",
- "missing": "a"
- }"#;
- let avg: Result<AverageAggregation, _> = serde_json::from_str(json);
- assert!(avg.is_err());
- assert!(avg
- .unwrap_err()
- .to_string()
- .contains("Failed to parse f64 from string: \"a\""));
-
- // Disallow NaN
- let json = r#"{
- "field": "score",
- "missing": "NaN"
- }"#;
- let avg: Result<AverageAggregation, _> = serde_json::from_str(json);
- assert!(avg.is_err());
- assert!(avg.unwrap_err().to_string().contains("NaN"));
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/count.rs b/crates/tantivy/src/aggregation/metric/count.rs
deleted file mode 100644
index ac550a38..00000000
--- a/crates/tantivy/src/aggregation/metric/count.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::*;
-
-/// A single-value metric aggregation that counts the number of values that are
-/// extracted from the aggregated documents.
-/// See [super::SingleMetricResult] for return value.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "value_count": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct CountAggregation {
- /// The field name to compute the count on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl CountAggregation {
- /// Creates a new [`CountAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- Self {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Intermediate result of the count aggregation that can be combined with other intermediate
-/// results.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateCount {
- stats: IntermediateStats,
-}
-
-impl IntermediateCount {
- /// Creates a new [`IntermediateCount`] instance from a [`SegmentStatsCollector`].
- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
- Self {
- stats: collector.stats,
- }
- }
- /// Merges the other intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateCount) {
- self.stats.merge_fruits(other.stats);
- }
- /// Computes the final count value.
- pub fn finalize(&self) -> Option<f64> {
- Some(self.stats.finalize().count as f64)
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/extended_stats.rs b/crates/tantivy/src/aggregation/metric/extended_stats.rs
deleted file mode 100644
index d3cd59d9..00000000
--- a/crates/tantivy/src/aggregation/metric/extended_stats.rs
+++ /dev/null
@@ -1,1181 +0,0 @@
-use std::fmt::Debug;
-use std::mem;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::agg_req_with_accessor::{
- AggregationWithAccessor, AggregationsWithAccessor,
-};
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
-};
-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::*;
-use crate::{DocId, TantivyError};
-
-/// A multi-value metric aggregation that computes a collection of extended statistics
-/// on numeric values that are extracted
-/// from the aggregated documents.
-/// See [`ExtendedStats`] for returned statistics.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "extended_stats": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct ExtendedStatsAggregation {
- /// The field name to compute the stats on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default)]
- pub missing: Option<f64>,
- /// The sigma parameter defines how standard_deviation_bound_are_calculated.
- /// This can be a useful way to visualize variance of your data.
- /// The default value is 2. Examples in JSON format:
- /// { "field": "my_numbers", "sigma": "3.0" }
- #[serde(default)]
- pub sigma: Option<f64>,
-}
-
-impl ExtendedStatsAggregation {
- /// Creates a new [`ExtendedStatsAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- ExtendedStatsAggregation {
- field: field_name,
- missing: None,
- sigma: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Extended stats contains a collection of statistics
-/// they extends stats adding variance, standard deviation
-/// and bound informations
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct ExtendedStats {
- /// The number of documents.
- pub count: u64,
- /// The sum of the fast field values.
- pub sum: f64,
- /// The min value of the fast field values.
- pub min: Option<f64>,
- /// The max value of the fast field values.
- pub max: Option<f64>,
- /// The average of the fast field values. `None` if count equals zero.
- pub avg: Option<f64>,
- /// The sum of squares of the fast field values. `None` if count equals zero.
- pub sum_of_squares: Option<f64>,
- /// The variance of the fast field values. `None` if count is is 0 or 1.
- pub variance: Option<f64>,
- /// The variance population of the fast field values, always equal to variance. `None` if count
- /// is is 0 or 1.
- pub variance_population: Option<f64>,
- /// The variance sampling of the fast field values, always equal to variance. `None` if count
- /// is is 0 or 1.
- pub variance_sampling: Option<f64>,
- /// The standard deviation of the fast field values. `None` if count is is 0 or 1.
- pub std_deviation: Option<f64>,
- /// The standard deviation of the fast field values, always equal to variance. `None` if count
- /// is is 0 or 1.
- pub std_deviation_population: Option<f64>,
- /// The standard deviation sampling of the fast field values. `None`
- /// if count is is 0 or 1.
- pub std_deviation_sampling: Option<f64>,
- /// The standard deviation bounds of the fast field values, always equal to variance. `None`
- /// if count is is 0 or 1.
- pub std_deviation_bounds: Option<StandardDeviationBounds>,
-}
-
-/// A sub struct for ExtendedStat containing deviation bounds
-/// the values depend on sigma and represent
-/// the bounds from the average with a distance of
-/// std_deviation*sigma
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct StandardDeviationBounds {
- /// upper bound -> avg + std_dev*sigma
- pub upper: f64,
- /// lower bound -> avg - std_dev*sigma
- pub lower: f64,
- /// upper bound sampling -> avg + std_dev_sampling*sigma
- pub upper_sampling: f64,
- /// lower bound sampling -> avg - std_dev_sampling*sigma
- pub lower_sampling: f64,
- /// same as upper
- pub upper_population: f64,
- /// same as lower
- pub lower_population: f64,
-}
-
-impl ExtendedStats {
- pub(crate) fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {
- match agg_property {
- "count" => Ok(Some(self.count as f64)),
- "sum" => Ok(Some(self.sum)),
- "min" => Ok(self.min),
- "max" => Ok(self.max),
- "avg" => Ok(self.avg),
- "variance" => Ok(self.variance),
- "variance_sampling" => Ok(self.variance_sampling),
- "variance_population" => Ok(self.variance_population),
- "sum_of_squares" => Ok(self.sum_of_squares),
- "std_deviation" => Ok(self.std_deviation),
- "std_deviation_sampling" => Ok(self.std_deviation_sampling),
- "std_deviation_population" => Ok(self.std_deviation_population),
- "std_deviation_bounds.lower" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.lower)),
- "std_deviation_bounds.lower_population" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.lower_population)),
- "std_deviation_bounds.lower_sampling" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.lower_sampling)),
- "std_deviation_bounds.upper" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.upper)),
- "std_deviation_bounds.upper_population" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.upper_population)),
- "std_deviation_bounds.upper_sampling" => Ok(self
- .std_deviation_bounds
- .as_ref()
- .map(|bounds| bounds.upper_sampling)),
- _ => Err(TantivyError::InvalidArgument(format!(
- "Unknown property {agg_property} on stats metric aggregation"
- ))),
- }
- }
-}
-
-/// Intermediate result of the extended stats aggregation that can be combined with other
-/// intermediate results.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateExtendedStats {
- intermediate_stats: IntermediateStats,
- /// The number of extracted values.
- /// The sum of square values, it's referred as M2 in Welford's online algorithm
- sum_of_squares: f64,
- /// The sum of square values as computed by elastic search
- sum_of_squares_elastic: f64,
- /// The delta for sum of squares as computed by elastic search needed for the Kahan algorithm
- delta_sum_for_squares_elastic: f64,
- /// The mean is an intermediate value need for calculating the variance
- /// as per [Welford's online algorithm](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)
- mean: f64,
- /// The value used for computing standard deviation bounds
- sigma: f64,
-}
-
-impl Default for IntermediateExtendedStats {
- fn default() -> Self {
- Self {
- intermediate_stats: IntermediateStats::default(),
- sum_of_squares: 0.0,
- sum_of_squares_elastic: 0.0,
- delta_sum_for_squares_elastic: 0.0,
- mean: 0.0,
- // The default value is the same of ElasticSearch
- sigma: 2.0,
- }
- }
-}
-
-impl IntermediateExtendedStats {
- /// Creates a new IntermediateExtendedStats using an option
- /// containing the sigma to be used for calculating bound values.
- pub fn with_sigma(sigma: Option<f64>) -> Self {
- Self {
- intermediate_stats: IntermediateStats::default(),
- sum_of_squares: 0.0,
- sum_of_squares_elastic: 0.0,
- delta_sum_for_squares_elastic: 0.0,
- mean: 0.0,
- // The default value is the same of ElasticSearch
- sigma: sigma.unwrap_or(2.0),
- }
- }
- /// Merges the other stats intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateExtendedStats) {
- if other.intermediate_stats.count == 0 {
- return;
- }
- if self.intermediate_stats.count == 0 {
- let _ = mem::replace(self, other);
- return;
- }
- let new_count = self.intermediate_stats.count + other.intermediate_stats.count;
- let delta = other.mean - self.mean;
- self.sum_of_squares += other.sum_of_squares
- + delta
- * delta
- * self.intermediate_stats.count as f64
- * other.intermediate_stats.count as f64
- / new_count as f64;
- self.mean = (self.intermediate_stats.sum + other.intermediate_stats.sum) / new_count as f64;
- self.sum_of_squares_elastic += other.sum_of_squares_elastic;
- self.delta_sum_for_squares_elastic += other.delta_sum_for_squares_elastic;
- self.intermediate_stats
- .merge_fruits(other.intermediate_stats);
- }
-
- /// Computes the final stats value.
- pub fn finalize(&self) -> Box<ExtendedStats> {
- let (min, max, avg, sum_of_squares) = if self.intermediate_stats.count == 0 {
- (None, None, None, None)
- } else {
- (
- Some(self.intermediate_stats.min),
- Some(self.intermediate_stats.max),
- Some(self.mean),
- Some(self.sum_of_squares_elastic),
- )
- };
- let (variance, variance_sampling) = if self.intermediate_stats.count <= 1 {
- (None, None)
- } else {
- (
- Some(self.sum_of_squares / self.intermediate_stats.count as f64),
- Some(self.sum_of_squares / (self.intermediate_stats.count - 1) as f64),
- )
- };
- let std_deviation = variance.map(|v| v.sqrt());
- let std_deviation_sampling = variance_sampling.map(|v| v.sqrt());
- let std_deviation_bounds =
- if let (Some(std_deviation_val), Some(std_deviation_sampling_val)) =
- (std_deviation, std_deviation_sampling)
- {
- let upper = self.mean + std_deviation_val * self.sigma;
- let lower = self.mean - std_deviation_val * self.sigma;
- let upper_sampling = self.mean + std_deviation_sampling_val * self.sigma;
- let lower_sampling = self.mean - std_deviation_sampling_val * self.sigma;
- Some(StandardDeviationBounds {
- upper,
- lower,
- upper_sampling,
- lower_sampling,
- upper_population: upper,
- lower_population: lower,
- })
- } else {
- None
- };
- Box::new(ExtendedStats {
- count: self.intermediate_stats.count,
- sum: self.intermediate_stats.sum,
- min,
- max,
- avg,
- sum_of_squares,
- variance,
- variance_population: variance,
- variance_sampling,
- std_deviation,
- std_deviation_population: std_deviation,
- std_deviation_sampling,
- std_deviation_bounds,
- })
- }
-
- fn update_variance(&mut self, value: f64) {
- let delta = value - self.mean;
- // this is not what the Welford's online algorithm prescribes but
- // using the pseudo code from wikipedia there was a small rounding
- // error (in 15th decimal place) that caused a test
- //(test_aggregation_level1 in agg_test.rs)
- // failure
- self.mean = self.intermediate_stats.sum / self.intermediate_stats.count as f64;
- // self.mean += delta / self.count as f64;
- let delta2 = value - self.mean;
- self.sum_of_squares += delta * delta2;
- }
-
- #[inline]
- fn collect(&mut self, value: f64) {
- self.intermediate_stats.collect(value);
- // kahan algorithm for sum_of_squares_elastic
- let y = value * value - self.delta_sum_for_squares_elastic;
- let t = self.sum_of_squares_elastic + y;
- self.delta_sum_for_squares_elastic = (t - self.sum_of_squares_elastic) - y;
- self.sum_of_squares_elastic = t;
- self.update_variance(value);
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct SegmentExtendedStatsCollector {
- missing: Option<u64>,
- field_type: ColumnType,
- pub(crate) extended_stats: IntermediateExtendedStats,
- pub(crate) accessor_idx: usize,
- val_cache: Vec<u64>,
-}
-
-impl SegmentExtendedStatsCollector {
- pub fn from_req(
- field_type: ColumnType,
- sigma: Option<f64>,
- accessor_idx: usize,
- missing: Option<f64>,
- ) -> Self {
- let missing = missing.and_then(|val| f64_to_fastfield_u64(val, &field_type));
- Self {
- field_type,
- extended_stats: IntermediateExtendedStats::with_sigma(sigma),
- accessor_idx,
- missing,
- val_cache: Default::default(),
- }
- }
- #[inline]
- pub(crate) fn collect_block_with_field(
- &mut self,
- docs: &[DocId],
- agg_accessor: &mut AggregationWithAccessor,
- ) {
- if let Some(missing) = self.missing.as_ref() {
- agg_accessor.column_block_accessor.fetch_block_with_missing(
- docs,
- &agg_accessor.accessor,
- *missing,
- );
- } else {
- agg_accessor
- .column_block_accessor
- .fetch_block(docs, &agg_accessor.accessor);
- }
- for val in agg_accessor.column_block_accessor.iter_vals() {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.extended_stats.collect(val1);
- }
- }
-}
-
-impl SegmentAggregationCollector for SegmentExtendedStatsCollector {
- #[inline]
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- results.push(
- name,
- IntermediateAggregationResult::Metric(IntermediateMetricResult::ExtendedStats(
- self.extended_stats,
- )),
- )?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;
- if let Some(missing) = self.missing {
- let mut has_val = false;
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.extended_stats.collect(val1);
- has_val = true;
- }
- if !has_val {
- self.extended_stats
- .collect(f64_from_fastfield_u64(missing, &self.field_type));
- }
- } else {
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.extended_stats.collect(val1);
- }
- }
-
- Ok(())
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];
- self.collect_block_with_field(docs, field);
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
-
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::metric::IntermediateExtendedStats;
- use crate::aggregation::tests::get_test_index_from_values;
- use crate::aggregation::AggregationCollector;
- use crate::assert_nearly_equals;
- use crate::query::AllQuery;
-
- const EPSILON_FOR_TEST: f64 = 0.000000000002;
-
- #[test]
- fn test_aggregation_extended_stats_no_variance() -> crate::Result<()> {
- let values = vec![1.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "extended_stats": {
- "field": "score_f64",
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "count")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "min")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "max")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "avg")?
- .unwrap(),
- 1.0
- );
-
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_population")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?
- .is_none());
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum_of_squares")?
- .unwrap(),
- 1.0
- );
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "variance_population")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "variance")?
- .is_none());
- assert!(agg_res
- .get_value_from_aggregation("my_stats", "variance_sampling")?
- .is_none());
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_extended_stats() -> crate::Result<()> {
- let values = vec![1.0, 3.0, 4.0, 5.0, 8.0, 10.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "extended_stats": {
- "field": "score_f64",
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- const EXPECTED_VARIANCE: f64 = 9.138888888888888;
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "count")?
- .unwrap(),
- 6.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "min")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "max")?
- .unwrap(),
- 10.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum")?
- .unwrap(),
- 31.0
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "avg")?
- .unwrap(),
- 5.166666666666667,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_population")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?
- .unwrap(),
- 3.311595788538611,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?
- .unwrap(),
- -0.8794523824056837,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?
- .unwrap(),
- -0.8794523824056837,
- 0.00000000000001
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?
- .unwrap(),
- -1.4565249104105549,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?
- .unwrap(),
- 11.212785715739017,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?
- .unwrap(),
- 11.212785715739017,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?
- .unwrap(),
- 11.78985824374389,
- EPSILON_FOR_TEST
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum_of_squares")?
- .unwrap(),
- 215.0
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_population")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_sampling")?
- .unwrap(),
- 10.966666666666663,
- EPSILON_FOR_TEST
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_extended_stats_with_sigma() -> crate::Result<()> {
- let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "extended_stats": {
- "field": "score_f64",
- "sigma": 1.5
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- const EXPECTED_VARIANCE: f64 = 2.9166666666666665;
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "count")?
- .unwrap(),
- 6.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "min")?
- .unwrap(),
- 1.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "max")?
- .unwrap(),
- 6.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum")?
- .unwrap(),
- 21.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "avg")?
- .unwrap(),
- 3.5
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_population")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?
- .unwrap(),
- 1.8708286933869709,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?
- .unwrap(),
- 0.9382623085101005,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?
- .unwrap(),
- 0.9382623085101005,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?
- .unwrap(),
- 0.6937569599195434,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?
- .unwrap(),
- 6.061737691489899,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?
- .unwrap(),
- 6.061737691489899,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?
- .unwrap(),
- 6.3062430400804566,
- EPSILON_FOR_TEST
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum_of_squares")?
- .unwrap(),
- 91.0
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_population")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_sampling")?
- .unwrap(),
- 3.5,
- EPSILON_FOR_TEST
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_extended_stats_with_variance_similar_to_mean() -> crate::Result<()> {
- let values = vec![50.01, 50.02, 50.01, 50.03, 50.01, 50.02];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "extended_stats": {
- "field": "score_f64",
- "sigma": 1.5
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- const EXPECTED_VARIANCE: f64 = 5.5555555555608854e-5;
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "count")?
- .unwrap(),
- 6.0
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "min")?
- .unwrap(),
- 50.01
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "max")?
- .unwrap(),
- 50.03
- );
- assert_eq!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum")?
- .unwrap(),
- 300.1
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "avg")?
- .unwrap(),
- 50.01666666666667,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_population")?
- .unwrap(),
- EXPECTED_VARIANCE.sqrt(),
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?
- .unwrap(),
- 0.008164965809279263,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?
- .unwrap(),
- 50.00548632677917,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?
- .unwrap(),
- 50.00548632677917,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?
- .unwrap(),
- 50.00441921795275,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?
- .unwrap(),
- 50.027847006554175,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?
- .unwrap(),
- 50.027847006554175,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?
- .unwrap(),
- 50.028914115380594,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "sum_of_squares")?
- .unwrap(),
- 15010.002,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_population")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance")?
- .unwrap(),
- EXPECTED_VARIANCE,
- EPSILON_FOR_TEST
- );
- assert_nearly_equals!(
- agg_res
- .get_value_from_aggregation("my_stats", "variance_sampling")?
- .unwrap(),
- 6.666666666670718e-5,
- EPSILON_FOR_TEST
- );
-
- Ok(())
- }
-
- #[test]
- fn extended_stat_zero_value() {
- let intermediate_extend_stats = IntermediateExtendedStats::default();
- let extended_stats = intermediate_extend_stats.finalize();
- assert!(extended_stats.variance.is_none());
- assert!(extended_stats.variance_population.is_none());
- assert!(extended_stats.variance_sampling.is_none());
- assert!(extended_stats.sum_of_squares.is_none());
- assert!(extended_stats.std_deviation.is_none());
- assert!(extended_stats.std_deviation_population.is_none());
- assert!(extended_stats.std_deviation_sampling.is_none());
- assert!(extended_stats.std_deviation_bounds.is_none());
- }
-
- #[test]
- fn extended_stat_one_value() {
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.collect(1.0f64);
- let extended_stats = intermediate_extend_stats.finalize();
- assert!(extended_stats.variance.is_none());
- assert!(extended_stats.variance_population.is_none());
- assert!(extended_stats.variance_sampling.is_none());
- assert!(extended_stats.std_deviation.is_none());
- assert!(extended_stats.std_deviation_population.is_none());
- assert!(extended_stats.std_deviation_sampling.is_none());
- assert!(extended_stats.std_deviation_bounds.is_none());
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(1.0f64, sum_of_squares);
- }
-
- #[test]
- fn extended_stat_multiple_values() {
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.collect(1.0f64);
- intermediate_extend_stats.collect(3.0f64);
- intermediate_extend_stats.collect(4.0f64);
- intermediate_extend_stats.collect(5.0f64);
- intermediate_extend_stats.collect(8.0f64);
- intermediate_extend_stats.collect(10.0f64);
- let extended_stats = intermediate_extend_stats.finalize();
- let variance = extended_stats.variance.unwrap();
- const EXPECTED_VARIANCE: f64 = 9.138888888888888;
- assert_eq!(EXPECTED_VARIANCE, variance);
- let variance_population = extended_stats.variance_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE, variance_population);
- let variance_sampling = extended_stats.variance_sampling.unwrap();
- assert_eq!(10.966666666666665f64, variance_sampling);
- let std_deviation = extended_stats.std_deviation.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);
- let std_deviation_population = extended_stats.std_deviation_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);
- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();
- assert_eq!(10.966666666666665f64.sqrt(), std_deviation_sampling);
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(215.0, sum_of_squares);
- let avg = extended_stats.avg.unwrap();
- assert_eq!(5.166666666666667, avg);
- }
-
- #[test]
- fn merge_empty_with_one_value() {
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();
- intermediate_extend_stats1.collect(1.0f64);
- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);
- let extended_stats = intermediate_extend_stats.finalize();
- assert!(extended_stats.variance.is_none());
- assert!(extended_stats.variance_population.is_none());
- assert!(extended_stats.variance_sampling.is_none());
- assert!(extended_stats.std_deviation.is_none());
- assert!(extended_stats.std_deviation_population.is_none());
- assert!(extended_stats.std_deviation_sampling.is_none());
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(1.0f64, sum_of_squares);
- }
-
- #[test]
- fn merge_empty_with_multiple_values() {
- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();
- intermediate_extend_stats1.collect(1.0f64);
- intermediate_extend_stats1.collect(2.0f64);
- intermediate_extend_stats1.collect(3.0f64);
- intermediate_extend_stats1.collect(4.0f64);
- intermediate_extend_stats1.collect(5.0f64);
-
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);
- let extended_stats = intermediate_extend_stats.finalize();
- const EXPECTED_VARIANCE: f64 = 2.0;
- let variance = extended_stats.variance.unwrap();
- assert_eq!(EXPECTED_VARIANCE, variance);
- let variance_population = extended_stats.variance_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE, variance_population);
- let variance_sampling = extended_stats.variance_sampling.unwrap();
- assert_eq!(2.5f64, variance_sampling);
- let std_deviation = extended_stats.std_deviation.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);
- let std_deviation_population = extended_stats.std_deviation_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);
- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();
- assert_eq!(2.5f64.sqrt(), std_deviation_sampling);
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(55f64, sum_of_squares);
- }
-
- #[test]
- fn merge_non_empty_extended_stats() {
- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();
- intermediate_extend_stats1.collect(3.0f64);
- intermediate_extend_stats1.collect(4.0f64);
- intermediate_extend_stats1.collect(5.0f64);
-
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.collect(1.0f64);
- intermediate_extend_stats.collect(2.0f64);
- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);
- let extended_stats = intermediate_extend_stats.finalize();
-
- let variance = extended_stats.variance.unwrap();
- assert_eq!(2.0f64, variance);
- let variance_population = extended_stats.variance_population.unwrap();
- assert_eq!(2.0f64, variance_population);
- let variance_sampling = extended_stats.variance_sampling.unwrap();
- assert_eq!(2.5f64, variance_sampling);
- let std_deviation = extended_stats.std_deviation.unwrap();
- assert_eq!(2.0f64.sqrt(), std_deviation);
- let std_deviation_population = extended_stats.std_deviation_population.unwrap();
- assert_eq!(2.0f64.sqrt(), std_deviation_population);
- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();
- assert_eq!(2.5f64.sqrt(), std_deviation_sampling);
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(55f64, sum_of_squares);
-
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.collect(1.0f64);
- intermediate_extend_stats.collect(3.0f64);
- intermediate_extend_stats.collect(4.0f64);
- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();
- intermediate_extend_stats1.collect(5.0f64);
- intermediate_extend_stats1.collect(8.0f64);
- intermediate_extend_stats1.collect(10.0f64);
- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);
- let extended_stats = intermediate_extend_stats.finalize();
- const EXPECTED_VARIANCE: f64 = 9.138888888888888;
- let variance = extended_stats.variance.unwrap();
- assert_eq!(EXPECTED_VARIANCE, variance);
- let variance_population = extended_stats.variance_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE, variance_population);
- let variance_sampling = extended_stats.variance_sampling.unwrap();
- assert_eq!(10.966666666666665f64, variance_sampling);
- let std_deviation = extended_stats.std_deviation.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);
- let std_deviation_population = extended_stats.std_deviation_population.unwrap();
- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);
- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();
- assert_eq!(10.966666666666665f64.sqrt(), std_deviation_sampling);
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(215f64, sum_of_squares);
- let avg = extended_stats.avg.unwrap();
- assert_eq!(5.166666666666667, avg);
- }
-
- #[test]
- fn merge_and_then_collect_non_empty_extended_stats() {
- let mut intermediate_extend_stats = IntermediateExtendedStats::default();
- intermediate_extend_stats.collect(1.0f64);
- intermediate_extend_stats.collect(3.0f64);
-
- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();
- intermediate_extend_stats1.collect(5.0f64);
- intermediate_extend_stats1.collect(8.0f64);
- intermediate_extend_stats1.collect(10.0f64);
- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);
- intermediate_extend_stats.collect(4.0f64);
- let extended_stats = intermediate_extend_stats.finalize();
- const EXPECTED_VARIANCE: f64 = 9.138888888888888;
- let variance = extended_stats.variance.unwrap();
- assert_nearly_equals!(EXPECTED_VARIANCE, variance, EPSILON_FOR_TEST);
- let variance_population = extended_stats.variance_population.unwrap();
- assert_nearly_equals!(EXPECTED_VARIANCE, variance_population, EPSILON_FOR_TEST);
- let variance_sampling = extended_stats.variance_sampling.unwrap();
- assert_nearly_equals!(10.966666666666665, variance_sampling, EPSILON_FOR_TEST);
- let std_deviation = extended_stats.std_deviation.unwrap();
- assert_nearly_equals!(EXPECTED_VARIANCE.sqrt(), std_deviation, EPSILON_FOR_TEST);
- let std_deviation_population = extended_stats.std_deviation_population.unwrap();
- assert_nearly_equals!(
- EXPECTED_VARIANCE.sqrt(),
- std_deviation_population,
- EPSILON_FOR_TEST
- );
- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();
- assert_nearly_equals!(
- 10.966666666666665_f64.sqrt(),
- std_deviation_sampling,
- EPSILON_FOR_TEST
- );
- let sum_of_squares = extended_stats.sum_of_squares.unwrap();
- assert_eq!(215.0, sum_of_squares);
- let avg = extended_stats.avg.unwrap();
- assert_eq!(5.166666666666667, avg);
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/max.rs b/crates/tantivy/src/aggregation/metric/max.rs
deleted file mode 100644
index 89c6e445..00000000
--- a/crates/tantivy/src/aggregation/metric/max.rs
+++ /dev/null
@@ -1,120 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::*;
-
-/// A single-value metric aggregation that computes the maximum of numeric values that are
-/// extracted from the aggregated documents.
-/// See [super::SingleMetricResult] for return value.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "max": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct MaxAggregation {
- /// The field name to compute the maximum on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl MaxAggregation {
- /// Creates a new [`MaxAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- Self {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Intermediate result of the maximum aggregation that can be combined with other intermediate
-/// results.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateMax {
- stats: IntermediateStats,
-}
-
-impl IntermediateMax {
- /// Creates a new [`IntermediateMax`] instance from a [`SegmentStatsCollector`].
- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
- Self {
- stats: collector.stats,
- }
- }
- /// Merges the other intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateMax) {
- self.stats.merge_fruits(other.stats);
- }
- /// Computes the final maximum value.
- pub fn finalize(&self) -> Option<f64> {
- self.stats.finalize().max
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::tests::exec_request_with_query;
- use crate::schema::{Schema, FAST};
- use crate::{Index, IndexWriter};
-
- #[test]
- fn test_max_agg_with_missing() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with empty json
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
- // => Segment with json, but no field partially_empty
- index_writer
- .add_document(doc!(json => json!({"different_field": "blue"})))
- .unwrap();
- index_writer.commit().unwrap();
- //// => Segment with field partially_empty
- index_writer
- .add_document(doc!(json => json!({"partially_empty": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "max": {
- "field": "json.partially_empty",
- "missing": 100.0,
- }
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res["my_stats"],
- json!({
- "value": 100.0,
- })
- );
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/min.rs b/crates/tantivy/src/aggregation/metric/min.rs
deleted file mode 100644
index 61fd2ecd..00000000
--- a/crates/tantivy/src/aggregation/metric/min.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::*;
-
-/// A single-value metric aggregation that computes the minimum of numeric values that are
-/// extracted from the aggregated documents.
-/// See [super::SingleMetricResult] for return value.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "min": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct MinAggregation {
- /// The field name to compute the minimum on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl MinAggregation {
- /// Creates a new [`MinAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- Self {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Intermediate result of the minimum aggregation that can be combined with other intermediate
-/// results.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateMin {
- stats: IntermediateStats,
-}
-
-impl IntermediateMin {
- /// Creates a new [`IntermediateMin`] instance from a [`SegmentStatsCollector`].
- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
- Self {
- stats: collector.stats,
- }
- }
- /// Merges the other intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateMin) {
- self.stats.merge_fruits(other.stats);
- }
- /// Computes the final minimum value.
- pub fn finalize(&self) -> Option<f64> {
- self.stats.finalize().min
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/mod.rs b/crates/tantivy/src/aggregation/metric/mod.rs
deleted file mode 100644
index 4c64737b..00000000
--- a/crates/tantivy/src/aggregation/metric/mod.rs
+++ /dev/null
@@ -1,171 +0,0 @@
-//! Module for all metric aggregations.
-//!
-//! The aggregations in this family compute metrics based on values extracted
-//! from the documents that are being aggregated. Values are extracted from the fast field of
-//! the document.
-//! Some aggregations output a single numeric metric (e.g. Average) and are called
-//! single-value numeric metrics aggregation, others generate multiple metrics (e.g. Stats) and are
-//! called multi-value numeric metrics aggregation.
-//!
-//! ## Supported Metric Aggregations
-//! - [Average](AverageAggregation)
-//! - [Stats](StatsAggregation)
-//! - [Min](MinAggregation)
-//! - [Max](MaxAggregation)
-//! - [Sum](SumAggregation)
-//! - [Count](CountAggregation)
-//! - [Percentiles](PercentilesAggregationReq)
-
-mod average;
-mod count;
-mod extended_stats;
-mod max;
-mod min;
-mod percentiles;
-mod stats;
-mod sum;
-mod top_hits;
-
-use std::collections::HashMap;
-
-pub use average::*;
-pub use count::*;
-pub use extended_stats::*;
-pub use max::*;
-pub use min::*;
-pub use percentiles::*;
-use rustc_hash::FxHashMap;
-use serde::{Deserialize, Serialize};
-pub use stats::*;
-pub use sum::*;
-pub use top_hits::*;
-
-use crate::schema::OwnedValue;
-
-/// Single-metric aggregations use this common result structure.
-///
-/// Main reason to wrap it in value is to match elasticsearch output structure.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct SingleMetricResult {
- /// The value of the single value metric.
- pub value: Option<f64>,
-}
-
-impl From<f64> for SingleMetricResult {
- fn from(value: f64) -> Self {
- Self { value: Some(value) }
- }
-}
-
-impl From<Option<f64>> for SingleMetricResult {
- fn from(value: Option<f64>) -> Self {
- Self { value }
- }
-}
-
-/// This is the wrapper of percentile entries, which can be vector or hashmap
-/// depending on if it's keyed or not.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum PercentileValues {
- /// Vector format percentile entries
- Vec(Vec<PercentileValuesVecEntry>),
- /// HashMap format percentile entries. Key is the serialized percentile
- HashMap(FxHashMap<String, f64>),
-}
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-/// The entry when requesting percentiles with keyed: false
-pub struct PercentileValuesVecEntry {
- key: f64,
- value: f64,
-}
-
-/// Single-metric aggregations use this common result structure.
-///
-/// Main reason to wrap it in value is to match elasticsearch output structure.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct PercentilesMetricResult {
- /// The result of the percentile metric.
- pub values: PercentileValues,
-}
-
-/// The top_hits metric results entry
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct TopHitsVecEntry {
- /// The sort values of the document, depending on the sort criteria in the request.
- pub sort: Vec<Option<u64>>,
-
- /// Search results, for queries that include field retrieval requests
- /// (`docvalue_fields`).
- #[serde(rename = "docvalue_fields")]
- #[serde(skip_serializing_if = "HashMap::is_empty")]
- pub doc_value_fields: HashMap<String, OwnedValue>,
-}
-
-/// The top_hits metric aggregation results a list of top hits by sort criteria.
-///
-/// The main reason for wrapping it in `hits` is to match elasticsearch output structure.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct TopHitsMetricResult {
- /// The result of the top_hits metric.
- pub hits: Vec<TopHitsVecEntry>,
-}
-
-#[cfg(test)]
-mod tests {
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::AggregationCollector;
- use crate::query::AllQuery;
- use crate::schema::{NumericOptions, Schema};
- use crate::{Index, IndexWriter};
-
- #[test]
- fn test_metric_aggregations() {
- let mut schema_builder = Schema::builder();
- let field_options = NumericOptions::default().set_fast();
- let field = schema_builder.add_f64_field("price", field_options);
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
-
- for i in 0..3 {
- index_writer
- .add_document(doc!(
- field => i as f64,
- ))
- .unwrap();
- }
- index_writer.commit().unwrap();
-
- for i in 3..6 {
- index_writer
- .add_document(doc!(
- field => i as f64,
- ))
- .unwrap();
- }
- index_writer.commit().unwrap();
-
- let aggregations_json = r#"{
- "price_avg": { "avg": { "field": "price" } },
- "price_count": { "value_count": { "field": "price" } },
- "price_max": { "max": { "field": "price" } },
- "price_min": { "min": { "field": "price" } },
- "price_stats": { "stats": { "field": "price" } },
- "price_sum": { "sum": { "field": "price" } }
- }"#;
- let aggregations: Aggregations = serde_json::from_str(aggregations_json).unwrap();
- let collector = AggregationCollector::from_aggs(aggregations, Default::default());
- let reader = index.reader().unwrap();
- let searcher = reader.searcher();
- let aggregations_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let aggregations_res_json = serde_json::to_value(aggregations_res).unwrap();
-
- assert_eq!(aggregations_res_json["price_avg"]["value"], 2.5);
- assert_eq!(aggregations_res_json["price_count"]["value"], 6.0);
- assert_eq!(aggregations_res_json["price_max"]["value"], 5.0);
- assert_eq!(aggregations_res_json["price_min"]["value"], 0.0);
- assert_eq!(aggregations_res_json["price_sum"]["value"], 15.0);
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/percentiles.rs b/crates/tantivy/src/aggregation/metric/percentiles.rs
deleted file mode 100644
index 4a6bac3f..00000000
--- a/crates/tantivy/src/aggregation/metric/percentiles.rs
+++ /dev/null
@@ -1,691 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::agg_req_with_accessor::{
- AggregationWithAccessor, AggregationsWithAccessor,
-};
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
-};
-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::*;
-use crate::{DocId, TantivyError};
-
-/// # Percentiles
-///
-/// The percentiles aggregation is a useful tool for understanding the distribution
-/// of a data set. It calculates the values below which a given percentage of the
-/// data falls. For instance, the 95th percentile indicates the value below which
-/// 95% of the data points can be found.
-///
-/// This aggregation can be particularly interesting for analyzing website or service response
-/// times. For example, if the 95th percentile website load time is significantly higher than the
-/// median, this indicates that a small percentage of users are experiencing much slower load times
-/// than the majority.
-///
-/// To use the percentiles aggregation, you'll need to provide a field to
-/// aggregate on. In the case of website load times, this would typically be a
-/// field containing the duration of time it takes for the site to load.
-///
-/// The following example demonstrates a request for the percentiles of the "load_time"
-/// field:
-///
-/// ```JSON
-/// {
-/// "percentiles": {
-/// "field": "load_time"
-/// }
-/// }
-/// ```
-///
-/// This request will return an object containing the default percentiles (1, 5,
-/// 25, 50 (median), 75, 95, and 99). You can also customize the percentiles you want to
-/// calculate by providing an array of values in the "percents" parameter:
-///
-/// ```JSON
-/// {
-/// "percentiles": {
-/// "field": "load_time",
-/// "percents": [10, 20, 30, 40, 50, 60, 70, 80, 90]
-/// }
-/// }
-/// ```
-///
-/// In this example, the aggregation will return the 10th, 20th, 30th, 40th, 50th,
-/// 60th, 70th, 80th, and 90th percentiles of the "load_time" field.
-///
-/// Analyzing the percentiles of website load times can help you understand the
-/// user experience and identify areas for optimization. For example, if the 95th
-/// percentile load time is significantly higher than the median, this indicates
-/// that a small percentage of users are experiencing much slower load times than
-/// the majority.
-///
-/// # Estimating Percentiles
-///
-/// While percentiles provide valuable insights into the distribution of data, it's
-/// important to understand that they are often estimates. This is because
-/// calculating exact percentiles for large data sets can be computationally
-/// expensive and time-consuming. As a result, many percentile aggregation
-/// algorithms use approximation techniques to provide faster results.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct PercentilesAggregationReq {
- /// The field name to compute the percentiles on.
- pub field: String,
- /// The percentiles to compute.
- /// Defaults to [1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0]
- pub percents: Option<Vec<f64>>,
- /// Whether to return the percentiles as a hash map
- #[serde(default = "default_as_true")]
- pub keyed: bool,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(
- skip_serializing_if = "Option::is_none",
- default,
- deserialize_with = "deserialize_option_f64"
- )]
- pub missing: Option<f64>,
-}
-fn default_percentiles() -> &'static [f64] {
- &[1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0]
-}
-fn default_as_true() -> bool {
- true
-}
-
-impl PercentilesAggregationReq {
- /// Creates a new [`PercentilesAggregationReq`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- PercentilesAggregationReq {
- field: field_name,
- percents: None,
- keyed: default_as_true(),
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-
- fn validate(&self) -> crate::Result<()> {
- if let Some(percents) = self.percents.as_ref() {
- let all_in_range = percents
- .iter()
- .cloned()
- .all(|percent| (0.0..=100.0).contains(&percent));
- if !all_in_range {
- return Err(TantivyError::AggregationError(
- AggregationError::InvalidRequest(
- "All percentiles have to be between 0.0 and 100.0".to_string(),
- ),
- ));
- }
- }
-
- Ok(())
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct SegmentPercentilesCollector {
- field_type: ColumnType,
- pub(crate) percentiles: PercentilesCollector,
- pub(crate) accessor_idx: usize,
- missing: Option<u64>,
-}
-
-#[derive(Clone, Serialize, Deserialize)]
-/// The percentiles collector used during segment collection and for merging results.
-pub struct PercentilesCollector {
- sketch: sketches_ddsketch::DDSketch,
-}
-impl Default for PercentilesCollector {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl Debug for PercentilesCollector {
- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
- f.debug_struct("IntermediatePercentiles")
- .field("sketch_len", &self.sketch.length())
- .finish()
- }
-}
-impl PartialEq for PercentilesCollector {
- fn eq(&self, _other: &Self) -> bool {
- false
- }
-}
-
-fn format_percentil(percentil: f64) -> String {
- let mut out = percentil.to_string();
- // Slightly silly way to format trailing decimals
- if !out.contains('.') {
- out.push_str(".0");
- }
- out
-}
-
-impl PercentilesCollector {
- /// Convert result into final result. This will query the quantils from the underlying quantil
- /// collector.
- pub fn into_final_result(self, req: &PercentilesAggregationReq) -> PercentilesMetricResult {
- let percentiles: &[f64] = req
- .percents
- .as_ref()
- .map(|el| el.as_ref())
- .unwrap_or(default_percentiles());
- let iter_quantile_and_values = percentiles.iter().cloned().map(|percentile| {
- (
- percentile,
- self.sketch
- .quantile(percentile / 100.0)
- .expect(
- "quantil out of range. This error should have been caught during \
- validation phase",
- )
- .unwrap_or(f64::NAN),
- )
- });
-
- let values = if req.keyed {
- PercentileValues::HashMap(
- iter_quantile_and_values
- .map(|(val, quantil)| (format_percentil(val), quantil))
- .collect(),
- )
- } else {
- PercentileValues::Vec(
- iter_quantile_and_values
- .map(|(key, value)| PercentileValuesVecEntry { key, value })
- .collect(),
- )
- };
- PercentilesMetricResult { values }
- }
-
- fn new() -> Self {
- let ddsketch_config = sketches_ddsketch::Config::defaults();
- let sketch = sketches_ddsketch::DDSketch::new(ddsketch_config);
- Self { sketch }
- }
- fn collect(&mut self, val: f64) {
- self.sketch.add(val);
- }
-
- pub(crate) fn merge_fruits(&mut self, right: PercentilesCollector) -> crate::Result<()> {
- self.sketch.merge(&right.sketch).map_err(|err| {
- TantivyError::AggregationError(AggregationError::InternalError(format!(
- "Error while merging percentiles {err:?}"
- )))
- })?;
-
- Ok(())
- }
-}
-
-impl SegmentPercentilesCollector {
- pub fn from_req_and_validate(
- req: &PercentilesAggregationReq,
- field_type: ColumnType,
- accessor_idx: usize,
- ) -> crate::Result<Self> {
- req.validate()?;
- let missing = req
- .missing
- .and_then(|val| f64_to_fastfield_u64(val, &field_type));
-
- Ok(Self {
- field_type,
- percentiles: PercentilesCollector::new(),
- accessor_idx,
- missing,
- })
- }
- #[inline]
- pub(crate) fn collect_block_with_field(
- &mut self,
- docs: &[DocId],
- agg_accessor: &mut AggregationWithAccessor,
- ) {
- if let Some(missing) = self.missing.as_ref() {
- agg_accessor.column_block_accessor.fetch_block_with_missing(
- docs,
- &agg_accessor.accessor,
- *missing,
- );
- } else {
- agg_accessor
- .column_block_accessor
- .fetch_block(docs, &agg_accessor.accessor);
- }
-
- for val in agg_accessor.column_block_accessor.iter_vals() {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.percentiles.collect(val1);
- }
- }
-}
-
-impl SegmentAggregationCollector for SegmentPercentilesCollector {
- #[inline]
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
- let intermediate_metric_result = IntermediateMetricResult::Percentiles(self.percentiles);
-
- results.push(
- name,
- IntermediateAggregationResult::Metric(intermediate_metric_result),
- )?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;
-
- if let Some(missing) = self.missing {
- let mut has_val = false;
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.percentiles.collect(val1);
- has_val = true;
- }
- if !has_val {
- self.percentiles
- .collect(f64_from_fastfield_u64(missing, &self.field_type));
- }
- } else {
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.percentiles.collect(val1);
- }
- }
-
- Ok(())
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];
- self.collect_block_with_field(docs, field);
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
-
- use itertools::Itertools;
- use more_asserts::{assert_ge, assert_le};
- use rand::rngs::StdRng;
- use rand::SeedableRng;
- use serde_json::Value;
-
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::tests::{
- exec_request_with_query, get_test_index_from_values, get_test_index_from_values_and_terms,
- };
- use crate::aggregation::AggregationCollector;
- use crate::query::AllQuery;
- use crate::schema::{Schema, FAST};
- use crate::Index;
-
- #[test]
- fn test_aggregation_percentiles_empty_index() -> crate::Result<()> {
- // test index without segments
- let values = vec![];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "percentiles": {
- "percentiles": {
- "field": "score",
- }
- },
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- assert_eq!(
- res["percentiles"]["values"],
- json!({
- "1.0": Value::Null,
- "5.0": Value::Null,
- "25.0": Value::Null,
- "50.0": Value::Null,
- "75.0": Value::Null,
- "95.0": Value::Null,
- "99.0": Value::Null,
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_percentile_simple() -> crate::Result<()> {
- let values = vec![10.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "percentiles": {
- "percentiles": {
- "field": "score",
- }
- },
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
-
- let percents = vec!["1.0", "5.0", "25.0", "50.0", "75.0", "95.0", "99.0"];
- let range = 9.9..10.1;
- for percent in percents {
- let val = res["percentiles"]["values"][percent].as_f64().unwrap();
- assert!(range.contains(&val));
- }
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_percentile_parameters() -> crate::Result<()> {
- let values = vec![10.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_str = r#"
- {
- "mypercentiles": {
- "percentiles": {
- "field": "score",
- "percents": [ 95, 99, 99.9 ]
- }
- }
- } "#;
- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
-
- let percents = vec!["95.0", "99.0", "99.9"];
- let expected_range = 9.9..10.1;
- for percent in percents {
- let val = res["mypercentiles"]["values"][percent].as_f64().unwrap();
- assert!(expected_range.contains(&val));
- }
- // Keyed false
- //
- let agg_req_str = r#"
- {
- "mypercentiles": {
- "percentiles": {
- "field": "score",
- "percents": [ 95, 99, 99.9 ],
- "keyed": false
- }
- }
- } "#;
- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
-
- let vals = &res["mypercentiles"]["values"];
- assert_eq!(vals[0]["key"].as_f64().unwrap(), 95.0);
- assert_eq!(vals[1]["key"].as_f64().unwrap(), 99.0);
- assert_eq!(vals[2]["key"].as_f64().unwrap(), 99.9);
- assert_eq!(vals[3]["key"], serde_json::Value::Null);
- assert!(expected_range.contains(&vals[0]["value"].as_f64().unwrap()));
- assert!(expected_range.contains(&vals[1]["value"].as_f64().unwrap()));
- assert!(expected_range.contains(&vals[2]["value"].as_f64().unwrap()));
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_percentiles_single_seg() -> crate::Result<()> {
- test_aggregation_percentiles(true)
- }
-
- #[test]
- fn test_aggregation_percentiles_multi_seg() -> crate::Result<()> {
- test_aggregation_percentiles(false)
- }
-
- fn test_aggregation_percentiles(merge_segments: bool) -> crate::Result<()> {
- use rand_distr::Distribution;
- let num_values_in_segment = [100, 30_000, 8000];
- let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();
- let mut rng = StdRng::from_seed([1u8; 32]);
-
- let segment_data = |i| {
- (0..num_values_in_segment[i])
- .map(|_| lg_norm.sample(&mut rng))
- .collect_vec()
- };
-
- let values = (0..=2).map(segment_data).collect_vec();
-
- let mut all_values = values
- .iter()
- .flat_map(|el| el.iter().cloned())
- .collect_vec();
- all_values.sort_unstable_by(|a, b| a.total_cmp(b));
-
- fn get_exact_quantil(q: f64, all_values: &[f64]) -> f64 {
- let q = q / 100.0;
- assert!((0f64..=1f64).contains(&q));
-
- let index = (all_values.len() as f64 * q).ceil() as usize;
- let index = index.min(all_values.len() - 1);
- all_values[index]
- }
-
- let segment_and_values = values
- .into_iter()
- .map(|segment_data| {
- segment_data
- .into_iter()
- .map(|val| (val, val.to_string()))
- .collect_vec()
- })
- .collect_vec();
-
- let index =
- get_test_index_from_values_and_terms(merge_segments, &segment_and_values).unwrap();
-
- let reader = index.reader()?;
-
- let agg_req_str = r#"
- {
- "mypercentiles": {
- "percentiles": {
- "field": "score_f64",
- "percents": [ 95, 99, 99.9 ]
- }
- }
- } "#;
- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- let vals = &res["mypercentiles"]["values"];
-
- let check_quantil = |exact_quantil: f64, val: f64| {
- let lower = exact_quantil - exact_quantil * 0.02;
- let upper = exact_quantil + exact_quantil * 0.02;
- assert_le!(val, upper);
- assert_ge!(val, lower);
- };
-
- let val = vals["95.0"].as_f64().unwrap();
- let exact_quantil = get_exact_quantil(95.0, &all_values);
- check_quantil(exact_quantil, val);
-
- let val = vals["99.0"].as_f64().unwrap();
- let exact_quantil = get_exact_quantil(99.0, &all_values);
- check_quantil(exact_quantil, val);
-
- let val = vals["99.9"].as_f64().unwrap();
- let exact_quantil = get_exact_quantil(99.9, &all_values);
- check_quantil(exact_quantil, val);
-
- Ok(())
- }
-
- #[test]
- fn test_percentiles_missing_sub_agg() -> crate::Result<()> {
- // This test verifies the `collect` method (in contrast to `collect_block`), which is
- // called when the sub-aggregations are flushed.
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("texts", FAST);
- let score_field_f64 = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- let mut index_writer = index.writer_for_tests()?;
- // writing the segment
- index_writer.add_document(doc!(
- score_field_f64 => 10.0f64,
- text_field => "a"
- ))?;
- index_writer.add_document(doc!(
- score_field_f64 => 10.0f64,
- text_field => "a"
- ))?;
-
- index_writer.add_document(doc!(text_field => "a"))?;
-
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = {
- serde_json::from_value(json!({
- "range_with_stats": {
- "terms": {
- "field": "texts"
- },
- "aggs": {
- "percentiles": {
- "percentiles": {
- "field": "score",
- "missing": 5.0
- }
- }
- }
- }
- }))
- .unwrap()
- };
-
- let res = exec_request_with_query(agg_req, &index, None)?;
- assert_eq!(res["range_with_stats"]["buckets"][0]["doc_count"], 3);
-
- assert_eq!(
- res["range_with_stats"]["buckets"][0]["percentiles"]["values"]["1.0"],
- 5.0028295751107414
- );
- assert_eq!(
- res["range_with_stats"]["buckets"][0]["percentiles"]["values"]["99.0"],
- 10.07469668951144
- );
-
- Ok(())
- }
-
- #[test]
- fn test_percentiles_missing() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("texts", FAST);
- let score_field_f64 = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- let mut index_writer = index.writer_for_tests()?;
- // writing the segment
- index_writer.add_document(doc!(
- score_field_f64 => 10.0f64,
- text_field => "a"
- ))?;
- index_writer.add_document(doc!(
- score_field_f64 => 10.0f64,
- text_field => "a"
- ))?;
-
- index_writer.add_document(doc!(text_field => "a"))?;
-
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = {
- serde_json::from_value(json!({
- "percentiles": {
- "percentiles": {
- "field": "score",
- "missing": 5.0
- }
- }
- }))
- .unwrap()
- };
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(res["percentiles"]["values"]["1.0"], 5.0028295751107414);
- assert_eq!(res["percentiles"]["values"]["99.0"], 10.07469668951144);
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/stats.rs b/crates/tantivy/src/aggregation/metric/stats.rs
deleted file mode 100644
index 8b8be399..00000000
--- a/crates/tantivy/src/aggregation/metric/stats.rs
+++ /dev/null
@@ -1,684 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::agg_req_with_accessor::{
- AggregationWithAccessor, AggregationsWithAccessor,
-};
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,
-};
-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::*;
-use crate::{DocId, TantivyError};
-
-/// A multi-value metric aggregation that computes a collection of statistics on numeric values that
-/// are extracted from the aggregated documents.
-/// See [`Stats`] for returned statistics.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "stats": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct StatsAggregation {
- /// The field name to compute the stats on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl StatsAggregation {
- /// Creates a new [`StatsAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- StatsAggregation {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Stats contains a collection of statistics.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct Stats {
- /// The number of documents.
- pub count: u64,
- /// The sum of the fast field values.
- pub sum: f64,
- /// The min value of the fast field values.
- pub min: Option<f64>,
- /// The max value of the fast field values.
- pub max: Option<f64>,
- /// The average of the fast field values. `None` if count equals zero.
- pub avg: Option<f64>,
-}
-
-impl Stats {
- pub(crate) fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {
- match agg_property {
- "count" => Ok(Some(self.count as f64)),
- "sum" => Ok(Some(self.sum)),
- "min" => Ok(self.min),
- "max" => Ok(self.max),
- "avg" => Ok(self.avg),
- _ => Err(TantivyError::InvalidArgument(format!(
- "Unknown property {agg_property} on stats metric aggregation"
- ))),
- }
- }
-}
-
-/// Intermediate result of the stats aggregation that can be combined with other intermediate
-/// results.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateStats {
- /// The number of extracted values.
- pub(crate) count: u64,
- /// The sum of the extracted values.
- pub(crate) sum: f64,
- /// delta for sum needed for [Kahan algorithm for summation](https://en.wikipedia.org/wiki/Kahan_summation_algorithm)
- pub(crate) delta: f64,
- /// The min value.
- pub(crate) min: f64,
- /// The max value.
- pub(crate) max: f64,
-}
-
-impl Default for IntermediateStats {
- fn default() -> Self {
- Self {
- count: 0,
- sum: 0.0,
- delta: 0.0,
- min: f64::MAX,
- max: f64::MIN,
- }
- }
-}
-
-impl IntermediateStats {
- /// Merges the other stats intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateStats) {
- self.count += other.count;
-
- // kahan algorithm for sum
- let y = other.sum - (self.delta + other.delta);
- let t = self.sum + y;
- self.delta = (t - self.sum) - y;
- self.sum = t;
-
- self.min = self.min.min(other.min);
- self.max = self.max.max(other.max);
- }
-
- /// Computes the final stats value.
- pub fn finalize(&self) -> Stats {
- let min = if self.count == 0 {
- None
- } else {
- Some(self.min)
- };
- let max = if self.count == 0 {
- None
- } else {
- Some(self.max)
- };
- let avg = if self.count == 0 {
- None
- } else {
- Some(self.sum / (self.count as f64))
- };
- Stats {
- count: self.count,
- sum: self.sum,
- min,
- max,
- avg,
- }
- }
-
- #[inline]
- pub(in crate::aggregation::metric) fn collect(&mut self, value: f64) {
- self.count += 1;
-
- // kahan algorithm for sum
- let y = value - self.delta;
- let t = self.sum + y;
- self.delta = (t - self.sum) - y;
- self.sum = t;
-
- self.min = self.min.min(value);
- self.max = self.max.max(value);
- }
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) enum SegmentStatsType {
- Average,
- Count,
- Max,
- Min,
- Stats,
- Sum,
-}
-
-#[derive(Clone, Debug, PartialEq)]
-pub(crate) struct SegmentStatsCollector {
- missing: Option<u64>,
- field_type: ColumnType,
- pub(crate) collecting_for: SegmentStatsType,
- pub(crate) stats: IntermediateStats,
- pub(crate) accessor_idx: usize,
- val_cache: Vec<u64>,
-}
-
-impl SegmentStatsCollector {
- pub fn from_req(
- field_type: ColumnType,
- collecting_for: SegmentStatsType,
- accessor_idx: usize,
- missing: Option<f64>,
- ) -> Self {
- let missing = missing.and_then(|val| f64_to_fastfield_u64(val, &field_type));
- Self {
- field_type,
- collecting_for,
- stats: IntermediateStats::default(),
- accessor_idx,
- missing,
- val_cache: Default::default(),
- }
- }
- #[inline]
- pub(crate) fn collect_block_with_field(
- &mut self,
- docs: &[DocId],
- agg_accessor: &mut AggregationWithAccessor,
- ) {
- if let Some(missing) = self.missing.as_ref() {
- agg_accessor.column_block_accessor.fetch_block_with_missing(
- docs,
- &agg_accessor.accessor,
- *missing,
- );
- } else {
- agg_accessor
- .column_block_accessor
- .fetch_block(docs, &agg_accessor.accessor);
- }
- for val in agg_accessor.column_block_accessor.iter_vals() {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.stats.collect(val1);
- }
- }
-}
-
-impl SegmentAggregationCollector for SegmentStatsCollector {
- #[inline]
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
-
- let intermediate_metric_result = match self.collecting_for {
- SegmentStatsType::Average => {
- IntermediateMetricResult::Average(IntermediateAverage::from_collector(*self))
- }
- SegmentStatsType::Count => {
- IntermediateMetricResult::Count(IntermediateCount::from_collector(*self))
- }
- SegmentStatsType::Max => {
- IntermediateMetricResult::Max(IntermediateMax::from_collector(*self))
- }
- SegmentStatsType::Min => {
- IntermediateMetricResult::Min(IntermediateMin::from_collector(*self))
- }
- SegmentStatsType::Stats => IntermediateMetricResult::Stats(self.stats),
- SegmentStatsType::Sum => {
- IntermediateMetricResult::Sum(IntermediateSum::from_collector(*self))
- }
- };
-
- results.push(
- name,
- IntermediateAggregationResult::Metric(intermediate_metric_result),
- )?;
-
- Ok(())
- }
-
- #[inline]
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;
- if let Some(missing) = self.missing {
- let mut has_val = false;
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.stats.collect(val1);
- has_val = true;
- }
- if !has_val {
- self.stats
- .collect(f64_from_fastfield_u64(missing, &self.field_type));
- }
- } else {
- for val in field.values_for_doc(doc) {
- let val1 = f64_from_fastfield_u64(val, &self.field_type);
- self.stats.collect(val1);
- }
- }
-
- Ok(())
- }
-
- #[inline]
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];
- self.collect_block_with_field(docs, field);
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use serde_json::Value;
-
- use crate::aggregation::agg_req::{Aggregation, Aggregations};
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::tests::{
- exec_request_with_query, get_test_index_2_segments, get_test_index_from_values,
- };
- use crate::aggregation::AggregationCollector;
- use crate::query::{AllQuery, TermQuery};
- use crate::schema::{IndexRecordOption, Schema, FAST};
- use crate::{Index, IndexWriter, Term};
-
- #[test]
- fn test_aggregation_stats_empty_index() -> crate::Result<()> {
- // test index without segments
- let values = vec![];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "stats": {
- "stats": {
- "field": "score",
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- assert_eq!(
- res["stats"],
- json!({
- "avg": Value::Null,
- "count": 0,
- "max": Value::Null,
- "min": Value::Null,
- "sum": 0.0
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_stats_simple() -> crate::Result<()> {
- let values = vec![10.0];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "stats": {
- "stats": {
- "field": "score",
- },
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- assert_eq!(
- res["stats"],
- json!({
- "avg": 10.0,
- "count": 1,
- "max": 10.0,
- "min": 10.0,
- "sum": 10.0
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_stats() -> crate::Result<()> {
- let index = get_test_index_2_segments(false)?;
-
- let reader = index.reader()?;
- let text_field = reader.searcher().schema().get_field("text").unwrap();
-
- let term_query = TermQuery::new(
- Term::from_field_text(text_field, "cool"),
- IndexRecordOption::Basic,
- );
-
- let range_agg: Aggregation = {
- serde_json::from_value(json!({
- "range": {
- "field": "score",
- "ranges": [ { "from": 3.0f64, "to": 7.0f64 }, { "from": 7.0f64, "to": 19.0f64 }, { "from": 19.0f64, "to": 20.0f64 } ]
- },
- "aggs": {
- "stats": {
- "stats": {
- "field": "score"
- }
- }
- }
- }))
- .unwrap()
- };
-
- let agg_req_1: Aggregations = serde_json::from_value(json!({
- "stats_i64": {
- "stats": {
- "field": "score_i64",
- },
- },
- "stats_f64": {
- "stats": {
- "field": "score_f64",
- },
- },
- "stats": {
- "stats": {
- "field": "score",
- },
- },
- "range": range_agg
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());
-
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&term_query, &collector).unwrap();
-
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- assert_eq!(
- res["stats"],
- json!({
- "avg": 12.142857142857142,
- "count": 7,
- "max": 44.0,
- "min": 1.0,
- "sum": 85.0
- })
- );
-
- assert_eq!(
- res["stats_i64"],
- json!({
- "avg": 12.142857142857142,
- "count": 7,
- "max": 44.0,
- "min": 1.0,
- "sum": 85.0
- })
- );
-
- assert_eq!(
- res["stats_f64"],
- json!({
- "avg": 12.214285714285714,
- "count": 7,
- "max": 44.5,
- "min": 1.0,
- "sum": 85.5
- })
- );
-
- assert_eq!(
- res["range"]["buckets"][2]["stats"],
- json!({
- "avg": 10.666666666666666,
- "count": 3,
- "max": 14.0,
- "min": 7.0,
- "sum": 32.0
- })
- );
-
- assert_eq!(
- res["range"]["buckets"][3]["stats"],
- json!({
- "avg": serde_json::Value::Null,
- "count": 0,
- "max": serde_json::Value::Null,
- "min": serde_json::Value::Null,
- "sum": 0.0,
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_stats_json() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with empty json
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
- // => Segment with json, but no field partially_empty
- index_writer
- .add_document(doc!(json => json!({"different_field": "blue"})))
- .unwrap();
- index_writer.commit().unwrap();
- //// => Segment with field partially_empty
- index_writer
- .add_document(doc!(json => json!({"partially_empty": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "stats": {
- "field": "json.partially_empty"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res["my_stats"],
- json!({
- "avg": 10.0,
- "count": 1,
- "max": 10.0,
- "min": 10.0,
- "sum": 10.0
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_stats_json_missing() -> crate::Result<()> {
- let mut schema_builder = Schema::builder();
- let json = schema_builder.add_json_field("json", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();
- // => Segment with empty json
- index_writer.add_document(doc!()).unwrap();
- index_writer.commit().unwrap();
- // => Segment with json, but no field partially_empty
- index_writer
- .add_document(doc!(json => json!({"different_field": "blue"})))
- .unwrap();
- index_writer.commit().unwrap();
- //// => Segment with field partially_empty
- index_writer
- .add_document(doc!(json => json!({"partially_empty": 10.0})))
- .unwrap();
- index_writer.add_document(doc!())?;
- index_writer.commit().unwrap();
-
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "stats": {
- "field": "json.partially_empty",
- "missing": 0.0
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res["my_stats"],
- json!({
- "avg": 2.5,
- "count": 4,
- "max": 10.0,
- "min": 0.0,
- "sum": 10.0
- })
- );
-
- // From string
- let agg_req: Aggregations = serde_json::from_value(json!({
- "my_stats": {
- "stats": {
- "field": "json.partially_empty",
- "missing": "0.0"
- },
- }
- }))
- .unwrap();
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res["my_stats"],
- json!({
- "avg": 2.5,
- "count": 4,
- "max": 10.0,
- "min": 0.0,
- "sum": 10.0
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_stats_json_missing_sub_agg() -> crate::Result<()> {
- // This test verifies the `collect` method (in contrast to `collect_block`), which is
- // called when the sub-aggregations are flushed.
- let mut schema_builder = Schema::builder();
- let text_field = schema_builder.add_text_field("texts", FAST);
- let score_field_f64 = schema_builder.add_f64_field("score", FAST);
- let schema = schema_builder.build();
- let index = Index::create_in_ram(schema);
-
- {
- let mut index_writer = index.writer_for_tests()?;
- // writing the segment
- index_writer.add_document(doc!(
- score_field_f64 => 10.0f64,
- text_field => "a"
- ))?;
-
- index_writer.add_document(doc!(text_field => "a"))?;
-
- index_writer.commit()?;
- }
-
- let agg_req: Aggregations = {
- serde_json::from_value(json!({
- "range_with_stats": {
- "terms": {
- "field": "texts"
- },
- "aggs": {
- "my_stats": {
- "stats": {
- "field": "score",
- "missing": 0.0
- }
- }
- }
- }
- }))
- .unwrap()
- };
-
- let res = exec_request_with_query(agg_req, &index, None)?;
-
- assert_eq!(
- res["range_with_stats"]["buckets"][0]["my_stats"]["count"],
- 2
- );
- assert_eq!(
- res["range_with_stats"]["buckets"][0]["my_stats"]["min"],
- 0.0
- );
- assert_eq!(
- res["range_with_stats"]["buckets"][0]["my_stats"]["avg"],
- 5.0
- );
-
- Ok(())
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/sum.rs b/crates/tantivy/src/aggregation/metric/sum.rs
deleted file mode 100644
index 86f66167..00000000
--- a/crates/tantivy/src/aggregation/metric/sum.rs
+++ /dev/null
@@ -1,68 +0,0 @@
-use std::fmt::Debug;
-
-use serde::{Deserialize, Serialize};
-
-use super::*;
-use crate::aggregation::*;
-
-/// A single-value metric aggregation that sums up numeric values that are
-/// extracted from the aggregated documents.
-/// See [super::SingleMetricResult] for return value.
-///
-/// # JSON Format
-/// ```json
-/// {
-/// "sum": {
-/// "field": "score"
-/// }
-/// }
-/// ```
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct SumAggregation {
- /// The field name to compute the minimum on.
- pub field: String,
- /// The missing parameter defines how documents that are missing a value should be treated.
- /// By default they will be ignored but it is also possible to treat them as if they had a
- /// value. Examples in JSON format:
- /// { "field": "my_numbers", "missing": "10.0" }
- #[serde(default, deserialize_with = "deserialize_option_f64")]
- pub missing: Option<f64>,
-}
-
-impl SumAggregation {
- /// Creates a new [`SumAggregation`] instance from a field name.
- pub fn from_field_name(field_name: String) -> Self {
- Self {
- field: field_name,
- missing: None,
- }
- }
- /// Returns the field name the aggregation is computed on.
- pub fn field_name(&self) -> &str {
- &self.field
- }
-}
-
-/// Intermediate result of the minimum aggregation that can be combined with other intermediate
-/// results.
-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]
-pub struct IntermediateSum {
- stats: IntermediateStats,
-}
-
-impl IntermediateSum {
- /// Creates a new [`IntermediateSum`] instance from a [`SegmentStatsCollector`].
- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {
- Self {
- stats: collector.stats,
- }
- }
- /// Merges the other intermediate result into self.
- pub fn merge_fruits(&mut self, other: IntermediateSum) {
- self.stats.merge_fruits(other.stats);
- }
- /// Computes the final minimum value.
- pub fn finalize(&self) -> Option<f64> {
- Some(self.stats.finalize().sum)
- }
-}
diff --git a/crates/tantivy/src/aggregation/metric/top_hits.rs b/crates/tantivy/src/aggregation/metric/top_hits.rs
deleted file mode 100644
index 72c8e8ce..00000000
--- a/crates/tantivy/src/aggregation/metric/top_hits.rs
+++ /dev/null
@@ -1,918 +0,0 @@
-use std::collections::HashMap;
-use std::net::Ipv6Addr;
-
-use crate::columnar::{Column, ColumnType, ColumnarReader, DynamicColumn};
-use crate::common::json_path_writer::JSON_PATH_SEGMENT_SEP_STR;
-use crate::common::DateTime;
-use regex::Regex;
-use serde::ser::SerializeMap;
-use serde::{Deserialize, Deserializer, Serialize, Serializer};
-
-use super::{TopHitsMetricResult, TopHitsVecEntry};
-use crate::aggregation::bucket::Order;
-use crate::aggregation::intermediate_agg_result::{
- IntermediateAggregationResult, IntermediateMetricResult,
-};
-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;
-use crate::aggregation::AggregationError;
-use crate::collector::TopNComputer;
-use crate::schema::OwnedValue;
-use crate::{DocAddress, DocId, SegmentOrdinal};
-
-/// # Top Hits
-///
-/// The top hits aggregation is a useful tool to answer questions like:
-/// - "What are the most recent posts by each author?"
-/// - "What are the most popular items in each category?"
-///
-/// It does so by keeping track of the most relevant document being aggregated,
-/// in terms of a sort criterion that can consist of multiple fields and their
-/// sort-orders (ascending or descending).
-///
-/// `top_hits` should not be used as a top-level aggregation. It is intended to be
-/// used as a sub-aggregation, inside a `terms` aggregation or a `filters` aggregation,
-/// for example.
-///
-/// Note that this aggregator does not return the actual document addresses, but
-/// rather a list of the values of the fields that were requested to be retrieved.
-/// These values can be specified in the `docvalue_fields` parameter, which can include
-/// a list of fast fields to be retrieved. At the moment, only fast fields are supported
-/// but it is possible that we support the `fields` parameter to retrieve any stored
-/// field in the future.
-///
-/// The following example demonstrates a request for the top_hits aggregation:
-/// ```JSON
-/// {
-/// "aggs": {
-/// "top_authors": {
-/// "terms": {
-/// "field": "author",
-/// "size": 5
-/// }
-/// },
-/// "aggs": {
-/// "top_hits": {
-/// "size": 2,
-/// "from": 0
-/// "sort": [
-/// { "date": "desc" }
-/// ]
-/// "docvalue_fields": ["date", "title", "iden"]
-/// }
-/// }
-/// }
-/// ```
-///
-/// This request will return an object containing the top two documents, sorted
-/// by the `date` field in descending order. You can also sort by multiple fields, which
-/// helps to resolve ties. The aggregation object for each bucket will look like:
-/// ```JSON
-/// {
-/// "hits": [
-/// {
-/// "score": [<time_u64>],
-/// "docvalue_fields": {
-/// "date": "<date_RFC3339>",
-/// "title": "<title>",
-/// "iden": "<iden>"
-/// }
-/// },
-/// {
-/// "score": [<time_u64>]
-/// "docvalue_fields": {
-/// "date": "<date_RFC3339>",
-/// "title": "<title>",
-/// "iden": "<iden>"
-/// }
-/// }
-/// ]
-/// }
-/// ```
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
-pub struct TopHitsAggregation {
- sort: Vec<KeyOrder>,
- size: usize,
- from: Option<usize>,
-
- #[serde(rename = "docvalue_fields")]
- #[serde(default)]
- doc_value_fields: Vec<String>,
-
- // Not supported
- _source: Option<serde_json::Value>,
- fields: Option<serde_json::Value>,
- script_fields: Option<serde_json::Value>,
- highlight: Option<serde_json::Value>,
- explain: Option<serde_json::Value>,
- version: Option<serde_json::Value>,
-}
-
-#[derive(Debug, Clone, PartialEq, Default)]
-struct KeyOrder {
- field: String,
- order: Order,
-}
-
-impl Serialize for KeyOrder {
- fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
- let KeyOrder { field, order } = self;
- let mut map = serializer.serialize_map(Some(1))?;
- map.serialize_entry(field, order)?;
- map.end()
- }
-}
-
-impl<'de> Deserialize<'de> for KeyOrder {
- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
- where
- D: Deserializer<'de>,
- {
- let mut key_order = <HashMap<String, Order>>::deserialize(deserializer)?.into_iter();
- let (field, order) = key_order.next().ok_or(serde::de::Error::custom(
- "Expected exactly one key-value pair in sort parameter of top_hits, found none",
- ))?;
- if key_order.next().is_some() {
- return Err(serde::de::Error::custom(format!(
- "Expected exactly one key-value pair in sort parameter of top_hits, found \
- {key_order:?}"
- )));
- }
- Ok(Self { field, order })
- }
-}
-
-// Tranform a glob (`pattern*`, for example) into a regex::Regex (`^pattern.*$`)
-fn globbed_string_to_regex(glob: &str) -> Result<Regex, crate::TantivyError> {
- // Replace `*` glob with `.*` regex
- let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));
- Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {
- crate::TantivyError::SchemaError(format!("Invalid regex '{glob}' in docvalue_fields: {e}"))
- })
-}
-
-fn use_doc_value_fields_err(parameter: &str) -> crate::Result<()> {
- Err(crate::TantivyError::AggregationError(
- AggregationError::InvalidRequest(format!(
- "The `{parameter}` parameter is not supported, only `docvalue_fields` is supported in \
- `top_hits` aggregation"
- )),
- ))
-}
-fn unsupported_err(parameter: &str) -> crate::Result<()> {
- Err(crate::TantivyError::AggregationError(
- AggregationError::InvalidRequest(format!(
- "The `{parameter}` parameter is not supported in the `top_hits` aggregation"
- )),
- ))
-}
-
-impl TopHitsAggregation {
- /// Validate and resolve field retrieval parameters
- pub fn validate_and_resolve_field_names(
- &mut self,
- reader: &ColumnarReader,
- ) -> crate::Result<()> {
- if self._source.is_some() {
- use_doc_value_fields_err("_source")?;
- }
- if self.fields.is_some() {
- use_doc_value_fields_err("fields")?;
- }
- if self.script_fields.is_some() {
- use_doc_value_fields_err("script_fields")?;
- }
- if self.explain.is_some() {
- unsupported_err("explain")?;
- }
- if self.highlight.is_some() {
- unsupported_err("highlight")?;
- }
- if self.version.is_some() {
- unsupported_err("version")?;
- }
-
- self.doc_value_fields = self
- .doc_value_fields
- .iter()
- .map(|field| {
- if !field.contains('*')
- && reader
- .iter_columns()?
- .any(|(name, _)| name.as_str() == field)
- {
- return Ok(vec![field.to_owned()]);
- }
-
- let pattern = globbed_string_to_regex(field)?;
- let fields = reader
- .iter_columns()?
- .map(|(name, _)| {
- // normalize path from internal fast field repr
- name.replace(JSON_PATH_SEGMENT_SEP_STR, ".")
- })
- .filter(|name| pattern.is_match(name))
- .collect::<Vec<_>>();
- assert!(
- !fields.is_empty(),
- "No fields matched the glob '{field}' in docvalue_fields"
- );
- Ok(fields)
- })
- .collect::<crate::Result<Vec<_>>>()?
- .into_iter()
- .flatten()
- .collect();
-
- Ok(())
- }
-
- /// Return fields accessed by the aggregator, in order.
- pub fn field_names(&self) -> Vec<&str> {
- self.sort
- .iter()
- .map(|KeyOrder { field, .. }| field.as_str())
- .collect()
- }
-
- /// Return fields accessed by the aggregator's value retrieval.
- pub fn value_field_names(&self) -> Vec<&str> {
- self.doc_value_fields.iter().map(|s| s.as_str()).collect()
- }
-
- fn get_document_field_data(
- &self,
- accessors: &HashMap<String, Vec<DynamicColumn>>,
- doc_id: DocId,
- ) -> HashMap<String, FastFieldValue> {
- let doc_value_fields = self
- .doc_value_fields
- .iter()
- .map(|field| {
- let accessors = accessors
- .get(field)
- .unwrap_or_else(|| panic!("field '{field}' not found in accessors"));
-
- let values: Vec<FastFieldValue> = accessors
- .iter()
- .flat_map(|accessor| match accessor {
- DynamicColumn::U64(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::U64)
- .collect::<Vec<_>>(),
- DynamicColumn::I64(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::I64)
- .collect::<Vec<_>>(),
- DynamicColumn::F64(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::F64)
- .collect::<Vec<_>>(),
- DynamicColumn::Bytes(accessor) => accessor
- .term_ords(doc_id)
- .map(|term_ord| {
- let mut buffer = vec![];
- assert!(
- accessor
- .ord_to_bytes(term_ord, &mut buffer)
- .expect("could not read term dictionary"),
- "term corresponding to term_ord does not exist"
- );
- FastFieldValue::Bytes(buffer)
- })
- .collect::<Vec<_>>(),
- DynamicColumn::Str(accessor) => accessor
- .term_ords(doc_id)
- .map(|term_ord| {
- let mut buffer = vec![];
- assert!(
- accessor
- .ord_to_bytes(term_ord, &mut buffer)
- .expect("could not read term dictionary"),
- "term corresponding to term_ord does not exist"
- );
- FastFieldValue::Str(String::from_utf8(buffer).unwrap())
- })
- .collect::<Vec<_>>(),
- DynamicColumn::Bool(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::Bool)
- .collect::<Vec<_>>(),
- DynamicColumn::IpAddr(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::IpAddr)
- .collect::<Vec<_>>(),
- DynamicColumn::DateTime(accessor) => accessor
- .values_for_doc(doc_id)
- .map(FastFieldValue::Date)
- .collect::<Vec<_>>(),
- })
- .collect();
-
- (field.to_owned(), FastFieldValue::Array(values))
- })
- .collect();
- doc_value_fields
- }
-}
-
-/// A retrieved value from a fast field.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum FastFieldValue {
- /// The str type is used for any text information.
- Str(String),
- /// Unsigned 64-bits Integer `u64`
- U64(u64),
- /// Signed 64-bits Integer `i64`
- I64(i64),
- /// 64-bits Float `f64`
- F64(f64),
- /// Bool value
- Bool(bool),
- /// Date/time with nanoseconds precision
- Date(DateTime),
- /// Arbitrarily sized byte array
- Bytes(Vec<u8>),
- /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
- IpAddr(Ipv6Addr),
- /// A list of values.
- Array(Vec<Self>),
-}
-
-impl From<FastFieldValue> for OwnedValue {
- fn from(value: FastFieldValue) -> Self {
- match value {
- FastFieldValue::Str(s) => OwnedValue::Str(s),
- FastFieldValue::U64(u) => OwnedValue::U64(u),
- FastFieldValue::I64(i) => OwnedValue::I64(i),
- FastFieldValue::F64(f) => OwnedValue::F64(f),
- FastFieldValue::Bool(b) => OwnedValue::Bool(b),
- FastFieldValue::Date(d) => OwnedValue::Date(d),
- FastFieldValue::Bytes(b) => OwnedValue::Bytes(b),
- FastFieldValue::IpAddr(ip) => OwnedValue::IpAddr(ip),
- FastFieldValue::Array(a) => {
- OwnedValue::Array(a.into_iter().map(OwnedValue::from).collect())
- }
- }
- }
-}
-
-/// Holds a fast field value in its u64 representation, and the order in which it should be sorted.
-#[derive(Clone, Serialize, Deserialize, Debug)]
-struct DocValueAndOrder {
- /// A fast field value in its u64 representation.
- value: Option<u64>,
- /// Sort order for the value
- order: Order,
-}
-
-impl Ord for DocValueAndOrder {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- let invert = |cmp: std::cmp::Ordering| match self.order {
- Order::Asc => cmp,
- Order::Desc => cmp.reverse(),
- };
-
- match (self.value, other.value) {
- (Some(self_value), Some(other_value)) => invert(self_value.cmp(&other_value)),
- (Some(_), None) => std::cmp::Ordering::Greater,
- (None, Some(_)) => std::cmp::Ordering::Less,
- (None, None) => std::cmp::Ordering::Equal,
- }
- }
-}
-
-impl PartialOrd for DocValueAndOrder {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl PartialEq for DocValueAndOrder {
- fn eq(&self, other: &Self) -> bool {
- self.value.cmp(&other.value) == std::cmp::Ordering::Equal
- }
-}
-
-impl Eq for DocValueAndOrder {}
-
-#[derive(Clone, Serialize, Deserialize, Debug)]
-struct DocSortValuesAndFields {
- sorts: Vec<DocValueAndOrder>,
-
- #[serde(rename = "docvalue_fields")]
- #[serde(skip_serializing_if = "HashMap::is_empty")]
- doc_value_fields: HashMap<String, FastFieldValue>,
-}
-
-impl Ord for DocSortValuesAndFields {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- for (self_feature, other_feature) in self.sorts.iter().zip(other.sorts.iter()) {
- let cmp = self_feature.cmp(other_feature);
- if cmp != std::cmp::Ordering::Equal {
- return cmp;
- }
- }
- std::cmp::Ordering::Equal
- }
-}
-
-impl PartialOrd for DocSortValuesAndFields {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl PartialEq for DocSortValuesAndFields {
- fn eq(&self, other: &Self) -> bool {
- self.cmp(other) == std::cmp::Ordering::Equal
- }
-}
-
-impl Eq for DocSortValuesAndFields {}
-
-/// The TopHitsCollector used for collecting over segments and merging results.
-#[derive(Clone, Serialize, Deserialize, Debug)]
-pub struct TopHitsTopNComputer {
- req: TopHitsAggregation,
- top_n: TopNComputer<DocSortValuesAndFields, DocAddress, false>,
-}
-
-impl std::cmp::PartialEq for TopHitsTopNComputer {
- fn eq(&self, _other: &Self) -> bool {
- false
- }
-}
-
-impl TopHitsTopNComputer {
- /// Create a new TopHitsCollector
- pub fn new(req: &TopHitsAggregation) -> Self {
- Self {
- top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
- req: req.clone(),
- }
- }
-
- fn collect(&mut self, features: DocSortValuesAndFields, doc: DocAddress) {
- self.top_n.push(features, doc);
- }
-
- pub(crate) fn merge_fruits(&mut self, other_fruit: Self) -> crate::Result<()> {
- for doc in other_fruit.top_n.into_vec() {
- self.collect(doc.feature, doc.doc);
- }
- Ok(())
- }
-
- /// Finalize by converting self into the final result form
- pub fn into_final_result(self) -> TopHitsMetricResult {
- let mut hits: Vec<TopHitsVecEntry> = self
- .top_n
- .into_sorted_vec()
- .into_iter()
- .map(|doc| TopHitsVecEntry {
- sort: doc.feature.sorts.iter().map(|f| f.value).collect(),
- doc_value_fields: doc
- .feature
- .doc_value_fields
- .into_iter()
- .map(|(k, v)| (k, v.into()))
- .collect(),
- })
- .collect();
-
- // Remove the first `from` elements
- // Truncating from end would be more efficient, but we need to truncate from the front
- // because `into_sorted_vec` gives us a descending order because of the inverted
- // `Ord` semantics of the heap elements.
- hits.drain(..self.req.from.unwrap_or(0));
- TopHitsMetricResult { hits }
- }
-}
-
-#[derive(Clone, Debug)]
-pub(crate) struct TopHitsSegmentCollector {
- segment_ordinal: SegmentOrdinal,
- accessor_idx: usize,
- top_n: TopNComputer<Vec<DocValueAndOrder>, DocAddress, false>,
-}
-
-impl TopHitsSegmentCollector {
- pub fn from_req(
- req: &TopHitsAggregation,
- accessor_idx: usize,
- segment_ordinal: SegmentOrdinal,
- ) -> Self {
- Self {
- top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),
- segment_ordinal,
- accessor_idx,
- }
- }
- fn into_top_hits_collector(
- self,
- value_accessors: &HashMap<String, Vec<DynamicColumn>>,
- req: &TopHitsAggregation,
- ) -> TopHitsTopNComputer {
- let mut top_hits_computer = TopHitsTopNComputer::new(req);
- let top_results = self.top_n.into_vec();
-
- for res in top_results {
- let doc_value_fields = req.get_document_field_data(value_accessors, res.doc.doc_id);
- top_hits_computer.collect(
- DocSortValuesAndFields {
- sorts: res.feature,
- doc_value_fields,
- },
- res.doc,
- );
- }
-
- top_hits_computer
- }
-
- /// TODO add a specialized variant for a single sort field
- fn collect_with(
- &mut self,
- doc_id: crate::DocId,
- req: &TopHitsAggregation,
- accessors: &[(Column<u64>, ColumnType)],
- ) -> crate::Result<()> {
- let sorts: Vec<DocValueAndOrder> = req
- .sort
- .iter()
- .enumerate()
- .map(|(idx, KeyOrder { order, .. })| {
- let order = *order;
- let value = accessors
- .get(idx)
- .expect("could not find field in accessors")
- .0
- .values_for_doc(doc_id)
- .next();
- DocValueAndOrder { value, order }
- })
- .collect();
-
- self.top_n.push(
- sorts,
- DocAddress {
- segment_ord: self.segment_ordinal,
- doc_id,
- },
- );
- Ok(())
- }
-}
-
-impl SegmentAggregationCollector for TopHitsSegmentCollector {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
- results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,
- ) -> crate::Result<()> {
- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();
-
- let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;
- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
- .agg
- .agg
- .as_top_hits()
- .expect("aggregation request must be of type top hits");
-
- let intermediate_result = IntermediateMetricResult::TopHits(
- self.into_top_hits_collector(value_accessors, tophits_req),
- );
- results.push(
- name,
- IntermediateAggregationResult::Metric(intermediate_result),
- )
- }
-
- /// TODO: Consider a caching layer to reduce the call overhead
- fn collect(
- &mut self,
- doc_id: crate::DocId,
- agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
- .agg
- .agg
- .as_top_hits()
- .expect("aggregation request must be of type top hits");
- let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
- self.collect_with(doc_id, tophits_req, accessors)?;
- Ok(())
- }
-
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,
- ) -> crate::Result<()> {
- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]
- .agg
- .agg
- .as_top_hits()
- .expect("aggregation request must be of type top hits");
- let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;
- // TODO: Consider getting fields with the column block accessor.
- for doc in docs {
- self.collect_with(*doc, tophits_req, accessors)?;
- }
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::common::DateTime;
- use pretty_assertions::assert_eq;
- use serde_json::Value;
- use time::macros::datetime;
-
- use super::{DocSortValuesAndFields, DocValueAndOrder, Order};
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::bucket::tests::get_test_index_from_docs;
- use crate::aggregation::tests::get_test_index_from_values;
- use crate::aggregation::AggregationCollector;
- use crate::collector::ComparableDoc;
- use crate::query::AllQuery;
- use crate::schema::OwnedValue;
-
- fn invert_order(cmp_feature: DocValueAndOrder) -> DocValueAndOrder {
- let DocValueAndOrder { value, order } = cmp_feature;
- let order = match order {
- Order::Asc => Order::Desc,
- Order::Desc => Order::Asc,
- };
- DocValueAndOrder { value, order }
- }
-
- fn collector_with_capacity(capacity: usize) -> super::TopHitsTopNComputer {
- super::TopHitsTopNComputer {
- top_n: super::TopNComputer::new(capacity),
- req: Default::default(),
- }
- }
-
- fn invert_order_features(mut cmp_features: DocSortValuesAndFields) -> DocSortValuesAndFields {
- cmp_features.sorts = cmp_features
- .sorts
- .into_iter()
- .map(invert_order)
- .collect::<Vec<_>>();
- cmp_features
- }
-
- #[test]
- fn test_comparable_doc_feature() -> crate::Result<()> {
- let small = DocValueAndOrder {
- value: Some(1),
- order: Order::Asc,
- };
- let big = DocValueAndOrder {
- value: Some(2),
- order: Order::Asc,
- };
- let none = DocValueAndOrder {
- value: None,
- order: Order::Asc,
- };
-
- assert!(small < big);
- assert!(none < small);
- assert!(none < big);
-
- let small = invert_order(small);
- let big = invert_order(big);
- let none = invert_order(none);
-
- assert!(small > big);
- assert!(none < small);
- assert!(none < big);
-
- Ok(())
- }
-
- #[test]
- fn test_comparable_doc_features() -> crate::Result<()> {
- let features_1 = DocSortValuesAndFields {
- sorts: vec![DocValueAndOrder {
- value: Some(1),
- order: Order::Asc,
- }],
- doc_value_fields: Default::default(),
- };
-
- let features_2 = DocSortValuesAndFields {
- sorts: vec![DocValueAndOrder {
- value: Some(2),
- order: Order::Asc,
- }],
- doc_value_fields: Default::default(),
- };
-
- assert!(features_1 < features_2);
-
- assert!(invert_order_features(features_1.clone()) > invert_order_features(features_2));
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_top_hits_empty_index() -> crate::Result<()> {
- let values = vec![];
-
- let index = get_test_index_from_values(false, &values)?;
-
- let d: Aggregations = serde_json::from_value(json!({
- "top_hits_req": {
- "top_hits": {
- "size": 2,
- "sort": [
- { "date": "desc" }
- ],
- "from": 0,
- }
- }
- }))
- .unwrap();
-
- let collector = AggregationCollector::from_aggs(d, Default::default());
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
-
- let res: Value = serde_json::from_str(
- &serde_json::to_string(&agg_res).expect("JSON serialization failed"),
- )
- .expect("JSON parsing failed");
-
- assert_eq!(
- res,
- json!({
- "top_hits_req": {
- "hits": []
- }
- })
- );
-
- Ok(())
- }
-
- #[test]
- fn test_top_hits_collector_single_feature() -> crate::Result<()> {
- let docs = vec![
- ComparableDoc::<_, _, false> {
- doc: crate::DocAddress {
- segment_ord: 0,
- doc_id: 0,
- },
- feature: DocSortValuesAndFields {
- sorts: vec![DocValueAndOrder {
- value: Some(1),
- order: Order::Asc,
- }],
- doc_value_fields: Default::default(),
- },
- },
- ComparableDoc {
- doc: crate::DocAddress {
- segment_ord: 0,
- doc_id: 2,
- },
- feature: DocSortValuesAndFields {
- sorts: vec![DocValueAndOrder {
- value: Some(3),
- order: Order::Asc,
- }],
- doc_value_fields: Default::default(),
- },
- },
- ComparableDoc {
- doc: crate::DocAddress {
- segment_ord: 0,
- doc_id: 1,
- },
- feature: DocSortValuesAndFields {
- sorts: vec![DocValueAndOrder {
- value: Some(5),
- order: Order::Asc,
- }],
- doc_value_fields: Default::default(),
- },
- },
- ];
-
- let mut collector = collector_with_capacity(3);
- for doc in docs.clone() {
- collector.collect(doc.feature, doc.doc);
- }
-
- let res = collector.into_final_result();
-
- assert_eq!(
- res,
- super::TopHitsMetricResult {
- hits: vec![
- super::TopHitsVecEntry {
- sort: vec![docs[0].feature.sorts[0].value],
- doc_value_fields: Default::default(),
- },
- super::TopHitsVecEntry {
- sort: vec![docs[1].feature.sorts[0].value],
- doc_value_fields: Default::default(),
- },
- super::TopHitsVecEntry {
- sort: vec![docs[2].feature.sorts[0].value],
- doc_value_fields: Default::default(),
- },
- ]
- }
- );
-
- Ok(())
- }
-
- fn test_aggregation_top_hits(merge_segments: bool) -> crate::Result<()> {
- let docs = vec![
- vec![
- r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb", "text2": "bbb", "mixed": { "dyn_arr": [1, "2"] } }"#,
- r#"{ "date": "2017-06-15T00:00:00Z", "text": "ccc", "text2": "ddd", "mixed": { "dyn_arr": [3, "4"] } }"#,
- ],
- vec![
- r#"{ "text": "aaa", "text2": "bbb", "date": "2018-01-02T00:00:00Z", "mixed": { "dyn_arr": ["9", 8] } }"#,
- r#"{ "text": "aaa", "text2": "bbb", "date": "2016-01-02T00:00:00Z", "mixed": { "dyn_arr": ["7", 6] } }"#,
- ],
- ];
-
- let index = get_test_index_from_docs(merge_segments, &docs)?;
-
- let d: Aggregations = serde_json::from_value(json!({
- "top_hits_req": {
- "top_hits": {
- "size": 2,
- "sort": [
- { "date": "desc" }
- ],
- "from": 1,
- "docvalue_fields": [
- "date",
- "tex*",
- "mixed.*",
- ],
- }
- }
- }))?;
-
- let collector = AggregationCollector::from_aggs(d, Default::default());
- let reader = index.reader()?;
- let searcher = reader.searcher();
-
- let agg_res =
- serde_json::to_value(searcher.search(&AllQuery, &collector).unwrap()).unwrap();
-
- let date_2017 = datetime!(2017-06-15 00:00:00 UTC);
- let date_2016 = datetime!(2016-01-02 00:00:00 UTC);
-
- assert_eq!(
- agg_res["top_hits_req"],
- json!({
- "hits": [
- {
- "sort": [crate::common::i64_to_u64(date_2017.unix_timestamp_nanos() as i64)],
- "docvalue_fields": {
- "date": [ OwnedValue::Date(DateTime::from_utc(date_2017)) ],
- "text": [ "ccc" ],
- "text2": [ "ddd" ],
- "mixed.dyn_arr": [ 3, "4" ],
- }
- },
- {
- "sort": [crate::common::i64_to_u64(date_2016.unix_timestamp_nanos() as i64)],
- "docvalue_fields": {
- "date": [ OwnedValue::Date(DateTime::from_utc(date_2016)) ],
- "text": [ "aaa" ],
- "text2": [ "bbb" ],
- "mixed.dyn_arr": [ 6, "7" ],
- }
- }
- ]
- }),
- );
-
- Ok(())
- }
-
- #[test]
- fn test_aggregation_top_hits_single_segment() -> crate::Result<()> {
- test_aggregation_top_hits(true)
- }
-
- #[test]
- fn test_aggregation_top_hits_multi_segment() -> crate::Result<()> {
- test_aggregation_top_hits(false)
- }
-}
diff --git a/crates/tantivy/src/aggregation/mod.rs b/crates/tantivy/src/aggregation/mod.rs
deleted file mode 100644
index 8c0a4b00..00000000
--- a/crates/tantivy/src/aggregation/mod.rs
+++ /dev/null
@@ -1,695 +0,0 @@
-//! # Aggregations
-//!
-//! An aggregation summarizes your data as statistics on buckets or metrics.
-//!
-//! Aggregations can provide answer to questions like:
-//! - What is the average price of all sold articles?
-//! - How many errors with status code 500 do we have per day?
-//! - What is the average listing price of cars grouped by color?
-//!
-//! There are two categories: [Metrics](metric) and [Buckets](bucket).
-//!
-//! ## Prerequisite
-//! Currently aggregations work only on [fast fields](`crate::fastfield`). Fast fields
-//! of type `u64`, `f64`, `i64`, `date` and fast fields on text fields.
-//!
-//! ## Usage
-//! To use aggregations, build an aggregation request by constructing
-//! [`Aggregations`](agg_req::Aggregations).
-//! Create an [`AggregationCollector`] from this request. `AggregationCollector` implements the
-//! [`Collector`](crate::collector::Collector) trait and can be passed as collector into
-//! [`Searcher::search()`](crate::Searcher::search).
-//!
-//!
-//! ## JSON Format
-//! Aggregations request and result structures de/serialize into elasticsearch compatible JSON.
-//!
-//! Notice: Intermediate aggregation results should not be de/serialized via JSON format.
-//! Postcard is a good choice.
-//!
-//! ```verbatim
-//! let agg_req: Aggregations = serde_json::from_str(json_request_string).unwrap();
-//! let collector = AggregationCollector::from_aggs(agg_req, None);
-//! let searcher = reader.searcher();
-//! let agg_res = searcher.search(&term_query, &collector).unwrap_err();
-//! let json_response_string: String = &serde_json::to_string(&agg_res)?;
-//! ```
-//!
-//! ## Supported Aggregations
-//! - [Bucket](bucket)
-//! - [Histogram](bucket::HistogramAggregation)
-//! - [DateHistogram](bucket::DateHistogramAggregationReq)
-//! - [Range](bucket::RangeAggregation)
-//! - [Terms](bucket::TermsAggregation)
-//! - [Metric](metric)
-//! - [Average](metric::AverageAggregation)
-//! - [Stats](metric::StatsAggregation)
-//! - [Min](metric::MinAggregation)
-//! - [Max](metric::MaxAggregation)
-//! - [Sum](metric::SumAggregation)
-//! - [Count](metric::CountAggregation)
-//! - [Percentiles](metric::PercentilesAggregationReq)
-//!
-//! # Example
-//! Compute the average metric, by building [`agg_req::Aggregations`], which is built from an
-//! `(String, agg_req::Aggregation)` iterator.
-//!
-//! Requests are compatible with the elasticsearch JSON request format.
-//!
-//! ```
-//! use tantivy::aggregation::agg_req::Aggregations;
-//!
-//! let elasticsearch_compatible_json_req = r#"
-//! {
-//! "average": {
-//! "avg": { "field": "score" }
-//! },
-//! "range": {
-//! "range": {
-//! "field": "score",
-//! "ranges": [
-//! { "to": 3.0 },
-//! { "from": 3.0, "to": 7.0 },
-//! { "from": 7.0, "to": 20.0 },
-//! { "from": 20.0 }
-//! ]
-//! },
-//! "aggs": {
-//! "average_in_range": { "avg": { "field": "score" } }
-//! }
-//! }
-//! }
-//! "#;
-//! let agg_req: Aggregations =
-//! serde_json::from_str(elasticsearch_compatible_json_req).unwrap();
-//! ```
-//! # Code Organization
-//!
-//! Check the [README](https://github.com/quickwit-oss/tantivy/tree/main/src/aggregation#readme) on github to see how the code is organized.
-//!
-//! # Nested Aggregation
-//!
-//! Buckets can contain sub-aggregations. In this example we create buckets with the range
-//! aggregation and then calculate the average on each bucket.
-//! ```
-//! use tantivy::aggregation::agg_req::*;
-//! use serde_json::json;
-//!
-//! let agg_req_1: Aggregations = serde_json::from_value(json!({
-//! "rangef64": {
-//! "range": {
-//! "field": "score",
-//! "ranges": [
-//! { "from": 3, "to": 7000 },
-//! { "from": 7000, "to": 20000 },
-//! { "from": 50000, "to": 60000 }
-//! ]
-//! },
-//! "aggs": {
-//! "average_in_range": { "avg": { "field": "score" } }
-//! }
-//! },
-//! }))
-//! .unwrap();
-//! ```
-//!
-//! # Distributed Aggregation
-//! When the data is distributed on different [`Index`](crate::Index) instances, the
-//! [`DistributedAggregationCollector`] provides functionality to merge data between independent
-//! search calls by returning
-//! [`IntermediateAggregationResults`](intermediate_agg_result::IntermediateAggregationResults).
-//! `IntermediateAggregationResults` provides the
-//! [`merge_fruits`](intermediate_agg_result::IntermediateAggregationResults::merge_fruits) method
-//! to merge multiple results. The merged result can then be converted into
-//! [`AggregationResults`](agg_result::AggregationResults) via the
-//! [`into_final_result`](intermediate_agg_result::IntermediateAggregationResults::into_final_result) method.
-
-mod agg_limits;
-pub mod agg_req;
-mod agg_req_with_accessor;
-pub mod agg_result;
-pub mod bucket;
-mod buf_collector;
-mod collector;
-mod date;
-mod error;
-pub mod intermediate_agg_result;
-pub mod metric;
-
-mod segment_agg_result;
-use std::collections::HashMap;
-use std::fmt::Display;
-
-use core::fmt;
-
-use crate::columnar::{ColumnType, MonotonicallyMappableToU64};
-pub use agg_limits::AggregationLimits;
-pub use collector::{
- AggregationCollector, AggregationSegmentCollector, DistributedAggregationCollector,
- DEFAULT_BUCKET_LIMIT,
-};
-pub(crate) use date::format_date;
-pub use error::AggregationError;
-use itertools::Itertools;
-use serde::de::{self, Visitor};
-use serde::{Deserialize, Deserializer, Serialize};
-
-fn parse_str_into_f64<E: de::Error>(value: &str) -> Result<f64, E> {
- let parsed = value
- .parse::<f64>()
- .map_err(|_err| de::Error::custom(format!("Failed to parse f64 from string: {value:?}")))?;
-
- // Check if the parsed value is NaN or infinity
- if parsed.is_nan() || parsed.is_infinite() {
- Err(de::Error::custom(format!(
- "Value is not a valid f64 (NaN or Infinity): {value:?}"
- )))
- } else {
- Ok(parsed)
- }
-}
-
-/// deserialize Option<f64> from string or float
-pub(crate) fn deserialize_option_f64<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>
-where
- D: Deserializer<'de>,
-{
- struct StringOrFloatVisitor;
-
- impl<'de> Visitor<'de> for StringOrFloatVisitor {
- type Value = Option<f64>;
-
- fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
- formatter.write_str("a string or a float")
- }
-
- fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- parse_str_into_f64(value).map(Some)
- }
-
- fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(Some(value))
- }
-
- fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(Some(value as f64))
- }
-
- fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(Some(value as f64))
- }
-
- fn visit_none<E>(self) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(None)
- }
-
- fn visit_unit<E>(self) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(None)
- }
- }
-
- deserializer.deserialize_any(StringOrFloatVisitor)
-}
-
-/// deserialize f64 from string or float
-pub(crate) fn deserialize_f64<'de, D>(deserializer: D) -> Result<f64, D::Error>
-where
- D: Deserializer<'de>,
-{
- struct StringOrFloatVisitor;
-
- impl<'de> Visitor<'de> for StringOrFloatVisitor {
- type Value = f64;
-
- fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
- formatter.write_str("a string or a float")
- }
-
- fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- parse_str_into_f64(value)
- }
-
- fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(value)
- }
-
- fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(value as f64)
- }
-
- fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>
- where
- E: de::Error,
- {
- Ok(value as f64)
- }
- }
-
- deserializer.deserialize_any(StringOrFloatVisitor)
-}
-
-/// Represents an associative array `(key => values)` in a very efficient manner.
-#[derive(PartialEq, Serialize, Deserialize)]
-pub(crate) struct VecWithNames<T> {
- pub(crate) values: Vec<T>,
- keys: Vec<String>,
-}
-
-impl<T: Clone> Clone for VecWithNames<T> {
- fn clone(&self) -> Self {
- Self {
- values: self.values.clone(),
- keys: self.keys.clone(),
- }
- }
-}
-
-impl<T> Default for VecWithNames<T> {
- fn default() -> Self {
- Self {
- values: Default::default(),
- keys: Default::default(),
- }
- }
-}
-
-impl<T: std::fmt::Debug> std::fmt::Debug for VecWithNames<T> {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_map().entries(self.iter()).finish()
- }
-}
-
-impl<T> From<HashMap<String, T>> for VecWithNames<T> {
- fn from(map: HashMap<String, T>) -> Self {
- VecWithNames::from_entries(map.into_iter().collect_vec())
- }
-}
-
-impl<T> VecWithNames<T> {
- fn from_entries(mut entries: Vec<(String, T)>) -> Self {
- // Sort to ensure order of elements match across multiple instances
- entries.sort_by(|left, right| left.0.cmp(&right.0));
- let mut data = Vec::with_capacity(entries.len());
- let mut data_names = Vec::with_capacity(entries.len());
- for entry in entries {
- data_names.push(entry.0);
- data.push(entry.1);
- }
- VecWithNames {
- values: data,
- keys: data_names,
- }
- }
- fn iter(&self) -> impl Iterator<Item = (&str, &T)> + '_ {
- self.keys().zip(self.values.iter())
- }
- fn keys(&self) -> impl Iterator<Item = &str> + '_ {
- self.keys.iter().map(|key| key.as_str())
- }
- fn values_mut(&mut self) -> impl Iterator<Item = &mut T> + '_ {
- self.values.iter_mut()
- }
- fn is_empty(&self) -> bool {
- self.keys.is_empty()
- }
- fn len(&self) -> usize {
- self.keys.len()
- }
- fn get(&self, name: &str) -> Option<&T> {
- self.keys()
- .position(|key| key == name)
- .map(|pos| &self.values[pos])
- }
-}
-
-/// The serialized key is used in a `HashMap`.
-pub type SerializedKey = String;
-
-#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd)]
-/// The key to identify a bucket.
-#[serde(untagged)]
-pub enum Key {
- /// String key
- Str(String),
- /// `f64` key
- F64(f64),
-}
-impl Eq for Key {}
-impl std::hash::Hash for Key {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- core::mem::discriminant(self).hash(state);
- match self {
- Key::Str(text) => text.hash(state),
- Key::F64(val) => val.to_bits().hash(state),
- }
- }
-}
-
-impl PartialEq for Key {
- fn eq(&self, other: &Self) -> bool {
- match (self, other) {
- (Self::Str(l), Self::Str(r)) => l == r,
- (Self::F64(l), Self::F64(r)) => l == r,
- _ => false,
- }
- }
-}
-
-impl Display for Key {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- Key::Str(val) => f.write_str(val),
- Key::F64(val) => f.write_str(&val.to_string()),
- }
- }
-}
-
-/// Inverse of `to_fastfield_u64`. Used to convert to `f64` for metrics.
-///
-/// # Panics
-/// Only `u64`, `f64`, `date`, and `i64` are supported.
-pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &ColumnType) -> f64 {
- match field_type {
- ColumnType::U64 => val as f64,
- ColumnType::I64 | ColumnType::DateTime => i64::from_u64(val) as f64,
- ColumnType::F64 => f64::from_u64(val),
- ColumnType::Bool => val as f64,
- _ => {
- panic!("unexpected type {field_type:?}. This should not happen")
- }
- }
-}
-
-/// Converts the `f64` value to fast field value space, which is always u64.
-///
-/// If the fast field has `u64`, values are stored unchanged as `u64` in the fast field.
-///
-/// If the fast field has `f64` values are converted and stored to `u64` using a
-/// monotonic mapping.
-/// A `f64` value of e.g. `2.0` needs to be converted using the same monotonic
-/// conversion function, so that the value matches the `u64` value stored in the fast
-/// field.
-pub(crate) fn f64_to_fastfield_u64(val: f64, field_type: &ColumnType) -> Option<u64> {
- match field_type {
- ColumnType::U64 => Some(val as u64),
- ColumnType::I64 | ColumnType::DateTime => Some((val as i64).to_u64()),
- ColumnType::F64 => Some(val.to_u64()),
- ColumnType::Bool => Some(val as u64),
- _ => None,
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::net::Ipv6Addr;
-
- use crate::columnar::DateTime;
- use serde_json::Value;
- use time::OffsetDateTime;
-
- use super::agg_req::Aggregations;
- use super::*;
- use crate::indexer::NoMergePolicy;
- use crate::query::{AllQuery, TermQuery};
- use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};
- use crate::{Index, IndexWriter, Term};
-
- pub fn get_test_index_with_num_docs(
- merge_segments: bool,
- num_docs: usize,
- ) -> crate::Result<Index> {
- get_test_index_from_values(
- merge_segments,
- &(0..num_docs).map(|el| el as f64).collect::<Vec<f64>>(),
- )
- }
-
- pub fn exec_request(agg_req: Aggregations, index: &Index) -> crate::Result<Value> {
- exec_request_with_query(agg_req, index, None)
- }
- pub fn exec_request_with_query(
- agg_req: Aggregations,
- index: &Index,
- query: Option<(&str, &str)>,
- ) -> crate::Result<Value> {
- exec_request_with_query_and_memory_limit(agg_req, index, query, Default::default())
- }
-
- pub fn exec_request_with_query_and_memory_limit(
- agg_req: Aggregations,
- index: &Index,
- query: Option<(&str, &str)>,
- limits: AggregationLimits,
- ) -> crate::Result<Value> {
- let collector = AggregationCollector::from_aggs(agg_req, limits);
-
- let reader = index.reader()?;
- let searcher = reader.searcher();
- let agg_res = if let Some((field, term)) = query {
- let text_field = reader.searcher().schema().get_field(field).unwrap();
-
- let term_query = TermQuery::new(
- Term::from_field_text(text_field, term),
- IndexRecordOption::Basic,
- );
-
- searcher.search(&term_query, &collector)?
- } else {
- searcher.search(&AllQuery, &collector)?
- };
-
- // Test serialization/deserialization roundtrip
- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;
- Ok(res)
- }
-
- pub fn get_test_index_from_values(
- merge_segments: bool,
- values: &[f64],
- ) -> crate::Result<Index> {
- // Every value gets its own segment
- let mut segment_and_values = vec![];
- for value in values {
- segment_and_values.push(vec![(*value, value.to_string())]);
- }
- get_test_index_from_values_and_terms(merge_segments, &segment_and_values)
- }
-
- pub fn get_test_index_from_terms(
- merge_segments: bool,
- values: &[Vec<&str>],
- ) -> crate::Result<Index> {
- // Every value gets its own segment
- let segment_and_values = values
- .iter()
- .map(|terms| {
- terms
- .iter()
- .enumerate()
- .map(|(i, term)| (i as f64, term.to_string()))
- .collect()
- })
- .collect::<Vec<_>>();
- get_test_index_from_values_and_terms(merge_segments, &segment_and_values)
- }
-
- pub fn get_test_index_from_values_and_terms(
- merge_segments: bool,
- segment_and_values: &[Vec<(f64, String)>],
- ) -> crate::Result<Index> {
- let mut schema_builder = Schema::builder();
- let text_fieldtype = crate::schema::TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default()
- .set_index_option(IndexRecordOption::Basic)
- .set_fieldnorms(false),
- )
- .set_fast(None)
- .set_stored();
- let text_field = schema_builder.add_text_field("text", text_fieldtype.clone());
- let text_field_id = schema_builder.add_text_field("text_id", text_fieldtype);
- let string_field_id = schema_builder.add_text_field("string_id", STRING | FAST);
- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();
- let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());
- let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
- let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
- let fraction_field = schema_builder.add_f64_field(
- "fraction_f64",
- crate::schema::NumericOptions::default().set_fast(),
- );
- let index = Index::create_in_ram(schema_builder.build());
- {
- // let mut index_writer = index.writer_for_tests()?;
- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;
- index_writer.set_merge_policy(Box::new(NoMergePolicy));
- for values in segment_and_values {
- for (i, term) in values {
- let i = *i;
- // writing the segment
- index_writer.add_document(doc!(
- text_field => "cool",
- text_field_id => term.to_string(),
- string_field_id => term.to_string(),
- score_field => i as u64,
- score_field_f64 => i,
- score_field_i64 => i as i64,
- fraction_field => i/100.0,
- ))?;
- }
- index_writer.commit()?;
- }
- }
- if merge_segments {
- let segment_ids = index
- .searchable_segment_ids()
- .expect("Searchable segments failed.");
- if segment_ids.len() > 1 {
- let mut index_writer: IndexWriter = index.writer_for_tests()?;
- index_writer.merge(&segment_ids).wait()?;
- index_writer.wait_merging_threads()?;
- }
- }
-
- Ok(index)
- }
-
- pub fn get_test_index_2_segments(merge_segments: bool) -> crate::Result<Index> {
- let mut schema_builder = Schema::builder();
- let text_fieldtype = crate::schema::TextOptions::default()
- .set_indexing_options(
- TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),
- )
- .set_fast(None)
- .set_stored();
- let text_field = schema_builder.add_text_field("text", text_fieldtype);
- let date_field = schema_builder.add_date_field("date", FAST);
- schema_builder.add_text_field("dummy_text", STRING);
- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();
- let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());
- let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());
- let ip_addr_field = schema_builder.add_ip_addr_field("ip_addr", FAST);
-
- let multivalue = crate::schema::NumericOptions::default().set_fast();
- let scores_field_i64 = schema_builder.add_i64_field("scores_i64", multivalue);
-
- let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);
- let index = Index::create_in_ram(schema_builder.build());
- {
- let mut index_writer = index.writer_for_tests()?;
- // writing the segment
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800).unwrap()),
- score_field => 1u64,
- ip_addr_field => Ipv6Addr::from(1u128),
- score_field_f64 => 1f64,
- score_field_i64 => 1i64,
- scores_field_i64 => 1i64,
- scores_field_i64 => 2i64,
- ))?;
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),
- score_field => 3u64,
- score_field_f64 => 3f64,
- score_field_i64 => 3i64,
- scores_field_i64 => 5i64,
- scores_field_i64 => 5i64,
- ))?;
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),
- score_field => 5u64,
- score_field_f64 => 5f64,
- score_field_i64 => 5i64,
- ))?;
- index_writer.add_document(doc!(
- text_field => "nohit",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),
- score_field => 6u64,
- score_field_f64 => 6f64,
- score_field_i64 => 6i64,
- ))?;
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),
- score_field => 7u64,
- score_field_f64 => 7f64,
- score_field_i64 => 7i64,
- ))?;
- index_writer.commit()?;
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),
- score_field => 11u64,
- score_field_f64 => 11f64,
- score_field_i64 => 11i64,
- ))?;
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),
- score_field => 14u64,
- score_field_f64 => 14f64,
- score_field_i64 => 14i64,
- ))?;
-
- index_writer.add_document(doc!(
- text_field => "cool",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),
- score_field => 44u64,
- score_field_f64 => 44.5f64,
- score_field_i64 => 44i64,
- ))?;
-
- index_writer.commit()?;
-
- // no hits segment
- index_writer.add_document(doc!(
- text_field => "nohit",
- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),
- score_field => 44u64,
- score_field_f64 => 44.5f64,
- score_field_i64 => 44i64,
- ))?;
-
- index_writer.commit()?;
- }
- if merge_segments {
- let segment_ids = index
- .searchable_segment_ids()
- .expect("Searchable segments failed.");
- let mut index_writer: IndexWriter = index.writer_for_tests()?;
- index_writer.merge(&segment_ids).wait()?;
- index_writer.wait_merging_threads()?;
- }
-
- Ok(index)
- }
-}
diff --git a/crates/tantivy/src/aggregation/segment_agg_result.rs b/crates/tantivy/src/aggregation/segment_agg_result.rs
deleted file mode 100644
index cc0aab5a..00000000
--- a/crates/tantivy/src/aggregation/segment_agg_result.rs
+++ /dev/null
@@ -1,245 +0,0 @@
-//! Contains aggregation trees which is used during collection in a segment.
-//! This tree contains datastructrues optimized for fast collection.
-//! The tree can be converted to an intermediate tree, which contains datastructrues optimized for
-//! merging.
-
-use std::fmt::Debug;
-
-pub(crate) use super::agg_limits::AggregationLimits;
-use super::agg_req::AggregationVariants;
-use super::agg_req_with_accessor::{AggregationWithAccessor, AggregationsWithAccessor};
-use super::bucket::{SegmentHistogramCollector, SegmentRangeCollector, SegmentTermCollector};
-use super::intermediate_agg_result::IntermediateAggregationResults;
-use super::metric::{
- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,
- SegmentPercentilesCollector, SegmentStatsCollector, SegmentStatsType, StatsAggregation,
- SumAggregation,
-};
-use crate::aggregation::bucket::TermMissingAgg;
-use crate::aggregation::metric::{SegmentExtendedStatsCollector, TopHitsSegmentCollector};
-
-pub(crate) trait SegmentAggregationCollector: CollectorClone + Debug {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()>;
-
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()>;
-
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()>;
-
- /// Finalize method. Some Aggregator collect blocks of docs before calling `collect_block`.
- /// This method ensures those staged docs will be collected.
- fn flush(&mut self, _agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- Ok(())
- }
-}
-
-pub(crate) trait CollectorClone {
- fn clone_box(&self) -> Box<dyn SegmentAggregationCollector>;
-}
-
-impl<T> CollectorClone for T
-where T: 'static + SegmentAggregationCollector + Clone
-{
- fn clone_box(&self) -> Box<dyn SegmentAggregationCollector> {
- Box::new(self.clone())
- }
-}
-
-impl Clone for Box<dyn SegmentAggregationCollector> {
- fn clone(&self) -> Box<dyn SegmentAggregationCollector> {
- self.clone_box()
- }
-}
-
-pub(crate) fn build_segment_agg_collector(
- req: &mut AggregationsWithAccessor,
-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
- // Single collector special case
- if req.aggs.len() == 1 {
- let req = &mut req.aggs.values[0];
- let accessor_idx = 0;
- return build_single_agg_segment_collector(req, accessor_idx);
- }
-
- let agg = GenericSegmentAggregationResultsCollector::from_req_and_validate(req)?;
- Ok(Box::new(agg))
-}
-
-pub(crate) fn build_single_agg_segment_collector(
- req: &mut AggregationWithAccessor,
- accessor_idx: usize,
-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {
- use AggregationVariants::*;
- match &req.agg.agg {
- Terms(terms_req) => {
- if req.accessors.is_empty() {
- Ok(Box::new(SegmentTermCollector::from_req_and_validate(
- terms_req,
- &mut req.sub_aggregation,
- req.field_type,
- accessor_idx,
- )?))
- } else {
- Ok(Box::new(TermMissingAgg::new(
- accessor_idx,
- &mut req.sub_aggregation,
- )?))
- }
- }
- Range(range_req) => Ok(Box::new(SegmentRangeCollector::from_req_and_validate(
- range_req,
- &mut req.sub_aggregation,
- &req.limits,
- req.field_type,
- accessor_idx,
- )?)),
- Histogram(histogram) => Ok(Box::new(SegmentHistogramCollector::from_req_and_validate(
- histogram.clone(),
- &mut req.sub_aggregation,
- req.field_type,
- accessor_idx,
- )?)),
- DateHistogram(histogram) => Ok(Box::new(SegmentHistogramCollector::from_req_and_validate(
- histogram.to_histogram_req()?,
- &mut req.sub_aggregation,
- req.field_type,
- accessor_idx,
- )?)),
- Average(AverageAggregation { missing, .. }) => {
- Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Average,
- accessor_idx,
- *missing,
- )))
- }
- Count(CountAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Count,
- accessor_idx,
- *missing,
- ))),
- Max(MaxAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Max,
- accessor_idx,
- *missing,
- ))),
- Min(MinAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Min,
- accessor_idx,
- *missing,
- ))),
- Stats(StatsAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Stats,
- accessor_idx,
- *missing,
- ))),
- ExtendedStats(ExtendedStatsAggregation { missing, sigma, .. }) => Ok(Box::new(
- SegmentExtendedStatsCollector::from_req(req.field_type, *sigma, accessor_idx, *missing),
- )),
- Sum(SumAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(
- req.field_type,
- SegmentStatsType::Sum,
- accessor_idx,
- *missing,
- ))),
- Percentiles(percentiles_req) => Ok(Box::new(
- SegmentPercentilesCollector::from_req_and_validate(
- percentiles_req,
- req.field_type,
- accessor_idx,
- )?,
- )),
- TopHits(top_hits_req) => Ok(Box::new(TopHitsSegmentCollector::from_req(
- top_hits_req,
- accessor_idx,
- req.segment_ordinal,
- ))),
- }
-}
-
-#[derive(Clone, Default)]
-/// The GenericSegmentAggregationResultsCollector is the generic version of the collector, which
-/// can handle arbitrary complexity of sub-aggregations. Ideally we never have to pick this one
-/// and can provide specialized versions instead, that remove some of its overhead.
-pub(crate) struct GenericSegmentAggregationResultsCollector {
- pub(crate) aggs: Vec<Box<dyn SegmentAggregationCollector>>,
-}
-
-impl Debug for GenericSegmentAggregationResultsCollector {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("SegmentAggregationResultsCollector")
- .field("aggs", &self.aggs)
- .finish()
- }
-}
-
-impl SegmentAggregationCollector for GenericSegmentAggregationResultsCollector {
- fn add_intermediate_aggregation_result(
- self: Box<Self>,
- agg_with_accessor: &AggregationsWithAccessor,
- results: &mut IntermediateAggregationResults,
- ) -> crate::Result<()> {
- for agg in self.aggs {
- agg.add_intermediate_aggregation_result(agg_with_accessor, results)?;
- }
-
- Ok(())
- }
-
- fn collect(
- &mut self,
- doc: crate::DocId,
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- self.collect_block(&[doc], agg_with_accessor)?;
-
- Ok(())
- }
-
- fn collect_block(
- &mut self,
- docs: &[crate::DocId],
- agg_with_accessor: &mut AggregationsWithAccessor,
- ) -> crate::Result<()> {
- for collector in &mut self.aggs {
- collector.collect_block(docs, agg_with_accessor)?;
- }
-
- Ok(())
- }
-
- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {
- for collector in &mut self.aggs {
- collector.flush(agg_with_accessor)?;
- }
- Ok(())
- }
-}
-
-impl GenericSegmentAggregationResultsCollector {
- pub(crate) fn from_req_and_validate(req: &mut AggregationsWithAccessor) -> crate::Result<Self> {
- let aggs = req
- .aggs
- .values_mut()
- .enumerate()
- .map(|(accessor_idx, req)| build_single_agg_segment_collector(req, accessor_idx))
- .collect::<crate::Result<Vec<Box<dyn SegmentAggregationCollector>>>>()?;
-
- Ok(GenericSegmentAggregationResultsCollector { aggs })
- }
-}
diff --git a/crates/tantivy/src/error.rs b/crates/tantivy/src/error.rs
index 487cc21a..ade94d54 100644
--- a/crates/tantivy/src/error.rs
+++ b/crates/tantivy/src/error.rs
@@ -6,7 +6,6 @@ use std::{fmt, io};
use thiserror::Error;
-use crate::aggregation::AggregationError;
use crate::directory::error::{
Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,
};
@@ -55,9 +54,6 @@ impl fmt::Debug for DataCorruption {
/// The library's error enum
#[derive(Debug, Clone, Error)]
pub enum TantivyError {
- /// Error when handling aggregations.
- #[error(transparent)]
- AggregationError(#[from] AggregationError),
/// Failed to open the directory.
#[error("Failed to open the directory: '{0:?}'")]
OpenDirectoryError(#[from] OpenDirectoryError),
diff --git a/crates/tantivy/src/indexer/mod.rs b/crates/tantivy/src/indexer/mod.rs
index 64b121d4..60bed2cc 100644
--- a/crates/tantivy/src/indexer/mod.rs
+++ b/crates/tantivy/src/indexer/mod.rs
@@ -61,12 +61,9 @@ type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;
#[cfg(test)]
mod tests_mmap {
- use crate::aggregation::agg_req::Aggregations;
- use crate::aggregation::agg_result::AggregationResults;
- use crate::aggregation::AggregationCollector;
use crate::collector::{Count, TopDocs};
use crate::index::FieldMetadata;
- use crate::query::{AllQuery, QueryParser};
+ use crate::query::QueryParser;
use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};
use crate::{Index, IndexWriter, Term};
@@ -226,41 +223,6 @@ mod tests_mmap {
test_query(format!("json.a{field_name_out}.a{field_name_out}:test6").as_str());
test_query(format!("json.{field_name_out}a:test7").as_str());
- let test_agg = |field_name: &str, expected: &str| {
- let agg_req_str = json!(
- {
- "termagg": {
- "terms": {
- "field": field_name,
- }
- }
- });
-
- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
- let collector = AggregationCollector::from_aggs(agg_req, Default::default());
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let res = serde_json::to_value(agg_res).unwrap();
- assert_eq!(res["termagg"]["buckets"][0]["doc_count"], 1);
- assert_eq!(res["termagg"]["buckets"][0]["key"], expected);
- };
-
- test_agg(format!("json.{field_name_out}").as_str(), "test1");
- test_agg(format!("json.a{field_name_out}").as_str(), "test2");
- test_agg(format!("json.a{field_name_out}a").as_str(), "test3");
- test_agg(
- format!("json.a{field_name_out}a{field_name_out}").as_str(),
- "test4",
- );
- test_agg(
- format!("json.a{field_name_out}.ab{field_name_out}").as_str(),
- "test5",
- );
- test_agg(
- format!("json.a{field_name_out}.a{field_name_out}").as_str(),
- "test6",
- );
- test_agg(format!("json.{field_name_out}a").as_str(), "test7");
-
// `.` is stored as `\u{0001}` internally in tantivy
let field_name_out_internal = if field_name_out == "." {
"\u{0001}"
@@ -530,94 +492,5 @@ mod tests_mmap {
assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);
}
}
- // Test if returned field name can be used for aggregation
- for fast_field in fields_metadata.iter().filter(|meta| meta.fast) {
- let agg_req_str = json!(
- {
- "termagg": {
- "terms": {
- "field": fast_field.field_name,
- }
- }
- });
-
- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
- let collector = AggregationCollector::from_aggs(agg_req, Default::default());
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let res = serde_json::to_value(agg_res).unwrap();
- if !fast_field.field_name.contains("empty") && fast_field.typ != Type::Json {
- assert!(
- !res["termagg"]["buckets"].as_array().unwrap().is_empty(),
- "{}",
- fast_field.field_name
- );
- }
- }
- }
-
- #[test]
- fn test_json_field_shadowing_field_name_bug() {
- /// This test is only there to display a bug on addressing a field if it gets shadowed
- /// The issues only occurs if the field name that shadows contains a dot.
- ///
- /// Happens independently of the `expand_dots` option. Since that option does not
- /// affect the field name itself.
- use pretty_assertions::assert_eq;
- let mut schema_builder = Schema::builder();
- let json_options: JsonObjectOptions =
- JsonObjectOptions::from(TEXT).set_fast(None).set_stored();
- // let json_options = json_options.set_expand_dots_enabled();
- let json_field_shadow = schema_builder.add_json_field("json.shadow", json_options.clone());
- let json_field = schema_builder.add_json_field("json", json_options.clone());
- let index = Index::create_in_ram(schema_builder.build());
- let mut index_writer = index.writer_for_tests().unwrap();
- index_writer
- .add_document(
- doc!(json_field_shadow=>json!({"val": "b"}), json_field=>json!({"shadow": {"val": "a"}})),
- )
- .unwrap();
- index_writer.commit().unwrap();
- let reader = index.reader().unwrap();
-
- let searcher = reader.searcher();
-
- let fields_and_vals = [
- ("json.shadow\u{1}val".to_string(), "a"), // Succeeds
- //("json.shadow.val".to_string(), "a"), // Fails
- ("json.shadow.val".to_string(), "b"),
- ];
-
- let query_parser = QueryParser::for_index(&index, vec![]);
- // Test if field name can be queried
- for (indexed_field, val) in fields_and_vals.iter() {
- let query_str = &format!("{indexed_field}:{val}");
- let query = query_parser.parse_query(query_str).unwrap();
- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();
- assert!(!count_docs.is_empty(), "{indexed_field}:{val}");
- }
- // Test if field name can be used for aggregation
- for (field_name, val) in fields_and_vals.iter() {
- let agg_req_str = json!(
- {
- "termagg": {
- "terms": {
- "field": field_name,
- }
- }
- });
-
- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();
- let collector = AggregationCollector::from_aggs(agg_req, Default::default());
- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();
- let res = serde_json::to_value(agg_res).unwrap();
- assert_eq!(
- res["termagg"]["buckets"].as_array().unwrap()[0]["key"]
- .as_str()
- .unwrap(),
- *val,
- "{}",
- field_name
- );
- }
}
}
diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rs
index 726a2008..14e52c50 100644
--- a/crates/tantivy/src/lib.rs
+++ b/crates/tantivy/src/lib.rs
@@ -191,7 +191,6 @@ mod sstable;
pub mod error;
pub mod tokenizer;
-pub mod aggregation;
pub mod collector;
pub mod directory;
pub mod fastfield;