web search done right
remove aggregations for simplicity
Mikkel Denker4fb6af6fed4f
diff --git a/crates/tantivy/examples/aggregation.rs b/crates/tantivy/examples/aggregation.rsdeleted file mode 100644index 52080b8d..00000000--- a/crates/tantivy/examples/aggregation.rs+++ /dev/null@@ -1,266 +0,0 @@-// # Aggregation example-//-// This example shows how you can use built-in aggregations.-// We will use nested aggregations with buckets and metrics:-// - Range buckets and compute the average in each bucket.-// - Term aggregation and compute the min price in each bucket-// -----use serde_json::{Deserializer, Value};-use tantivy::aggregation::agg_req::Aggregations;-use tantivy::aggregation::agg_result::AggregationResults;-use tantivy::aggregation::AggregationCollector;-use tantivy::query::AllQuery;-use tantivy::schema::{self, IndexRecordOption, Schema, TextFieldIndexing, FAST};-use tantivy::{Index, IndexWriter, TantivyDocument};--fn main() -> tantivy::Result<()> {- // # Create Schema- //- // Lets create a schema for a footwear shop, with 4 fields: name, category, stock and price.- // category, stock and price will be fast fields as that's the requirement- // for aggregation queries.- //-- let mut schema_builder = Schema::builder();- // In preparation of the `TermsAggregation`, the category field is configured with:- // - `set_fast`- // - `raw` tokenizer- //- // The tokenizer is set to "raw", because the fast field uses the same dictionary as the- // inverted index. (This behaviour will change in tantivy 0.20, where the fast field will- // always be raw tokenized independent from the regular tokenizing)- //- let text_fieldtype = schema::TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_index_option(IndexRecordOption::WithFreqs)- .set_tokenizer("raw"),- )- .set_fast(None)- .set_stored();- schema_builder.add_text_field("category", text_fieldtype);- schema_builder.add_f64_field("stock", FAST);- schema_builder.add_f64_field("price", FAST);-- let schema = schema_builder.build();-- // # Indexing documents- //- // Lets index a bunch of documents for this example.- let index = Index::create_in_ram(schema.clone());-- let data = r#"{- "name": "Almond Toe Court Shoes, Patent Black",- "category": "Womens Footwear",- "price": 99.00,- "stock": 5- }- {- "name": "Suede Shoes, Blue",- "category": "Womens Footwear",- "price": 42.00,- "stock": 4- }- {- "name": "Leather Driver Saddle Loafers, Tan",- "category": "Mens Footwear",- "price": 34.00,- "stock": 12- }- {- "name": "Flip Flops, Red",- "category": "Mens Footwear",- "price": 19.00,- "stock": 6- }- {- "name": "Flip Flops, Blue",- "category": "Mens Footwear",- "price": 19.00,- "stock": 0- }- {- "name": "Gold Button Cardigan, Black",- "category": "Womens Casualwear",- "price": 167.00,- "stock": 6- }- {- "name": "Cotton Shorts, Medium Red",- "category": "Womens Casualwear",- "price": 30.00,- "stock": 5- }- {- "name": "Fine Stripe Short SleeveShirt, Grey",- "category": "Mens Casualwear",- "price": 49.99,- "stock": 9- }- {- "name": "Fine Stripe Short SleeveShirt, Green",- "category": "Mens Casualwear",- "price": 49.99,- "offer": 39.99,- "stock": 9- }- {- "name": "Sharkskin Waistcoat, Charcoal",- "category": "Mens Formalwear",- "price": 75.00,- "stock": 2- }- {- "name": "Lightweight Patch PocketBlazer, Deer",- "category": "Mens Formalwear",- "price": 175.50,- "stock": 1- }- {- "name": "Bird Print Dress, Black",- "category": "Womens Formalwear",- "price": 270.00,- "stock": 10- }- {- "name": "Mid Twist Cut-Out Dress, Pink",- "category": "Womens Formalwear",- "price": 540.00,- "stock": 5- }"#;-- let stream = Deserializer::from_str(data).into_iter::<Value>();-- let mut index_writer: IndexWriter = index.writer(50_000_000)?;- let mut num_indexed = 0;- for value in stream {- let doc = TantivyDocument::parse_json(&schema, &serde_json::to_string(&value.unwrap())?)?;- index_writer.add_document(doc)?;- num_indexed += 1;- if num_indexed > 4 {- // Writing the first segment- index_writer.commit()?;- }- }-- // Writing the second segment- index_writer.commit()?;-- // We have two segments now. The `AggregationCollector` will run the aggregation on each- // segment and then merge the results into an `IntermediateAggregationResult`.-- let reader = index.reader()?;- let searcher = reader.searcher();- // ---- // # Aggregation Query- //- //- // We can construct the query by building the request structure or by deserializing from JSON.- // The JSON API is more stable and therefore recommended.- //- // ## Request 1-- let agg_req_str = r#"- {- "group_by_stock": {- "aggs": {- "average_price": { "avg": { "field": "price" } }- },- "range": {- "field": "stock",- "ranges": [- { "key": "few", "to": 1.0 },- { "key": "some", "from": 1.0, "to": 10.0 },- { "key": "many", "from": 10.0 }- ]- }- }- } "#;-- // In this Aggregation we want to get the average price for different groups, depending on how- // many items are in stock. We define custom ranges `few`, `some`, `many` via the- // range aggregation.- // For every bucket we want the average price, so we create a nested metric aggregation on the- // range bucket aggregation. Only buckets support nested aggregations.- // ### Request JSON API- //-- let agg_req: Aggregations = serde_json::from_str(agg_req_str)?;- let collector = AggregationCollector::from_aggs(agg_req, Default::default());-- // We use the `AllQuery` which will pass all documents to the AggregationCollector.- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let res: Value = serde_json::to_value(agg_res)?;-- // ### Aggregation Result- //- // The resulting structure deserializes in the same JSON format as elastic search.- //- let expected_res = r#"- {- "group_by_stock":{- "buckets":[- {"average_price":{"value":19.0},"doc_count":1,"key":"few","to":1.0},- {"average_price":{"value":124.748},"doc_count":10,"from":1.0,"key":"some","to":10.0},- {"average_price":{"value":152.0},"doc_count":2,"from":10.0,"key":"many"}- ]- }- }- "#;- let expected_json: Value = serde_json::from_str(expected_res)?;- assert_eq!(expected_json, res);-- // ### Request 2- //- // Now we are interested in the minimum price per category, so we create a bucket per- // category via `TermsAggregation`. We are interested in the highest minimum prices, and set the- // order of the buckets `"order": { "min_price": "desc" }` to be sorted by the the metric of- // the sub aggregation. (awesome)- //- let agg_req_str = r#"- {- "min_price_per_category": {- "aggs": {- "min_price": { "min": { "field": "price" } }- },- "terms": {- "field": "category",- "min_doc_count": 1,- "order": { "min_price": "desc" }- }- }- } "#;-- let agg_req: Aggregations = serde_json::from_str(agg_req_str)?;-- let collector = AggregationCollector::from_aggs(agg_req, Default::default());-- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let res: Value = serde_json::to_value(agg_res)?;-- // Minimum price per category, sorted by minimum price descending- //- // As you can see, the starting prices for `Formalwear` are higher than `Casualwear`.- //- let expected_res = r#"- {- "min_price_per_category": {- "buckets": [- { "doc_count": 2, "key": "Womens Formalwear", "min_price": { "value": 270.0 } },- { "doc_count": 2, "key": "Mens Formalwear", "min_price": { "value": 75.0 } },- { "doc_count": 2, "key": "Mens Casualwear", "min_price": { "value": 49.99 } },- { "doc_count": 2, "key": "Womens Footwear", "min_price": { "value": 42.0 } },- { "doc_count": 2, "key": "Womens Casualwear", "min_price": { "value": 30.0 } },- { "doc_count": 3, "key": "Mens Footwear", "min_price": { "value": 19.0 } }- ],- "sum_other_doc_count": 0- }- }- "#;- let expected_json: Value = serde_json::from_str(expected_res)?;-- assert_eq!(expected_json, res);-- Ok(())-}diff --git a/crates/tantivy/src/aggregation/README.md b/crates/tantivy/src/aggregation/README.mddeleted file mode 100644index c96454d0..00000000--- a/crates/tantivy/src/aggregation/README.md+++ /dev/null@@ -1,36 +0,0 @@-# Contributing--When adding new bucket aggregation make sure to extend the "test_aggregation_flushing" test for at least 2 levels.----# Code Organization--Tantivy's aggregations have been designed to mimic the-[aggregations of elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html).--The code is organized in submodules:--## bucket-Contains all bucket aggregations, like range aggregation. These bucket aggregations group documents into buckets and can contain sub-aggregations.--## metric-Contains all metric aggregations, like average aggregation. Metric aggregations do not have sub aggregations.--#### agg_req-agg_req contains the users aggregation request. Deserialization from json is compatible with elasticsearch aggregation requests.--#### agg_req_with_accessor-agg_req_with_accessor contains the users aggregation request enriched with fast field accessors etc, which are-used during collection.--#### segment_agg_result-segment_agg_result contains the aggregation result tree, which is used for collection of a segment.-The tree from agg_req_with_accessor is passed during collection.--#### intermediate_agg_result-intermediate_agg_result contains the aggregation tree for merging with other trees.--#### agg_result-agg_result contains the final aggregation tree.-diff --git a/crates/tantivy/src/aggregation/agg_limits.rs b/crates/tantivy/src/aggregation/agg_limits.rsdeleted file mode 100644index 44d802cc..00000000--- a/crates/tantivy/src/aggregation/agg_limits.rs+++ /dev/null@@ -1,138 +0,0 @@-use std::collections::HashMap;-use std::sync::atomic::{AtomicU64, Ordering};-use std::sync::Arc;--use crate::common::ByteCount;--use super::collector::DEFAULT_MEMORY_LIMIT;-use super::{AggregationError, DEFAULT_BUCKET_LIMIT};--/// An estimate for memory consumption. Non recursive-pub trait MemoryConsumption {- fn memory_consumption(&self) -> usize;-}--impl<K, V, S> MemoryConsumption for HashMap<K, V, S> {- fn memory_consumption(&self) -> usize {- let capacity = self.capacity();- (std::mem::size_of::<K>() + std::mem::size_of::<V>() + 1) * capacity- }-}--/// Aggregation memory limit after which the request fails. Defaults to DEFAULT_MEMORY_LIMIT-/// (500MB). The limit is shared by all SegmentCollectors-pub struct AggregationLimits {- /// The counter which is shared between the aggregations for one request.- memory_consumption: Arc<AtomicU64>,- /// The memory_limit in bytes- memory_limit: ByteCount,- /// The maximum number of buckets _returned_- /// This is not counting intermediate buckets.- bucket_limit: u32,-}-impl Clone for AggregationLimits {- fn clone(&self) -> Self {- Self {- memory_consumption: Arc::clone(&self.memory_consumption),- memory_limit: self.memory_limit,- bucket_limit: self.bucket_limit,- }- }-}--impl Default for AggregationLimits {- fn default() -> Self {- Self {- memory_consumption: Default::default(),- memory_limit: DEFAULT_MEMORY_LIMIT.into(),- bucket_limit: DEFAULT_BUCKET_LIMIT,- }- }-}--impl AggregationLimits {- /// *memory_limit*- /// memory_limit is defined in bytes.- /// Aggregation fails when the estimated memory consumption of the aggregation is higher than- /// memory_limit.- /// memory_limit will default to `DEFAULT_MEMORY_LIMIT` (500MB)- ///- /// *bucket_limit*- /// Limits the maximum number of buckets returned from an aggregation request.- /// bucket_limit will default to `DEFAULT_BUCKET_LIMIT` (65000)- ///- /// Note: The returned instance contains a Arc shared counter to track memory consumption.- pub fn new(memory_limit: Option<u64>, bucket_limit: Option<u32>) -> Self {- Self {- memory_consumption: Default::default(),- memory_limit: memory_limit.unwrap_or(DEFAULT_MEMORY_LIMIT).into(),- bucket_limit: bucket_limit.unwrap_or(DEFAULT_BUCKET_LIMIT),- }- }-- /// Create a new ResourceLimitGuard, that will release the memory when dropped.- pub fn new_guard(&self) -> ResourceLimitGuard {- ResourceLimitGuard {- // The counter which is shared between the aggregations for one request.- memory_consumption: Arc::clone(&self.memory_consumption),- // The memory_limit in bytes- memory_limit: self.memory_limit,- allocated_with_the_guard: 0,- }- }-- pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {- let prev_value = self- .memory_consumption- .fetch_add(add_num_bytes, Ordering::Relaxed);- validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;- Ok(())- }-- pub(crate) fn get_bucket_limit(&self) -> u32 {- self.bucket_limit- }-}--fn validate_memory_consumption(- memory_consumption: u64,- memory_limit: ByteCount,-) -> Result<(), AggregationError> {- // Load the estimated memory consumed by the aggregations- let memory_consumed: ByteCount = memory_consumption.into();- if memory_consumed > memory_limit {- return Err(AggregationError::MemoryExceeded {- limit: memory_limit,- current: memory_consumed,- });- }- Ok(())-}--pub struct ResourceLimitGuard {- /// The counter which is shared between the aggregations for one request.- memory_consumption: Arc<AtomicU64>,- /// The memory_limit in bytes- memory_limit: ByteCount,- /// Allocated memory with this guard.- allocated_with_the_guard: u64,-}--impl ResourceLimitGuard {- pub(crate) fn add_memory_consumed(&self, add_num_bytes: u64) -> crate::Result<()> {- let prev_value = self- .memory_consumption- .fetch_add(add_num_bytes, Ordering::Relaxed);- validate_memory_consumption(prev_value + add_num_bytes, self.memory_limit)?;- Ok(())- }-}--impl Drop for ResourceLimitGuard {- /// Removes the memory consumed tracked by this _instance_ of AggregationLimits.- /// This is used to clear the segment specific memory consumption all at once.- fn drop(&mut self) {- self.memory_consumption- .fetch_sub(self.allocated_with_the_guard, Ordering::Relaxed);- }-}diff --git a/crates/tantivy/src/aggregation/agg_req.rs b/crates/tantivy/src/aggregation/agg_req.rsdeleted file mode 100644index f849cad3..00000000--- a/crates/tantivy/src/aggregation/agg_req.rs+++ /dev/null@@ -1,219 +0,0 @@-//! Contains the aggregation request tree. Used to build an-//! [`AggregationCollector`](super::AggregationCollector).-//!-//! [`Aggregations`] is the top level entry point to create a request, which is a `HashMap<String,-//! Aggregation>`.-//!-//! Requests are compatible with the json format of elasticsearch.-//!-//! # Example-//!-//! ```-//! use tantivy::aggregation::agg_req::Aggregations;-//!-//! let elasticsearch_compatible_json_req = r#"-//! {-//! "range": {-//! "range": {-//! "field": "score",-//! "ranges": [-//! { "from": 3.0, "to": 7.0 },-//! { "from": 7.0, "to": 20.0 }-//! ]-//! }-//! }-//! }"#;-//! let _agg_req: Aggregations = serde_json::from_str(elasticsearch_compatible_json_req).unwrap();-//! ```--use std::collections::{HashMap, HashSet};--use serde::{Deserialize, Serialize};--use super::bucket::{- DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,-};-use super::metric::{- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,- PercentilesAggregationReq, StatsAggregation, SumAggregation, TopHitsAggregation,-};--/// The top-level aggregation request structure, which contains [`Aggregation`] and their user-/// defined names. It is also used in buckets aggregations to define sub-aggregations.-///-/// The key is the user defined name of the aggregation.-pub type Aggregations = HashMap<String, Aggregation>;--/// Aggregation request.-///-/// An aggregation is either a bucket or a metric.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(try_from = "AggregationForDeserialization")]-pub struct Aggregation {- /// The aggregation variant, which can be either a bucket or a metric.- #[serde(flatten)]- pub agg: AggregationVariants,- /// on the document set in the bucket.- #[serde(rename = "aggs")]- #[serde(skip_serializing_if = "Aggregations::is_empty")]- pub sub_aggregation: Aggregations,-}--/// In order to display proper error message, we cannot rely on flattening-/// the json enum. Instead we introduce an intermediary struct to separate-/// the aggregation from the subaggregation.-#[derive(Deserialize)]-struct AggregationForDeserialization {- #[serde(flatten)]- pub aggs_remaining_json: serde_json::Value,- #[serde(rename = "aggs")]- #[serde(default)]- pub sub_aggregation: Aggregations,-}--impl TryFrom<AggregationForDeserialization> for Aggregation {- type Error = serde_json::Error;-- fn try_from(value: AggregationForDeserialization) -> serde_json::Result<Self> {- let AggregationForDeserialization {- aggs_remaining_json,- sub_aggregation,- } = value;- let agg: AggregationVariants = serde_json::from_value(aggs_remaining_json)?;- Ok(Aggregation {- agg,- sub_aggregation,- })- }-}--impl Aggregation {- pub(crate) fn sub_aggregation(&self) -> &Aggregations {- &self.sub_aggregation- }-- fn get_fast_field_names(&self, fast_field_names: &mut HashSet<String>) {- fast_field_names.extend(- self.agg- .get_fast_field_names()- .iter()- .map(|s| s.to_string()),- );- fast_field_names.extend(get_fast_field_names(&self.sub_aggregation));- }-}--/// Extract all fast field names used in the tree.-pub fn get_fast_field_names(aggs: &Aggregations) -> HashSet<String> {- let mut fast_field_names = Default::default();- for el in aggs.values() {- el.get_fast_field_names(&mut fast_field_names)- }- fast_field_names-}--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-/// All aggregation types.-pub enum AggregationVariants {- // Bucket aggregation types- /// Put data into buckets of user-defined ranges.- #[serde(rename = "range")]- Range(RangeAggregation),- /// Put data into a histogram.- #[serde(rename = "histogram")]- Histogram(HistogramAggregation),- /// Put data into a date histogram.- #[serde(rename = "date_histogram")]- DateHistogram(DateHistogramAggregationReq),- /// Put data into buckets of terms.- #[serde(rename = "terms")]- Terms(TermsAggregation),-- // Metric aggregation types- /// Computes the average of the extracted values.- #[serde(rename = "avg")]- Average(AverageAggregation),- /// Counts the number of extracted values.- #[serde(rename = "value_count")]- Count(CountAggregation),- /// Finds the maximum value.- #[serde(rename = "max")]- Max(MaxAggregation),- /// Finds the minimum value.- #[serde(rename = "min")]- Min(MinAggregation),- /// Computes a collection of statistics (`min`, `max`, `sum`, `count`, and `avg`) over the- /// extracted values.- #[serde(rename = "stats")]- Stats(StatsAggregation),- /// Computes a collection of estended statistics (`min`, `max`, `sum`, `count`, `avg`,- /// `sum_of_squares`, `variance`, `variance_sampling`, `std_deviation`,- /// `std_deviation_sampling`) over the extracted values.- #[serde(rename = "extended_stats")]- ExtendedStats(ExtendedStatsAggregation),- /// Computes the sum of the extracted values.- #[serde(rename = "sum")]- Sum(SumAggregation),- /// Computes the sum of the extracted values.- #[serde(rename = "percentiles")]- Percentiles(PercentilesAggregationReq),- /// Finds the top k values matching some order- #[serde(rename = "top_hits")]- TopHits(TopHitsAggregation),-}--impl AggregationVariants {- /// Returns the name of the fields used by the aggregation.- pub fn get_fast_field_names(&self) -> Vec<&str> {- match self {- AggregationVariants::Terms(terms) => vec![terms.field.as_str()],- AggregationVariants::Range(range) => vec![range.field.as_str()],- AggregationVariants::Histogram(histogram) => vec![histogram.field.as_str()],- AggregationVariants::DateHistogram(histogram) => vec![histogram.field.as_str()],- AggregationVariants::Average(avg) => vec![avg.field_name()],- AggregationVariants::Count(count) => vec![count.field_name()],- AggregationVariants::Max(max) => vec![max.field_name()],- AggregationVariants::Min(min) => vec![min.field_name()],- AggregationVariants::Stats(stats) => vec![stats.field_name()],- AggregationVariants::ExtendedStats(extended_stats) => vec![extended_stats.field_name()],- AggregationVariants::Sum(sum) => vec![sum.field_name()],- AggregationVariants::Percentiles(per) => vec![per.field_name()],- AggregationVariants::TopHits(top_hits) => top_hits.field_names(),- }- }-- pub(crate) fn as_range(&self) -> Option<&RangeAggregation> {- match &self {- AggregationVariants::Range(range) => Some(range),- _ => None,- }- }- pub(crate) fn as_histogram(&self) -> crate::Result<Option<HistogramAggregation>> {- match &self {- AggregationVariants::Histogram(histogram) => Ok(Some(histogram.clone())),- AggregationVariants::DateHistogram(histogram) => {- Ok(Some(histogram.to_histogram_req()?))- }- _ => Ok(None),- }- }- pub(crate) fn as_term(&self) -> Option<&TermsAggregation> {- match &self {- AggregationVariants::Terms(terms) => Some(terms),- _ => None,- }- }- pub(crate) fn as_top_hits(&self) -> Option<&TopHitsAggregation> {- match &self {- AggregationVariants::TopHits(top_hits) => Some(top_hits),- _ => None,- }- }-- pub(crate) fn as_percentile(&self) -> Option<&PercentilesAggregationReq> {- match &self {- AggregationVariants::Percentiles(percentile_req) => Some(percentile_req),- _ => None,- }- }-}diff --git a/crates/tantivy/src/aggregation/agg_req_with_accessor.rs b/crates/tantivy/src/aggregation/agg_req_with_accessor.rsdeleted file mode 100644index 54534213..00000000--- a/crates/tantivy/src/aggregation/agg_req_with_accessor.rs+++ /dev/null@@ -1,430 +0,0 @@-//! This will enhance the request tree with access to the fastfield and metadata.--use std::collections::HashMap;-use std::io;--use crate::columnar::{Column, ColumnBlockAccessor, ColumnType, DynamicColumn, StrColumn};--use super::agg_limits::ResourceLimitGuard;-use super::agg_req::{Aggregation, AggregationVariants, Aggregations};-use super::bucket::{- DateHistogramAggregationReq, HistogramAggregation, RangeAggregation, TermsAggregation,-};-use super::metric::{- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,- StatsAggregation, SumAggregation,-};-use super::segment_agg_result::AggregationLimits;-use super::VecWithNames;-use crate::aggregation::{f64_to_fastfield_u64, Key};-use crate::index::SegmentReader;-use crate::SegmentOrdinal;--#[derive(Default)]-pub(crate) struct AggregationsWithAccessor {- pub aggs: VecWithNames<AggregationWithAccessor>,-}--impl AggregationsWithAccessor {- fn from_data(aggs: VecWithNames<AggregationWithAccessor>) -> Self {- Self { aggs }- }-- pub fn is_empty(&self) -> bool {- self.aggs.is_empty()- }-}--pub struct AggregationWithAccessor {- pub(crate) segment_ordinal: SegmentOrdinal,- /// In general there can be buckets without fast field access, e.g. buckets that are created- /// based on search terms. That is not that case currently, but eventually this needs to be- /// Option or moved.- pub(crate) accessor: Column<u64>,- /// Load insert u64 for missing use case- pub(crate) missing_value_for_accessor: Option<u64>,- pub(crate) str_dict_column: Option<StrColumn>,- pub(crate) field_type: ColumnType,- pub(crate) sub_aggregation: AggregationsWithAccessor,- pub(crate) limits: ResourceLimitGuard,- pub(crate) column_block_accessor: ColumnBlockAccessor<u64>,- /// Used for missing term aggregation, which checks all columns for existence.- /// And also for `top_hits` aggregation, which may sort on multiple fields.- /// By convention the missing aggregation is chosen, when this property is set- /// (instead bein set in `agg`).- /// If this needs to used by other aggregations, we need to refactor this.- // NOTE: we can make all other aggregations use this instead of the `accessor` and `field_type`- // (making them obsolete) But will it have a performance impact?- pub(crate) accessors: Vec<(Column<u64>, ColumnType)>,- /// Map field names to all associated column accessors.- /// This field is used for `docvalue_fields`, which is currently only supported for `top_hits`.- pub(crate) value_accessors: HashMap<String, Vec<DynamicColumn>>,- pub(crate) agg: Aggregation,-}--impl AggregationWithAccessor {- /// May return multiple accessors if the aggregation is e.g. on mixed field types.- fn try_from_agg(- agg: &Aggregation,- sub_aggregation: &Aggregations,- reader: &SegmentReader,- segment_ordinal: SegmentOrdinal,- limits: AggregationLimits,- ) -> crate::Result<Vec<AggregationWithAccessor>> {- let mut agg = agg.clone();-- let add_agg_with_accessor = |agg: &Aggregation,- accessor: Column<u64>,- column_type: ColumnType,- aggs: &mut Vec<AggregationWithAccessor>|- -> crate::Result<()> {- let res = AggregationWithAccessor {- segment_ordinal,- accessor,- accessors: Default::default(),- value_accessors: Default::default(),- field_type: column_type,- sub_aggregation: get_aggs_with_segment_accessor_and_validate(- sub_aggregation,- reader,- segment_ordinal,- &limits,- )?,- agg: agg.clone(),- limits: limits.new_guard(),- missing_value_for_accessor: None,- str_dict_column: None,- column_block_accessor: Default::default(),- };- aggs.push(res);- Ok(())- };-- let add_agg_with_accessors = |agg: &Aggregation,- accessors: Vec<(Column<u64>, ColumnType)>,- aggs: &mut Vec<AggregationWithAccessor>,- value_accessors: HashMap<String, Vec<DynamicColumn>>|- -> crate::Result<()> {- let (accessor, field_type) = accessors.first().expect("at least one accessor");- let res = AggregationWithAccessor {- segment_ordinal,- // TODO: We should do away with the `accessor` field altogether- accessor: accessor.clone(),- value_accessors,- field_type: *field_type,- accessors,- sub_aggregation: get_aggs_with_segment_accessor_and_validate(- sub_aggregation,- reader,- segment_ordinal,- &limits,- )?,- agg: agg.clone(),- limits: limits.new_guard(),- missing_value_for_accessor: None,- str_dict_column: None,- column_block_accessor: Default::default(),- };- aggs.push(res);- Ok(())- };-- let mut res: Vec<AggregationWithAccessor> = Vec::new();- use AggregationVariants::*;-- match agg.agg {- Range(RangeAggregation {- field: ref field_name,- ..- }) => {- let (accessor, column_type) =- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;- }- Histogram(HistogramAggregation {- field: ref field_name,- ..- }) => {- let (accessor, column_type) =- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;- }- DateHistogram(DateHistogramAggregationReq {- field: ref field_name,- ..- }) => {- let (accessor, column_type) =- // Only DateTime is supported for DateHistogram- get_ff_reader(reader, field_name, Some(&[ColumnType::DateTime]))?;- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;- }- Terms(TermsAggregation {- field: ref field_name,- ref missing,- ..- }) => {- let str_dict_column = reader.fast_fields().str(field_name)?;- let allowed_column_types = [- ColumnType::I64,- ColumnType::U64,- ColumnType::F64,- ColumnType::Str,- ColumnType::DateTime,- ColumnType::Bool,- ColumnType::IpAddr,- // ColumnType::Bytes Unsupported- ];-- // In case the column is empty we want the shim column to match the missing type- let fallback_type = missing- .as_ref()- .map(|missing| match missing {- Key::Str(_) => ColumnType::Str,- Key::F64(_) => ColumnType::F64,- })- .unwrap_or(ColumnType::U64);- let column_and_types = get_all_ff_reader_or_empty(- reader,- field_name,- Some(&allowed_column_types),- fallback_type,- )?;- let missing_and_more_than_one_col = column_and_types.len() > 1 && missing.is_some();- let text_on_non_text_col = column_and_types.len() == 1- && column_and_types[0].1.numerical_type().is_some()- && missing- .as_ref()- .map(|m| matches!(m, Key::Str(_)))- .unwrap_or(false);-- // Actually we could convert the text to a number and have the fast path, if it is- // provided in Rfc3339 format. But this use case is probably common- // enough to justify the effort.- let text_on_date_col = column_and_types.len() == 1- && column_and_types[0].1 == ColumnType::DateTime- && missing- .as_ref()- .map(|m| matches!(m, Key::Str(_)))- .unwrap_or(false);-- let use_special_missing_agg =- missing_and_more_than_one_col || text_on_non_text_col || text_on_date_col;- if use_special_missing_agg {- let column_and_types =- get_all_ff_reader_or_empty(reader, field_name, None, fallback_type)?;-- let accessors = column_and_types- .iter()- .map(|c_t| (c_t.0.clone(), c_t.1))- .collect();- add_agg_with_accessors(&agg, accessors, &mut res, Default::default())?;- }-- for (accessor, column_type) in column_and_types {- let missing_value_term_agg = if use_special_missing_agg {- None- } else {- missing.clone()- };-- let missing_value_for_accessor = if let Some(missing) =- missing_value_term_agg.as_ref()- {- get_missing_val(column_type, missing, agg.agg.get_fast_field_names()[0])?- } else {- None- };-- let agg = AggregationWithAccessor {- segment_ordinal,- missing_value_for_accessor,- accessor,- accessors: Default::default(),- value_accessors: Default::default(),- field_type: column_type,- sub_aggregation: get_aggs_with_segment_accessor_and_validate(- sub_aggregation,- reader,- segment_ordinal,- &limits,- )?,- agg: agg.clone(),- str_dict_column: str_dict_column.clone(),- limits: limits.new_guard(),- column_block_accessor: Default::default(),- };- res.push(agg);- }- }- Average(AverageAggregation {- field: ref field_name,- ..- })- | Count(CountAggregation {- field: ref field_name,- ..- })- | Max(MaxAggregation {- field: ref field_name,- ..- })- | Min(MinAggregation {- field: ref field_name,- ..- })- | Stats(StatsAggregation {- field: ref field_name,- ..- })- | ExtendedStats(ExtendedStatsAggregation {- field: ref field_name,- ..- })- | Sum(SumAggregation {- field: ref field_name,- ..- }) => {- let (accessor, column_type) =- get_ff_reader(reader, field_name, Some(get_numeric_or_date_column_types()))?;- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;- }- Percentiles(ref percentiles) => {- let (accessor, column_type) = get_ff_reader(- reader,- percentiles.field_name(),- Some(get_numeric_or_date_column_types()),- )?;- add_agg_with_accessor(&agg, accessor, column_type, &mut res)?;- }- TopHits(ref mut top_hits) => {- top_hits.validate_and_resolve_field_names(reader.fast_fields().columnar())?;- let accessors: Vec<(Column<u64>, ColumnType)> = top_hits- .field_names()- .iter()- .map(|field| {- get_ff_reader(reader, field, Some(get_numeric_or_date_column_types()))- })- .collect::<crate::Result<_>>()?;-- let value_accessors = top_hits- .value_field_names()- .iter()- .map(|field_name| {- Ok((- field_name.to_string(),- get_dynamic_columns(reader, field_name)?,- ))- })- .collect::<crate::Result<_>>()?;-- add_agg_with_accessors(&agg, accessors, &mut res, value_accessors)?;- }- };-- Ok(res)- }-}--fn get_missing_val(- column_type: ColumnType,- missing: &Key,- field_name: &str,-) -> crate::Result<Option<u64>> {- let missing_val = match missing {- Key::Str(_) if column_type == ColumnType::Str => Some(u64::MAX),- // Allow fallback to number on text fields- Key::F64(_) if column_type == ColumnType::Str => Some(u64::MAX),- Key::F64(val) if column_type.numerical_type().is_some() => {- f64_to_fastfield_u64(*val, &column_type)- }- _ => {- return Err(crate::TantivyError::InvalidArgument(format!(- "Missing value {missing:?} for field {field_name} is not supported for column \- type {column_type:?}"- )));- }- };- Ok(missing_val)-}--fn get_numeric_or_date_column_types() -> &'static [ColumnType] {- &[- ColumnType::F64,- ColumnType::U64,- ColumnType::I64,- ColumnType::DateTime,- ]-}--pub(crate) fn get_aggs_with_segment_accessor_and_validate(- aggs: &Aggregations,- reader: &SegmentReader,- segment_ordinal: SegmentOrdinal,- limits: &AggregationLimits,-) -> crate::Result<AggregationsWithAccessor> {- let mut aggss = Vec::new();- for (key, agg) in aggs.iter() {- let aggs = AggregationWithAccessor::try_from_agg(- agg,- agg.sub_aggregation(),- reader,- segment_ordinal,- limits.clone(),- )?;- for agg in aggs {- aggss.push((key.to_string(), agg));- }- }- Ok(AggregationsWithAccessor::from_data(- VecWithNames::from_entries(aggss),- ))-}--/// Get fast field reader or empty as default.-fn get_ff_reader(- reader: &SegmentReader,- field_name: &str,- allowed_column_types: Option<&[ColumnType]>,-) -> crate::Result<(crate::columnar::Column<u64>, ColumnType)> {- let ff_fields = reader.fast_fields();- let ff_field_with_type = ff_fields- .u64_lenient_for_type(allowed_column_types, field_name)?- .unwrap_or_else(|| {- (- Column::build_empty_column(reader.num_docs()),- ColumnType::U64,- )- });- Ok(ff_field_with_type)-}--fn get_dynamic_columns(- reader: &SegmentReader,- field_name: &str,-) -> crate::Result<Vec<crate::columnar::DynamicColumn>> {- let ff_fields = reader.fast_fields().dynamic_column_handles(field_name)?;- let cols = ff_fields- .iter()- .map(|h| h.open())- .collect::<io::Result<_>>()?;- assert!(!ff_fields.is_empty(), "field {field_name} not found");- Ok(cols)-}--/// Get all fast field reader or empty as default.-///-/// Is guaranteed to return at least one column.-fn get_all_ff_reader_or_empty(- reader: &SegmentReader,- field_name: &str,- allowed_column_types: Option<&[ColumnType]>,- fallback_type: ColumnType,-) -> crate::Result<Vec<(crate::columnar::Column<u64>, ColumnType)>> {- let ff_fields = reader.fast_fields();- let mut ff_field_with_type =- ff_fields.u64_lenient_for_type_all(allowed_column_types, field_name)?;- if ff_field_with_type.is_empty() {- ff_field_with_type.push((Column::build_empty_column(reader.num_docs()), fallback_type));- }- Ok(ff_field_with_type)-}diff --git a/crates/tantivy/src/aggregation/agg_result.rs b/crates/tantivy/src/aggregation/agg_result.rsdeleted file mode 100644index aac1efac..00000000--- a/crates/tantivy/src/aggregation/agg_result.rs+++ /dev/null@@ -1,306 +0,0 @@-//! Contains the final aggregation tree.-//! This tree can be converted via the `into()` method from `IntermediateAggregationResults`.-//! This conversion computes the final result. For example: The intermediate result contains-//! intermediate average results, which is the sum and the number of values. The actual average is-//! calculated on the step from intermediate to final aggregation result tree.--use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};--use super::bucket::GetDocCount;-use super::metric::{- ExtendedStats, PercentilesMetricResult, SingleMetricResult, Stats, TopHitsMetricResult,-};-use super::{AggregationError, Key};-use crate::TantivyError;--#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)]-/// The final aggegation result.-pub struct AggregationResults(pub FxHashMap<String, AggregationResult>);--impl AggregationResults {- pub(crate) fn get_bucket_count(&self) -> u64 {- self.0- .values()- .map(|agg| agg.get_bucket_count())- .sum::<u64>()- }-- pub(crate) fn get_value_from_aggregation(- &self,- name: &str,- agg_property: &str,- ) -> crate::Result<Option<f64>> {- if let Some(agg) = self.0.get(name) {- agg.get_value_from_aggregation(name, agg_property)- } else {- // Validation is be done during request parsing, so we can't reach this state.- Err(TantivyError::InternalError(format!(- "Can't find aggregation {name:?} in sub-aggregations"- )))- }- }-}--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(untagged)]-/// An aggregation is either a bucket or a metric.-pub enum AggregationResult {- /// Bucket result variant.- BucketResult(BucketResult),- /// Metric result variant.- MetricResult(MetricResult),-}--impl AggregationResult {- pub(crate) fn get_bucket_count(&self) -> u64 {- match self {- AggregationResult::BucketResult(bucket) => bucket.get_bucket_count(),- AggregationResult::MetricResult(_) => 0,- }- }-- pub(crate) fn get_value_from_aggregation(- &self,- _name: &str,- agg_property: &str,- ) -> crate::Result<Option<f64>> {- match self {- AggregationResult::BucketResult(_bucket) => Err(TantivyError::InternalError(- "Tried to retrieve value from bucket aggregation. This is not supported and \- should not happen during collection phase, but should be caught during validation"- .to_string(),- )),- AggregationResult::MetricResult(metric) => metric.get_value(agg_property),- }- }-}--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(untagged)]-/// MetricResult-pub enum MetricResult {- /// Average metric result.- Average(SingleMetricResult),- /// Count metric result.- Count(SingleMetricResult),- /// Max metric result.- Max(SingleMetricResult),- /// Min metric result.- Min(SingleMetricResult),- /// Stats metric result.- Stats(Stats),- /// ExtendedStats metric result.- ExtendedStats(Box<ExtendedStats>),- /// Sum metric result.- Sum(SingleMetricResult),- /// Percentiles metric result.- Percentiles(PercentilesMetricResult),- /// Top hits metric result- TopHits(TopHitsMetricResult),-}--impl MetricResult {- fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {- match self {- MetricResult::Average(avg) => Ok(avg.value),- MetricResult::Count(count) => Ok(count.value),- MetricResult::Max(max) => Ok(max.value),- MetricResult::Min(min) => Ok(min.value),- MetricResult::Stats(stats) => stats.get_value(agg_property),- MetricResult::ExtendedStats(extended_stats) => extended_stats.get_value(agg_property),- MetricResult::Sum(sum) => Ok(sum.value),- MetricResult::Percentiles(_) => Err(TantivyError::AggregationError(- AggregationError::InvalidRequest("percentiles can't be used to order".to_string()),- )),- MetricResult::TopHits(_) => Err(TantivyError::AggregationError(- AggregationError::InvalidRequest("top_hits can't be used to order".to_string()),- )),- }- }-}--/// BucketEntry holds bucket aggregation result types.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(untagged)]-pub enum BucketResult {- /// This is the range entry for a bucket, which contains a key, count, from, to, and optionally- /// sub-aggregations.- Range {- /// The range buckets sorted by range.- buckets: BucketEntries<RangeBucketEntry>,- },- /// This is the histogram entry for a bucket, which contains a key, count, and optionally- /// sub-aggregations.- Histogram {- /// The buckets.- ///- /// If there are holes depends on the request, if min_doc_count is 0, then there are no- /// holes between the first and last bucket.- /// See [`HistogramAggregation`](super::bucket::HistogramAggregation)- buckets: BucketEntries<BucketEntry>,- },- /// This is the term result- Terms {- /// The buckets.- ///- /// See [`TermsAggregation`](super::bucket::TermsAggregation)- buckets: Vec<BucketEntry>,- /// The number of documents that didn’t make it into to TOP N due to shard_size or size- sum_other_doc_count: u64,- #[serde(skip_serializing_if = "Option::is_none")]- /// The upper bound error for the doc count of each term.- doc_count_error_upper_bound: Option<u64>,- },-}--impl BucketResult {- pub(crate) fn get_bucket_count(&self) -> u64 {- match self {- BucketResult::Range { buckets } => {- buckets.iter().map(|bucket| bucket.get_bucket_count()).sum()- }- BucketResult::Histogram { buckets } => {- buckets.iter().map(|bucket| bucket.get_bucket_count()).sum()- }- BucketResult::Terms {- buckets,- sum_other_doc_count: _,- doc_count_error_upper_bound: _,- } => buckets.iter().map(|bucket| bucket.get_bucket_count()).sum(),- }- }-}--/// This is the wrapper of buckets entries, which can be vector or hashmap-/// depending on if it's keyed or not.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(untagged)]-pub enum BucketEntries<T> {- /// Vector format bucket entries- Vec(Vec<T>),- /// HashMap format bucket entries- HashMap(FxHashMap<String, T>),-}--impl<T> BucketEntries<T> {- fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = &T> + 'a> {- match self {- BucketEntries::Vec(vec) => Box::new(vec.iter()),- BucketEntries::HashMap(map) => Box::new(map.values()),- }- }-}--/// This is the default entry for a bucket, which contains a key, count, and optionally-/// sub-aggregations.-///-/// # JSON Format-/// ```json-/// {-/// ...-/// "my_histogram": {-/// "buckets": [-/// {-/// "key": "2.0",-/// "doc_count": 5-/// },-/// {-/// "key": "4.0",-/// "doc_count": 2-/// },-/// {-/// "key": "6.0",-/// "doc_count": 3-/// }-/// ]-/// }-/// ...-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct BucketEntry {- #[serde(skip_serializing_if = "Option::is_none")]- /// The string representation of the bucket.- pub key_as_string: Option<String>,- /// The identifier of the bucket.- pub key: Key,- /// Number of documents in the bucket.- pub doc_count: u64,- #[serde(flatten)]- /// Sub-aggregations in this bucket.- pub sub_aggregation: AggregationResults,-}-impl BucketEntry {- pub(crate) fn get_bucket_count(&self) -> u64 {- 1 + self.sub_aggregation.get_bucket_count()- }-}-impl GetDocCount for &BucketEntry {- fn doc_count(&self) -> u64 {- self.doc_count- }-}-impl GetDocCount for BucketEntry {- fn doc_count(&self) -> u64 {- self.doc_count- }-}--/// This is the range entry for a bucket, which contains a key, count, and optionally-/// sub-aggregations.-///-/// # JSON Format-/// ```json-/// {-/// ...-/// "my_ranges": {-/// "buckets": [-/// {-/// "key": "*-10",-/// "to": 10,-/// "doc_count": 5-/// },-/// {-/// "key": "10-20",-/// "from": 10,-/// "to": 20,-/// "doc_count": 2-/// },-/// {-/// "key": "20-*",-/// "from": 20,-/// "doc_count": 3-/// }-/// ]-/// }-/// ...-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct RangeBucketEntry {- /// The identifier of the bucket.- pub key: Key,- /// Number of documents in the bucket.- pub doc_count: u64,- #[serde(flatten)]- /// Sub-aggregations in this bucket.- pub sub_aggregation: AggregationResults,- /// The from range of the bucket. Equals `f64::MIN` when `None`.- #[serde(skip_serializing_if = "Option::is_none")]- pub from: Option<f64>,- /// The to range of the bucket. Equals `f64::MAX` when `None`.- #[serde(skip_serializing_if = "Option::is_none")]- pub to: Option<f64>,- /// The optional string representation for the `from` range.- #[serde(skip_serializing_if = "Option::is_none")]- pub from_as_string: Option<String>,- /// The optional string representation for the `to` range.- #[serde(skip_serializing_if = "Option::is_none")]- pub to_as_string: Option<String>,-}-impl RangeBucketEntry {- pub(crate) fn get_bucket_count(&self) -> u64 {- 1 + self.sub_aggregation.get_bucket_count()- }-}diff --git a/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rs b/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rsdeleted file mode 100644index e1ca3426..00000000--- a/crates/tantivy/src/aggregation/bucket/histogram/date_histogram.rs+++ /dev/null@@ -1,675 +0,0 @@-use serde::{Deserialize, Serialize};--use super::{HistogramAggregation, HistogramBounds};-use crate::aggregation::*;--/// DateHistogramAggregation is similar to `HistogramAggregation`, but it can only be used with date-/// type.-///-/// Currently only **fixed time** intervals are supported. Calendar-aware time intervals are not-/// supported.-///-/// Like the histogram, values are rounded down into the closest bucket.-///-/// For this calculation all fastfield values are converted to f64.-///-/// # Limitations/Compatibility-/// Only fixed time intervals are supported.-///-/// # JSON Format-/// ```json-/// {-/// "prices": {-/// "date_histogram": {-/// "field": "price",-/// "fixed_interval": "30d"-/// }-/// }-/// }-/// ```-///-/// Response-/// See [`BucketEntry`](crate::aggregation::agg_result::BucketEntry)-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]-pub struct DateHistogramAggregationReq {- #[doc(hidden)]- /// Only for validation- interval: Option<String>,- #[doc(hidden)]- /// Only for validation- calendar_interval: Option<String>,- /// The field to aggregate on.- pub field: String,- /// The format to format dates. Unsupported currently.- pub format: Option<String>,- /// The interval to chunk your data range. Each bucket spans a value range of- /// [0..fixed_interval). Accepted values- ///- /// Fixed intervals are configured with the `fixed_interval` parameter.- /// In contrast to calendar-aware intervals, fixed intervals are a fixed number of SI units and- /// never deviate, regardless of where they fall on the calendar. One second is always- /// composed of 1000ms. This allows fixed intervals to be specified in any multiple of the- /// supported units. However, it means fixed intervals cannot express other units such as- /// months, since the duration of a month is not a fixed quantity. Attempting to specify a- /// calendar interval like month or quarter will return an Error.- ///- /// The accepted units for fixed intervals are:- /// * `ms`: milliseconds- /// * `s`: seconds. Defined as 1000 milliseconds each.- /// * `m`: minutes. Defined as 60 seconds each (60_000 milliseconds).- /// * `h`: hours. Defined as 60 minutes each (3_600_000 milliseconds).- /// * `d`: days. Defined as 24 hours (86_400_000 milliseconds).- ///- /// Fractional time values are not supported, but you can address this by shifting to another- /// time unit (e.g., `1.5h` could instead be specified as `90m`).- ///- /// `Option` for validation, the parameter is not optional- pub fixed_interval: Option<String>,- /// Intervals implicitly defines an absolute grid of buckets `[interval * k, interval * (k +- /// 1))`.- ///- /// Offset makes it possible to shift this grid into- /// `[offset + interval * k, offset + interval * (k + 1))`. Offset has to be in the range [0,- /// interval).- ///- /// The `offset` parameter is has the same syntax as the `fixed_interval` parameter, but- /// also allows for negative values.- pub offset: Option<String>,- /// The minimum number of documents in a bucket to be returned. Defaults to 0.- pub min_doc_count: Option<u64>,- /// Limits the data range to `[min, max]` closed interval.- ///- /// This can be used to filter values if they are not in the data range.- ///- /// hard_bounds only limits the buckets, to force a range set both extended_bounds and- /// hard_bounds to the same range.- ///- /// Needs to be provided as timestamp in millisecond precision.- ///- /// ## Example- /// ```json- /// {- /// "sales_over_time": {- /// "date_histogram": {- /// "field": "dates",- /// "interval": "1d",- /// "hard_bounds": {- /// "min": 0,- /// "max": 1420502400000- /// }- /// }- /// }- /// }- /// ```- pub hard_bounds: Option<HistogramBounds>,- /// Can be set to extend your bounds. The range of the buckets is by default defined by the- /// data range of the values of the documents. As the name suggests, this can only be used to- /// extend the value range. If the bounds for min or max are not extending the range, the value- /// has no effect on the returned buckets.- ///- /// Cannot be set in conjunction with min_doc_count > 0, since the empty buckets from extended- /// bounds would not be returned.- pub extended_bounds: Option<HistogramBounds>,-- /// Whether to return the buckets as a hash map- #[serde(default)]- pub keyed: bool,-}--impl DateHistogramAggregationReq {- pub(crate) fn to_histogram_req(&self) -> crate::Result<HistogramAggregation> {- self.validate()?;- Ok(HistogramAggregation {- field: self.field.to_string(),- interval: parse_into_milliseconds(self.fixed_interval.as_ref().unwrap())? as f64,- offset: self- .offset- .as_ref()- .map(|offset| parse_offset_into_milliseconds(offset))- .transpose()?- .map(|el| el as f64),- min_doc_count: self.min_doc_count,- hard_bounds: self.hard_bounds,- extended_bounds: self.extended_bounds,- keyed: self.keyed,- is_normalized_to_ns: false,- })- }-- fn validate(&self) -> crate::Result<()> {- if let Some(interval) = self.interval.as_ref() {- return Err(crate::TantivyError::InvalidArgument(format!(- "`interval` parameter {interval:?} in date histogram is unsupported, only \- `fixed_interval` is supported"- )));- }- if let Some(interval) = self.calendar_interval.as_ref() {- return Err(crate::TantivyError::InvalidArgument(format!(- "`calendar_interval` parameter {interval:?} in date histogram is unsupported, \- only `fixed_interval` is supported"- )));- }- if self.format.is_some() {- return Err(crate::TantivyError::InvalidArgument(- "format parameter on date_histogram is unsupported".to_string(),- ));- }-- if self.fixed_interval.is_none() {- return Err(crate::TantivyError::InvalidArgument(- "fixed_interval in date histogram is missing".to_string(),- ));- }-- parse_into_milliseconds(self.fixed_interval.as_ref().unwrap())?;-- Ok(())- }-}--#[derive(Debug, Clone, PartialEq, Eq, Error)]-/// Errors when parsing the fixed interval for `DateHistogramAggregationReq`.-pub enum DateHistogramParseError {- /// Unit not recognized in passed String- #[error("Unit not recognized in passed String {0:?}")]- UnitNotRecognized(String),- /// Number not found in passed String- #[error("Number not found in passed String {0:?}")]- NumberMissing(String),- /// Unit not found in passed String- #[error("Unit not found in passed String {0:?}")]- UnitMissing(String),- /// Offset invalid- #[error("passed offset is invalid {0:?}")]- InvalidOffset(String),- /// Value out of bounds- #[error("passed value is out of bounds: {0:?}")]- OutOfBounds(String),-}--fn parse_offset_into_milliseconds(input: &str) -> Result<i64, AggregationError> {- let is_sign = |byte| &[byte] == b"-" || &[byte] == b"+";- if input.is_empty() {- return Err(DateHistogramParseError::InvalidOffset(input.to_string()).into());- }-- let has_sign = is_sign(input.as_bytes()[0]);- if has_sign {- let (sign, input) = input.split_at(1);- let val = parse_into_milliseconds(input)?;- if sign == "-" {- Ok(-val)- } else {- Ok(val)- }- } else {- parse_into_milliseconds(input)- }-}--fn parse_into_milliseconds(input: &str) -> Result<i64, AggregationError> {- let split_boundary = input- .as_bytes()- .iter()- .take_while(|byte| byte.is_ascii_digit())- .count();- let (number, unit) = input.split_at(split_boundary);- if number.is_empty() {- return Err(DateHistogramParseError::NumberMissing(input.to_string()).into());- }- if unit.is_empty() {- return Err(DateHistogramParseError::UnitMissing(input.to_string()).into());- }- let number: i64 = number- .parse()- // Technically this should never happen, but there was a bug- // here and being defensive does not hurt.- .map_err(|_err| DateHistogramParseError::NumberMissing(input.to_string()))?;-- let unit_in_ms = match unit {- "ms" | "milliseconds" => 1,- "s" | "seconds" => 1000,- "m" | "minutes" => 60 * 1000,- "h" | "hours" => 60 * 60 * 1000,- "d" | "days" => 24 * 60 * 60 * 1000,- _ => return Err(DateHistogramParseError::UnitNotRecognized(unit.to_string()).into()),- };-- let val = number * unit_in_ms;- // The field type is in nanoseconds precision, so validate the value to fit the range- val.checked_mul(1_000_000)- .ok_or_else(|| DateHistogramParseError::OutOfBounds(input.to_string()))?;-- Ok(val)-}--#[cfg(test)]-pub mod tests {- use pretty_assertions::assert_eq;-- use super::*;- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::tests::exec_request;- use crate::indexer::NoMergePolicy;- use crate::schema::{Schema, FAST, STRING};- use crate::{Index, IndexWriter, TantivyDocument};-- #[test]- fn test_parse_into_millisecs() {- assert_eq!(parse_into_milliseconds("1m").unwrap(), 60_000);- assert_eq!(parse_into_milliseconds("2m").unwrap(), 120_000);- assert_eq!(parse_into_milliseconds("2minutes").unwrap(), 120_000);- assert_eq!(- parse_into_milliseconds("2y").unwrap_err(),- DateHistogramParseError::UnitNotRecognized("y".to_string()).into()- );- assert_eq!(- parse_into_milliseconds("2000").unwrap_err(),- DateHistogramParseError::UnitMissing("2000".to_string()).into()- );- assert_eq!(- parse_into_milliseconds("ms").unwrap_err(),- DateHistogramParseError::NumberMissing("ms".to_string()).into()- );- }-- #[test]- fn test_parse_offset_into_milliseconds() {- assert_eq!(parse_offset_into_milliseconds("1m").unwrap(), 60_000);- assert_eq!(parse_offset_into_milliseconds("+1m").unwrap(), 60_000);- assert_eq!(parse_offset_into_milliseconds("-1m").unwrap(), -60_000);- assert_eq!(parse_offset_into_milliseconds("2m").unwrap(), 120_000);- assert_eq!(parse_offset_into_milliseconds("+2m").unwrap(), 120_000);- assert_eq!(parse_offset_into_milliseconds("-2m").unwrap(), -120_000);- assert_eq!(parse_offset_into_milliseconds("-2ms").unwrap(), -2);- assert_eq!(- parse_offset_into_milliseconds("2y").unwrap_err(),- DateHistogramParseError::UnitNotRecognized("y".to_string()).into()- );- assert_eq!(- parse_offset_into_milliseconds("2000").unwrap_err(),- DateHistogramParseError::UnitMissing("2000".to_string()).into()- );- assert_eq!(- parse_offset_into_milliseconds("ms").unwrap_err(),- DateHistogramParseError::NumberMissing("ms".to_string()).into()- );- }-- #[test]- fn test_parse_into_milliseconds_do_not_accept_non_ascii() {- assert!(parse_into_milliseconds("1m").is_err());- }-- pub fn get_test_index_from_docs(- merge_segments: bool,- segment_and_docs: &[Vec<&str>],- ) -> crate::Result<Index> {- let mut schema_builder = Schema::builder();- schema_builder.add_date_field("date", FAST);- schema_builder.add_json_field("mixed", FAST);- schema_builder.add_text_field("text", FAST | STRING);- schema_builder.add_text_field("text2", FAST | STRING);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema.clone());- {- let mut index_writer = index.writer_with_num_threads(1, 30_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- for values in segment_and_docs {- for doc_str in values {- let doc = TantivyDocument::parse_json(&schema, doc_str)?;- index_writer.add_document(doc)?;- }- // writing the segment- index_writer.commit()?;- }- }- if merge_segments {- let segment_ids = index- .searchable_segment_ids()- .expect("Searchable segments failed.");- if segment_ids.len() > 1 {- let mut index_writer: IndexWriter = index.writer_for_tests()?;- index_writer.merge(&segment_ids).wait()?;- index_writer.wait_merging_threads()?;- }- }-- Ok(index)- }-- #[test]- fn histogram_test_date_force_merge_segments() {- histogram_test_date_merge_segments(true)- }-- #[test]- fn histogram_test_date() {- histogram_test_date_merge_segments(false)- }-- fn histogram_test_date_merge_segments(merge_segments: bool) {- let docs = vec![- vec![r#"{ "date": "2015-01-01T12:10:30Z", "text": "aaa" }"#],- vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],- vec![r#"{ "date": "2015-01-01T11:11:30Z", "text": "bbb" }"#],- vec![r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb" }"#],- vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],- vec![r#"{ "date": "2015-01-06T00:00:00Z", "text": "ccc" }"#],- ];- let index = get_test_index_from_docs(merge_segments, &docs).unwrap();-- {- // 30day + offset- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "30d",- "offset": "-4d"- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!({- "sales_over_time" : {- "buckets" : [- {- "key_as_string" : "2015-01-01T00:00:00Z",- "key" : 1420070400000.0,- "doc_count" : 6- }- ]- }- });- assert_eq!(res, expected_res);- }-- {- // 30day + offset + sub_agg- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "30d",- "offset": "-4d"- },- "aggs": {- "texts": {- "terms": {"field": "text"}- }- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!({- "sales_over_time" : {- "buckets" : [- {- "key_as_string" : "2015-01-01T00:00:00Z",- "key" : 1420070400000.0,- "doc_count" : 6,- "texts": {- "buckets": [- {- "doc_count": 3,- "key": "bbb"- },- {- "doc_count": 2,- "key": "ccc"- },- {- "doc_count": 1,- "key": "aaa"- }- ],- "doc_count_error_upper_bound": 0,- "sum_other_doc_count": 0- }- }- ]- }- });- assert_eq!(res, expected_res);- }- {- // 1day- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "1d"- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!( {- "sales_over_time": {- "buckets": [- {- "doc_count": 3,- "key": 1420070400000.0,- "key_as_string": "2015-01-01T00:00:00Z"- },- {- "doc_count": 1,- "key": 1420156800000.0,- "key_as_string": "2015-01-02T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420243200000.0,- "key_as_string": "2015-01-03T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420329600000.0,- "key_as_string": "2015-01-04T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420416000000.0,- "key_as_string": "2015-01-05T00:00:00Z"- },- {- "doc_count": 2,- "key": 1420502400000.0,- "key_as_string": "2015-01-06T00:00:00Z"- }- ]- }- });- assert_eq!(res, expected_res);- }-- {- // 1day + extended_bounds- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "1d",- "extended_bounds": {- "min": 1419984000000.0,- "max": 1420588800000.0- }- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!({- "sales_over_time" : {- "buckets": [- {- "doc_count": 0,- "key": 1419984000000.0,- "key_as_string": "2014-12-31T00:00:00Z"- },- {- "doc_count": 3,- "key": 1420070400000.0,- "key_as_string": "2015-01-01T00:00:00Z"- },- {- "doc_count": 1,- "key": 1420156800000.0,- "key_as_string": "2015-01-02T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420243200000.0,- "key_as_string": "2015-01-03T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420329600000.0,- "key_as_string": "2015-01-04T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420416000000.0,- "key_as_string": "2015-01-05T00:00:00Z"- },- {- "doc_count": 2,- "key": 1420502400000.0,- "key_as_string": "2015-01-06T00:00:00Z"- },- {- "doc_count": 0,- "key": 1420588800000.0,- "key_as_string": "2015-01-07T00:00:00Z"- }- ]- }- });- assert_eq!(res, expected_res);- }- {- // 1day + hard_bounds + extended_bounds- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "1d",- "hard_bounds": {- "min": 1420156800000.0,- "max": 1420243200000.0- }- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!({- "sales_over_time" : {- "buckets": [- {- "doc_count": 1,- "key": 1420156800000.0,- "key_as_string": "2015-01-02T00:00:00Z"- }- ]- }- });- assert_eq!(res, expected_res);- }-- {- // 1day + hard_bounds as Rfc3339- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "fixed_interval": "1d",- "hard_bounds": {- "min": "2015-01-02T00:00:00Z",- "max": "2015-01-02T12:00:00Z"- }- }- }- }- );-- let agg_req: Aggregations = serde_json::from_str(- &serde_json::to_string(&elasticsearch_compatible_json).unwrap(),- )- .unwrap();- let res = exec_request(agg_req, &index).unwrap();- let expected_res = json!({- "sales_over_time" : {- "buckets": [- {- "doc_count": 1,- "key": 1420156800000.0,- "key_as_string": "2015-01-02T00:00:00Z"- }- ]- }- });- assert_eq!(res, expected_res);- }- }- #[test]- fn histogram_test_invalid_req() {- let docs = vec![];-- let index = get_test_index_from_docs(false, &docs).unwrap();- let elasticsearch_compatible_json = json!(- {- "sales_over_time": {- "date_histogram": {- "field": "date",- "interval": "30d",- "offset": "-4d"- }- }- }- );-- let agg_req: Aggregations =- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())- .unwrap();- let err = exec_request(agg_req, &index).unwrap_err();- assert_eq!(- err.to_string(),- r#"An invalid argument was passed: '`interval` parameter "30d" in date histogram is unsupported, only `fixed_interval` is supported'"#- );- }-}diff --git a/crates/tantivy/src/aggregation/bucket/histogram/histogram.rs b/crates/tantivy/src/aggregation/bucket/histogram/histogram.rsdeleted file mode 100644index cce91c14..00000000--- a/crates/tantivy/src/aggregation/bucket/histogram/histogram.rs+++ /dev/null@@ -1,595 +0,0 @@-use std::cmp::Ordering;--use crate::bitpacker::minmax;-use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};--use crate::aggregation::agg_limits::MemoryConsumption;-use crate::aggregation::agg_req::Aggregations;-use crate::aggregation::agg_req_with_accessor::{- AggregationWithAccessor, AggregationsWithAccessor,-};-use crate::aggregation::agg_result::BucketEntry;-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,- IntermediateHistogramBucketEntry,-};-use crate::aggregation::segment_agg_result::{- build_segment_agg_collector, SegmentAggregationCollector,-};-use crate::aggregation::*;-use crate::TantivyError;--/// Histogram is a bucket aggregation, where buckets are created dynamically for given `interval`.-/// Each document value is rounded down to its bucket.-///-/// E.g. if we have a price 18 and an interval of 5, the document will fall into the bucket with-/// the key 15. The formula used for this is:-/// `((val - offset) / interval).floor() * interval + offset`-///-/// For this calculation all fastfield values are converted to f64.-///-/// # Returned Buckets-/// By default buckets are returned between the min and max value of the documents, including empty-/// buckets.-/// Setting min_doc_count to != 0 will filter empty buckets.-///-/// The value range of the buckets can bet extended via-/// [extended_bounds](HistogramAggregation::extended_bounds) or limit the range via-/// [hard_bounds](HistogramAggregation::hard_bounds).-///-/// # Result-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with-/// [`BucketEntry`](crate::aggregation::agg_result::BucketEntry) on the-/// `AggregationCollector`.-///-/// Result type is-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with-/// [`IntermediateHistogramBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateHistogramBucketEntry) on the-/// `DistributedAggregationCollector`.-///-/// # Limitations/Compatibility-///-/// # JSON Format-/// ```json-/// {-/// "prices": {-/// "histogram": {-/// "field": "price",-/// "interval": 10-/// }-/// }-/// }-/// ```-///-/// Response-/// See [`BucketEntry`](crate::aggregation::agg_result::BucketEntry)-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]-pub struct HistogramAggregation {- /// The field to aggregate on.- pub field: String,- /// The interval to chunk your data range. Each bucket spans a value range of [0..interval).- /// Must be a positive value.- #[serde(deserialize_with = "deserialize_f64")]- pub interval: f64,- /// Intervals implicitly defines an absolute grid of buckets `[interval * k, interval * (k +- /// 1))`.- ///- /// Offset makes it possible to shift this grid into- /// `[offset + interval * k, offset + interval * (k + 1))`. Offset has to be in the range [0,- /// interval).- ///- /// As an example, if there are two documents with value 9 and 12 and interval 10.0, they would- /// fall into the buckets with the key 0 and 10.- /// With offset 5 and interval 10, they would both fall into the bucket with they key 5 and the- /// range [5..15)- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub offset: Option<f64>,- /// The minimum number of documents in a bucket to be returned. Defaults to 0.- pub min_doc_count: Option<u64>,- /// Limits the data range to `[min, max]` closed interval.- ///- /// This can be used to filter values if they are not in the data range.- ///- /// hard_bounds only limits the buckets, to force a range set both extended_bounds and- /// hard_bounds to the same range.- ///- /// ## Example- /// ```json- /// {- /// "prices": {- /// "histogram": {- /// "field": "price",- /// "interval": 10,- /// "hard_bounds": {- /// "min": 0,- /// "max": 100- /// }- /// }- /// }- /// }- /// ```- pub hard_bounds: Option<HistogramBounds>,- /// Can be set to extend your bounds. The range of the buckets is by default defined by the- /// data range of the values of the documents. As the name suggests, this can only be used to- /// extend the value range. If the bounds for min or max are not extending the range, the value- /// has no effect on the returned buckets.- ///- /// Cannot be set in conjunction with min_doc_count > 0, since the empty buckets from extended- /// bounds would not be returned.- pub extended_bounds: Option<HistogramBounds>,- /// Whether to return the buckets as a hash map- #[serde(default)]- pub keyed: bool,- /// Whether the values are normalized to ns for date time values. Defaults to false.- #[serde(default)]- pub is_normalized_to_ns: bool,-}--impl HistogramAggregation {- pub(crate) fn normalize_date_time(&mut self) {- if !self.is_normalized_to_ns {- // values are provided in ms, but the fastfield is in nano seconds- self.interval *= 1_000_000.0;- self.offset = self.offset.map(|off| off * 1_000_000.0);- self.hard_bounds = self.hard_bounds.map(|bounds| HistogramBounds {- min: bounds.min * 1_000_000.0,- max: bounds.max * 1_000_000.0,- });- self.extended_bounds = self.extended_bounds.map(|bounds| HistogramBounds {- min: bounds.min * 1_000_000.0,- max: bounds.max * 1_000_000.0,- });- self.is_normalized_to_ns = true;- }- }-- fn validate(&self) -> crate::Result<()> {- if self.interval <= 0.0f64 {- return Err(TantivyError::InvalidArgument(- "interval must be a positive value".to_string(),- ));- }-- if self.min_doc_count.unwrap_or(0) > 0 && self.extended_bounds.is_some() {- return Err(TantivyError::InvalidArgument(- "Cannot set min_doc_count and extended_bounds at the same time".to_string(),- ));- }-- if let (Some(hard_bounds), Some(extended_bounds)) = (self.hard_bounds, self.extended_bounds)- {- if extended_bounds.min < hard_bounds.min || extended_bounds.max > hard_bounds.max {- return Err(TantivyError::InvalidArgument(format!(- "extended_bounds have to be inside hard_bounds, extended_bounds: \- {extended_bounds}, hard_bounds {hard_bounds}"- )));- }- }-- Ok(())- }-- /// Returns the minimum number of documents required for a bucket to be returned.- pub fn min_doc_count(&self) -> u64 {- self.min_doc_count.unwrap_or(0)- }-}--/// Used to set extended or hard bounds on the histogram.-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]-pub struct HistogramBounds {- /// The lower bounds.- #[serde(deserialize_with = "deserialize_date_or_num")]- pub min: f64,- /// The upper bounds.- #[serde(deserialize_with = "deserialize_date_or_num")]- pub max: f64,-}--fn deserialize_date_or_num<'de, D>(deserializer: D) -> Result<f64, D::Error>-where- D: serde::Deserializer<'de>,-{- let value: serde_json::Value = Deserialize::deserialize(deserializer)?;-- // Check if the value is a string representing an Rfc3339 formatted date- if let serde_json::Value::String(date_str) = value {- // Parse the Rfc3339 formatted date string into a DateTime<Utc>- let date =- time::OffsetDateTime::parse(&date_str, &time::format_description::well_known::Rfc3339)- .map_err(|_| serde::de::Error::custom("Invalid Rfc3339 formatted date"))?;-- let milliseconds: i64 = (date.unix_timestamp_nanos() / 1_000_000)- .try_into()- .map_err(|_| serde::de::Error::custom("{date_str} out of allowed range"))?;-- // Return the milliseconds as f64- Ok(milliseconds as f64)- } else {- // The value is not a string, so assume it's a regular f64 number- value- .as_f64()- .ok_or_else(|| serde::de::Error::custom("Invalid number format"))- }-}--impl Display for HistogramBounds {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.write_fmt(format_args!("[{},{}]", self.min, self.max))- }-}--impl HistogramBounds {- fn contains(&self, val: f64) -> bool {- val >= self.min && val <= self.max- }-}--#[derive(Default, Clone, Debug, PartialEq)]-pub(crate) struct SegmentHistogramBucketEntry {- pub key: f64,- pub doc_count: u64,-}--impl SegmentHistogramBucketEntry {- pub(crate) fn into_intermediate_bucket_entry(- self,- sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,- agg_with_accessor: &AggregationsWithAccessor,- ) -> crate::Result<IntermediateHistogramBucketEntry> {- let mut sub_aggregation_res = IntermediateAggregationResults::default();- if let Some(sub_aggregation) = sub_aggregation {- sub_aggregation- .add_intermediate_aggregation_result(agg_with_accessor, &mut sub_aggregation_res)?;- }- Ok(IntermediateHistogramBucketEntry {- key: self.key,- doc_count: self.doc_count,- sub_aggregation: sub_aggregation_res,- })- }-}--/// The collector puts values from the fast field into the correct buckets and does a conversion to-/// the correct datatype.-#[derive(Clone, Debug)]-pub struct SegmentHistogramCollector {- /// The buckets containing the aggregation data.- buckets: FxHashMap<i64, SegmentHistogramBucketEntry>,- sub_aggregations: FxHashMap<i64, Box<dyn SegmentAggregationCollector>>,- sub_aggregation_blueprint: Option<Box<dyn SegmentAggregationCollector>>,- column_type: ColumnType,- interval: f64,- offset: f64,- bounds: HistogramBounds,- accessor_idx: usize,-}--impl SegmentAggregationCollector for SegmentHistogramCollector {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];-- let bucket = self.into_intermediate_bucket_result(agg_with_accessor)?;- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.collect_block(&[doc], agg_with_accessor)- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];-- let mem_pre = self.get_memory_consumption();-- let bounds = self.bounds;- let interval = self.interval;- let offset = self.offset;- let get_bucket_pos = |val| (get_bucket_pos_f64(val, interval, offset) as i64);-- bucket_agg_accessor- .column_block_accessor- .fetch_block(docs, &bucket_agg_accessor.accessor);-- for (doc, val) in bucket_agg_accessor- .column_block_accessor- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)- {- let val = self.f64_from_fastfield_u64(val);-- let bucket_pos = get_bucket_pos(val);-- if bounds.contains(val) {- let bucket = self.buckets.entry(bucket_pos).or_insert_with(|| {- let key = get_bucket_key_from_pos(bucket_pos as f64, interval, offset);- SegmentHistogramBucketEntry { key, doc_count: 0 }- });- bucket.doc_count += 1;- if let Some(sub_aggregation_blueprint) = self.sub_aggregation_blueprint.as_mut() {- self.sub_aggregations- .entry(bucket_pos)- .or_insert_with(|| sub_aggregation_blueprint.clone())- .collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;- }- }- }-- let mem_delta = self.get_memory_consumption() - mem_pre;- if mem_delta > 0 {- bucket_agg_accessor- .limits- .add_memory_consumed(mem_delta as u64)?;- }-- Ok(())- }-- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- let sub_aggregation_accessor =- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;-- for sub_aggregation in self.sub_aggregations.values_mut() {- sub_aggregation.flush(sub_aggregation_accessor)?;- }-- Ok(())- }-}--impl SegmentHistogramCollector {- fn get_memory_consumption(&self) -> usize {- let self_mem = std::mem::size_of::<Self>();- let sub_aggs_mem = self.sub_aggregations.memory_consumption();- let buckets_mem = self.buckets.memory_consumption();- self_mem + sub_aggs_mem + buckets_mem- }- /// Converts the collector result into a intermediate bucket result.- pub fn into_intermediate_bucket_result(- self,- agg_with_accessor: &AggregationWithAccessor,- ) -> crate::Result<IntermediateBucketResult> {- let mut buckets = Vec::with_capacity(self.buckets.len());-- for (bucket_pos, bucket) in self.buckets {- let bucket_res = bucket.into_intermediate_bucket_entry(- self.sub_aggregations.get(&bucket_pos).cloned(),- &agg_with_accessor.sub_aggregation,- );-- buckets.push(bucket_res?);- }- buckets.sort_unstable_by(|b1, b2| b1.key.total_cmp(&b2.key));-- Ok(IntermediateBucketResult::Histogram {- buckets,- is_date_agg: self.column_type == ColumnType::DateTime,- })- }-- pub(crate) fn from_req_and_validate(- mut req: HistogramAggregation,- sub_aggregation: &mut AggregationsWithAccessor,- field_type: ColumnType,- accessor_idx: usize,- ) -> crate::Result<Self> {- req.validate()?;- if field_type == ColumnType::DateTime {- req.normalize_date_time();- }-- let sub_aggregation_blueprint = if sub_aggregation.is_empty() {- None- } else {- let sub_aggregation = build_segment_agg_collector(sub_aggregation)?;- Some(sub_aggregation)- };-- let bounds = req.hard_bounds.unwrap_or(HistogramBounds {- min: f64::MIN,- max: f64::MAX,- });-- Ok(Self {- buckets: Default::default(),- column_type: field_type,- interval: req.interval,- offset: req.offset.unwrap_or(0.0),- bounds,- sub_aggregations: Default::default(),- sub_aggregation_blueprint,- accessor_idx,- })- }-- #[inline]- fn f64_from_fastfield_u64(&self, val: u64) -> f64 {- f64_from_fastfield_u64(val, &self.column_type)- }-}--#[inline]-fn get_bucket_pos_f64(val: f64, interval: f64, offset: f64) -> f64 {- ((val - offset) / interval).floor()-}--#[inline]-fn get_bucket_key_from_pos(bucket_pos: f64, interval: f64, offset: f64) -> f64 {- bucket_pos * interval + offset-}--// Convert to BucketEntry and fill gaps-fn intermediate_buckets_to_final_buckets_fill_gaps(- buckets: Vec<IntermediateHistogramBucketEntry>,- histogram_req: &HistogramAggregation,- sub_aggregation: &Aggregations,- limits: &AggregationLimits,-) -> crate::Result<Vec<BucketEntry>> {- // Generate the full list of buckets without gaps.- //- // The bounds are the min max from the current buckets, optionally extended by- // extended_bounds from the request- let min_max = minmax(buckets.iter().map(|bucket| bucket.key));-- // memory check upfront- let (_, first_bucket_num, last_bucket_num) =- generate_bucket_pos_with_opt_minmax(histogram_req, min_max);-- // It's based on user input, so we need to account for overflows- let added_buckets = ((last_bucket_num.saturating_sub(first_bucket_num)).max(0) as u64)- .saturating_sub(buckets.len() as u64);- limits.add_memory_consumed(- added_buckets * std::mem::size_of::<IntermediateHistogramBucketEntry>() as u64,- )?;- // create buckets- let fill_gaps_buckets = generate_buckets_with_opt_minmax(histogram_req, min_max);-- let empty_sub_aggregation = IntermediateAggregationResults::empty_from_req(sub_aggregation);-- // Use merge_join_by to fill in gaps, since buckets are sorted-- let final_buckets: Vec<BucketEntry> = buckets- .into_iter()- .merge_join_by(fill_gaps_buckets, |existing_bucket, fill_gaps_bucket| {- existing_bucket- .key- .partial_cmp(fill_gaps_bucket)- .unwrap_or(Ordering::Equal)- })- .map(|either| match either {- // Ignore the generated bucket- itertools::EitherOrBoth::Both(existing, _) => existing,- itertools::EitherOrBoth::Left(existing) => existing,- // Add missing bucket- itertools::EitherOrBoth::Right(missing_bucket) => IntermediateHistogramBucketEntry {- key: missing_bucket,- doc_count: 0,- sub_aggregation: empty_sub_aggregation.clone(),- },- })- .map(|intermediate_bucket| {- intermediate_bucket.into_final_bucket_entry(sub_aggregation, limits)- })- .collect::<crate::Result<Vec<_>>>()?;-- Ok(final_buckets)-}--// Convert to BucketEntry-pub(crate) fn intermediate_histogram_buckets_to_final_buckets(- buckets: Vec<IntermediateHistogramBucketEntry>,- is_date_agg: bool,- histogram_req: &HistogramAggregation,- sub_aggregation: &Aggregations,- limits: &AggregationLimits,-) -> crate::Result<Vec<BucketEntry>> {- // Normalization is column type dependent.- // The request used in the the call to final is not yet be normalized.- // Normalization is changing the precision from milliseconds to nanoseconds.- let mut histogram_req = histogram_req.clone();- if is_date_agg {- histogram_req.normalize_date_time();- }- let mut buckets = if histogram_req.min_doc_count() == 0 {- // With min_doc_count != 0, we may need to add buckets, so that there are no- // gaps, since intermediate result does not contain empty buckets (filtered to- // reduce serialization size).- intermediate_buckets_to_final_buckets_fill_gaps(- buckets,- &histogram_req,- sub_aggregation,- limits,- )?- } else {- buckets- .into_iter()- .filter(|histogram_bucket| histogram_bucket.doc_count >= histogram_req.min_doc_count())- .map(|histogram_bucket| {- histogram_bucket.into_final_bucket_entry(sub_aggregation, limits)- })- .collect::<crate::Result<Vec<_>>>()?- };-- // If we have a date type on the histogram buckets, we add the `key_as_string` field as rfc339- // and normalize from nanoseconds to milliseconds- if is_date_agg {- for bucket in buckets.iter_mut() {- if let crate::aggregation::Key::F64(ref mut val) = bucket.key {- let key_as_string = format_date(*val as i64)?;- *val /= 1_000_000.0;- bucket.key_as_string = Some(key_as_string);- }- }- }-- Ok(buckets)-}--/// Applies req extended_bounds/hard_bounds on the min_max value-///-/// May return `(f64::MAX, f64::MIN)`, if there is no range.-fn get_req_min_max(req: &HistogramAggregation, min_max: Option<(f64, f64)>) -> (f64, f64) {- let (mut min, mut max) = min_max.unwrap_or((f64::MAX, f64::MIN));-- if let Some(extended_bounds) = &req.extended_bounds {- min = min.min(extended_bounds.min);- max = max.max(extended_bounds.max);- }-- if let Some(hard_bounds) = &req.hard_bounds {- min = min.max(hard_bounds.min);- max = max.min(hard_bounds.max);- }-- (min, max)-}--/// Generates buckets with req.interval-/// Range is computed for provided min_max and request extended_bounds/hard_bounds-/// returns empty vec when there is no range to span-pub(crate) fn generate_bucket_pos_with_opt_minmax(- req: &HistogramAggregation,- min_max: Option<(f64, f64)>,-) -> (f64, i64, i64) {- let (min, max) = get_req_min_max(req, min_max);-- let offset = req.offset.unwrap_or(0.0);- let first_bucket_num = get_bucket_pos_f64(min, req.interval, offset) as i64;- let last_bucket_num = get_bucket_pos_f64(max, req.interval, offset) as i64;- (offset, first_bucket_num, last_bucket_num)-}--/// Generates buckets with req.interval-/// Range is computed for provided min_max and request extended_bounds/hard_bounds-/// returns empty vec when there is no range to span-pub(crate) fn generate_buckets_with_opt_minmax(- req: &HistogramAggregation,- min_max: Option<(f64, f64)>,-) -> Vec<f64> {- let (offset, first_bucket_num, last_bucket_num) =- generate_bucket_pos_with_opt_minmax(req, min_max);- let mut buckets = Vec::with_capacity((first_bucket_num..=last_bucket_num).count());- for bucket_pos in first_bucket_num..=last_bucket_num {- let bucket_key = bucket_pos as f64 * req.interval + offset;- buckets.push(bucket_key);- }-- buckets-}diff --git a/crates/tantivy/src/aggregation/bucket/histogram/mod.rs b/crates/tantivy/src/aggregation/bucket/histogram/mod.rsdeleted file mode 100644index 82f695ca..00000000--- a/crates/tantivy/src/aggregation/bucket/histogram/mod.rs+++ /dev/null@@ -1,4 +0,0 @@-mod date_histogram;-mod histogram;-pub use date_histogram::*;-pub use histogram::*;diff --git a/crates/tantivy/src/aggregation/bucket/mod.rs b/crates/tantivy/src/aggregation/bucket/mod.rsdeleted file mode 100644index f1eaa975..00000000--- a/crates/tantivy/src/aggregation/bucket/mod.rs+++ /dev/null@@ -1,148 +0,0 @@-//! Module for all bucket aggregations.-//!-//! BucketAggregations create buckets of documents.-//! Each bucket is associated with a rule which-//! determines whether or not a document in the falls into it. In other words, the buckets-//! effectively define document sets. Buckets are not necessarily disjunct, therefore a document can-//! fall into multiple buckets. In addition to the buckets themselves, the bucket aggregations also-//! compute and return the number of documents for each bucket. Bucket aggregations, as opposed to-//! metric aggregations, can hold sub-aggregations. These sub-aggregations will be aggregated for-//! the buckets created by their "parent" bucket aggregation. There are different bucket-//! aggregators, each with a different "bucketing" strategy. Some define a single bucket, some-//! define fixed number of multiple buckets, and others dynamically create the buckets during the-//! aggregation process.-//!-//! Results of final buckets are [`BucketResult`](super::agg_result::BucketResult).-//! Results of intermediate buckets are-//! [`IntermediateBucketResult`](super::intermediate_agg_result::IntermediateBucketResult)-//!-//! ## Supported Bucket Aggregations-//! - [Histogram](HistogramAggregation)-//! - [DateHistogram](DateHistogramAggregationReq)-//! - [Range](RangeAggregation)-//! - [Terms](TermsAggregation)--mod histogram;-mod range;-mod term_agg;-mod term_missing_agg;--use std::collections::HashMap;-use std::fmt;--pub use histogram::*;-pub use range::*;-use serde::{de, Deserialize, Deserializer, Serialize, Serializer};-pub use term_agg::*;-pub use term_missing_agg::*;--/// Order for buckets in a bucket aggregation.-#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Default)]-pub enum Order {- /// Asc order- #[serde(rename = "asc")]- Asc,- /// Desc order- #[serde(rename = "desc")]- #[default]- Desc,-}--#[derive(Clone, Debug, PartialEq)]-/// Order property by which to apply the order-#[derive(Default)]-pub enum OrderTarget {- /// The key of the bucket- Key,- /// The doc count of the bucket- #[default]- Count,- /// Order by value of the sub aggregation metric with identified by given `String`.- ///- /// Only single value metrics are supported currently- SubAggregation(String),-}--impl From<&str> for OrderTarget {- fn from(val: &str) -> Self {- match val {- "_key" => OrderTarget::Key,- "_count" => OrderTarget::Count,- _ => OrderTarget::SubAggregation(val.to_string()),- }- }-}--impl fmt::Display for OrderTarget {- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- match self {- OrderTarget::Key => f.write_str("_key"),- OrderTarget::Count => f.write_str("_count"),- OrderTarget::SubAggregation(agg) => agg.fmt(f),- }- }-}--/// Set the order. target is either "_count", "_key", or the name of-/// a metric sub_aggregation.-///-/// De/Serializes to elasticsearch compatible JSON.-///-/// Examples in JSON format:-/// { "_count": "asc" }-/// { "_key": "asc" }-/// { "average_price": "asc" }-#[derive(Clone, Default, Debug, PartialEq)]-pub struct CustomOrder {- /// The target property by which to sort by- pub target: OrderTarget,- /// The order asc or desc- pub order: Order,-}--impl Serialize for CustomOrder {- fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>- where S: Serializer {- let map: HashMap<String, Order> =- std::iter::once((self.target.to_string(), self.order)).collect();- map.serialize(serializer)- }-}--impl<'de> Deserialize<'de> for CustomOrder {- fn deserialize<D>(deserializer: D) -> Result<CustomOrder, D::Error>- where D: Deserializer<'de> {- HashMap::<String, Order>::deserialize(deserializer).and_then(|map| {- if let Some((key, value)) = map.into_iter().next() {- Ok(CustomOrder {- target: key.as_str().into(),- order: value,- })- } else {- Err(de::Error::custom(- "unexpected empty map in order".to_string(),- ))- }- })- }-}--#[test]-fn custom_order_serde_test() {- let order = CustomOrder {- target: OrderTarget::Key,- order: Order::Desc,- };-- let order_str = serde_json::to_string(&order).unwrap();- assert_eq!(order_str, "{\"_key\":\"desc\"}");- let order_deser = serde_json::from_str(&order_str).unwrap();-- assert_eq!(order, order_deser);-- let order_deser: serde_json::Result<CustomOrder> = serde_json::from_str("{}");- assert!(order_deser.is_err());-- let order_deser: serde_json::Result<CustomOrder> = serde_json::from_str("[]");- assert!(order_deser.is_err());-}diff --git a/crates/tantivy/src/aggregation/bucket/range.rs b/crates/tantivy/src/aggregation/bucket/range.rsdeleted file mode 100644index af86dde0..00000000--- a/crates/tantivy/src/aggregation/bucket/range.rs+++ /dev/null@@ -1,837 +0,0 @@-use std::fmt::Debug;-use std::ops::Range;--use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};--use crate::aggregation::agg_limits::ResourceLimitGuard;-use crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor;-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,- IntermediateRangeBucketEntry, IntermediateRangeBucketResult,-};-use crate::aggregation::segment_agg_result::{- build_segment_agg_collector, SegmentAggregationCollector,-};-use crate::aggregation::*;-use crate::TantivyError;--/// Provide user-defined buckets to aggregate on.-/// Two special buckets will automatically be created to cover the whole range of values.-/// The provided buckets have to be continuous.-/// During the aggregation, the values extracted from the fast_field `field` will be checked-/// against each bucket range. Note that this aggregation includes the from value and excludes the-/// to value for each range.-///-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with-/// [`RangeBucketEntry`](crate::aggregation::agg_result::RangeBucketEntry) on the-/// `AggregationCollector`.-///-/// Result type is-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with-/// [`IntermediateRangeBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateRangeBucketEntry) on the-/// `DistributedAggregationCollector`.-///-/// # Limitations/Compatibility-/// Overlapping ranges are not yet supported.-///-/// # Request JSON Format-/// ```json-/// {-/// "my_ranges": {-/// "field": "score",-/// "ranges": [-/// { "to": 3.0 },-/// { "from": 3.0, "to": 7.0 },-/// { "from": 7.0, "to": 20.0 },-/// { "from": 20.0 }-/// ]-/// }-/// }-/// ```-#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]-pub struct RangeAggregation {- /// The field to aggregate on.- pub field: String,- /// Note that this aggregation includes the from value and excludes the to value for each- /// range. Extra buckets will be created until the first to, and last from, if necessary.- pub ranges: Vec<RangeAggregationRange>,- /// Whether to return the buckets as a hash map- #[serde(default)]- pub keyed: bool,-}--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-/// The range for one range bucket.-pub struct RangeAggregationRange {- /// Custom key for the range bucket- #[serde(skip_serializing_if = "Option::is_none", default)]- pub key: Option<String>,- /// The from range value, which is inclusive in the range.- /// `None` equals to an open ended interval.- #[serde(- skip_serializing_if = "Option::is_none",- default,- deserialize_with = "deserialize_option_f64"- )]- pub from: Option<f64>,- /// The to range value, which is not inclusive in the range.- /// `None` equals to an open ended interval.- #[serde(- skip_serializing_if = "Option::is_none",- default,- deserialize_with = "deserialize_option_f64"- )]- pub to: Option<f64>,-}--impl From<Range<f64>> for RangeAggregationRange {- fn from(range: Range<f64>) -> Self {- let from = if range.start == f64::MIN {- None- } else {- Some(range.start)- };- let to = if range.end == f64::MAX {- None- } else {- Some(range.end)- };- RangeAggregationRange {- key: None,- from,- to,- }- }-}--#[derive(Clone, Debug, PartialEq)]-/// Internally used u64 range for one range bucket.-pub(crate) struct InternalRangeAggregationRange {- /// Custom key for the range bucket- key: Option<String>,- /// `u64` range value- range: Range<u64>,-}--impl From<Range<u64>> for InternalRangeAggregationRange {- fn from(range: Range<u64>) -> Self {- InternalRangeAggregationRange { key: None, range }- }-}--#[derive(Clone, Debug)]-pub(crate) struct SegmentRangeAndBucketEntry {- range: Range<u64>,- bucket: SegmentRangeBucketEntry,-}--/// The collector puts values from the fast field into the correct buckets and does a conversion to-/// the correct datatype.-#[derive(Clone, Debug)]-pub struct SegmentRangeCollector {- /// The buckets containing the aggregation data.- buckets: Vec<SegmentRangeAndBucketEntry>,- column_type: ColumnType,- pub(crate) accessor_idx: usize,-}--#[derive(Clone)]-pub(crate) struct SegmentRangeBucketEntry {- pub key: Key,- pub doc_count: u64,- pub sub_aggregation: Option<Box<dyn SegmentAggregationCollector>>,- /// The from range of the bucket. Equals `f64::MIN` when `None`.- pub from: Option<f64>,- /// The to range of the bucket. Equals `f64::MAX` when `None`. Open interval, `to` is not- /// inclusive.- pub to: Option<f64>,-}--impl Debug for SegmentRangeBucketEntry {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("SegmentRangeBucketEntry")- .field("key", &self.key)- .field("doc_count", &self.doc_count)- .field("from", &self.from)- .field("to", &self.to)- .finish()- }-}-impl SegmentRangeBucketEntry {- pub(crate) fn into_intermediate_bucket_entry(- self,- agg_with_accessor: &AggregationsWithAccessor,- ) -> crate::Result<IntermediateRangeBucketEntry> {- let mut sub_aggregation_res = IntermediateAggregationResults::default();- if let Some(sub_aggregation) = self.sub_aggregation {- sub_aggregation- .add_intermediate_aggregation_result(agg_with_accessor, &mut sub_aggregation_res)?- } else {- Default::default()- };-- Ok(IntermediateRangeBucketEntry {- key: self.key.into(),- doc_count: self.doc_count,- sub_aggregation: sub_aggregation_res,- from: self.from,- to: self.to,- })- }-}--impl SegmentAggregationCollector for SegmentRangeCollector {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let field_type = self.column_type;- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- let sub_agg = &agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;-- let buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry> = self- .buckets- .into_iter()- .map(move |range_bucket| {- Ok((- range_to_string(&range_bucket.range, &field_type)?,- range_bucket- .bucket- .into_intermediate_bucket_entry(sub_agg)?,- ))- })- .collect::<crate::Result<_>>()?;-- let bucket = IntermediateBucketResult::Range(IntermediateRangeBucketResult {- buckets,- column_type: Some(self.column_type),- });-- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.collect_block(&[doc], agg_with_accessor)- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];-- bucket_agg_accessor- .column_block_accessor- .fetch_block(docs, &bucket_agg_accessor.accessor);-- for (doc, val) in bucket_agg_accessor- .column_block_accessor- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)- {- let bucket_pos = self.get_bucket_pos(val);-- let bucket = &mut self.buckets[bucket_pos];-- bucket.bucket.doc_count += 1;- if let Some(sub_aggregation) = &mut bucket.bucket.sub_aggregation {- sub_aggregation.collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;- }- }-- Ok(())- }-- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- let sub_aggregation_accessor =- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;-- for bucket in self.buckets.iter_mut() {- if let Some(sub_agg) = bucket.bucket.sub_aggregation.as_mut() {- sub_agg.flush(sub_aggregation_accessor)?;- }- }-- Ok(())- }-}--impl SegmentRangeCollector {- pub(crate) fn from_req_and_validate(- req: &RangeAggregation,- sub_aggregation: &mut AggregationsWithAccessor,- limits: &ResourceLimitGuard,- field_type: ColumnType,- accessor_idx: usize,- ) -> crate::Result<Self> {- // The range input on the request is f64.- // We need to convert to u64 ranges, because we read the values as u64.- // The mapping from the conversion is monotonic so ordering is preserved.- let buckets: Vec<_> = extend_validate_ranges(&req.ranges, &field_type)?- .iter()- .map(|range| {- let key = range- .key- .clone()- .map(|key| Ok(Key::Str(key)))- .unwrap_or_else(|| range_to_key(&range.range, &field_type))?;- let to = if range.range.end == u64::MAX {- None- } else {- Some(f64_from_fastfield_u64(range.range.end, &field_type))- };- let from = if range.range.start == u64::MIN {- None- } else {- Some(f64_from_fastfield_u64(range.range.start, &field_type))- };- let sub_aggregation = if sub_aggregation.is_empty() {- None- } else {- Some(build_segment_agg_collector(sub_aggregation)?)- };-- Ok(SegmentRangeAndBucketEntry {- range: range.range.clone(),- bucket: SegmentRangeBucketEntry {- doc_count: 0,- sub_aggregation,- key,- from,- to,- },- })- })- .collect::<crate::Result<_>>()?;-- limits.add_memory_consumed(- buckets.len() as u64 * std::mem::size_of::<SegmentRangeAndBucketEntry>() as u64,- )?;-- Ok(SegmentRangeCollector {- buckets,- column_type: field_type,- accessor_idx,- })- }-- #[inline]- fn get_bucket_pos(&self, val: u64) -> usize {- let pos = self- .buckets- .binary_search_by_key(&val, |probe| probe.range.start)- .unwrap_or_else(|pos| pos - 1);- debug_assert!(self.buckets[pos].range.contains(&val));- pos- }-}--/// Converts the user provided f64 range value to fast field value space.-///-/// Internally fast field values are always stored as u64.-/// If the fast field has u64 `[1, 2, 5]`, these values are stored as is in the fast field.-/// A fast field with f64 `[1.0, 2.0, 5.0]` is converted to u64 space, using a-/// monotonic mapping function, so the order is preserved.-///-/// Consequently, a f64 user range 1.0..3.0 needs to be converted to fast field value space using-/// the same monotonic mapping function, so that the provided ranges contain the u64 values in the-/// fast field.-/// The alternative would be that every value read would be converted to the f64 range, but that is-/// more computational expensive when many documents are hit.-fn to_u64_range(- range: &RangeAggregationRange,- field_type: &ColumnType,-) -> crate::Result<InternalRangeAggregationRange> {- let start = if let Some(from) = range.from {- f64_to_fastfield_u64(from, field_type)- .ok_or_else(|| TantivyError::InvalidArgument("invalid field type".to_string()))?- } else {- u64::MIN- };-- let end = if let Some(to) = range.to {- f64_to_fastfield_u64(to, field_type)- .ok_or_else(|| TantivyError::InvalidArgument("invalid field type".to_string()))?- } else {- u64::MAX- };-- Ok(InternalRangeAggregationRange {- key: range.key.clone(),- range: start..end,- })-}--/// Extends the provided buckets to contain the whole value range, by inserting buckets at the-/// beginning and end and filling gaps.-fn extend_validate_ranges(- buckets: &[RangeAggregationRange],- field_type: &ColumnType,-) -> crate::Result<Vec<InternalRangeAggregationRange>> {- let mut converted_buckets = buckets- .iter()- .map(|range| to_u64_range(range, field_type))- .collect::<crate::Result<Vec<_>>>()?;-- converted_buckets.sort_by_key(|bucket| bucket.range.start);- if converted_buckets[0].range.start != u64::MIN {- converted_buckets.insert(0, (u64::MIN..converted_buckets[0].range.start).into());- }-- if converted_buckets[converted_buckets.len() - 1].range.end != u64::MAX {- converted_buckets- .push((converted_buckets[converted_buckets.len() - 1].range.end..u64::MAX).into());- }-- // fill up holes in the ranges- let find_hole = |converted_buckets: &[InternalRangeAggregationRange]| {- for (pos, ranges) in converted_buckets.windows(2).enumerate() {- if ranges[0].range.end > ranges[1].range.start {- return Err(TantivyError::InvalidArgument(format!(- "Overlapping ranges not supported range {:?}, range+1 {:?}",- ranges[0], ranges[1]- )));- }- if ranges[0].range.end != ranges[1].range.start {- return Ok(Some(pos));- }- }- Ok(None)- };-- while let Some(hole_pos) = find_hole(&converted_buckets)? {- let new_range =- converted_buckets[hole_pos].range.end..converted_buckets[hole_pos + 1].range.start;- converted_buckets.insert(hole_pos + 1, new_range.into());- }-- Ok(converted_buckets)-}--pub(crate) fn range_to_string(- range: &Range<u64>,- field_type: &ColumnType,-) -> crate::Result<String> {- // is_start is there for malformed requests, e.g. ig the user passes the range u64::MIN..0.0,- // it should be rendered as "*-0" and not "*-*"- let to_str = |val: u64, is_start: bool| {- if (is_start && val == u64::MIN) || (!is_start && val == u64::MAX) {- Ok("*".to_string())- } else if *field_type == ColumnType::DateTime {- let val = i64::from_u64(val);- format_date(val)- } else {- Ok(f64_from_fastfield_u64(val, field_type).to_string())- }- };-- Ok(format!(- "{}-{}",- to_str(range.start, true)?,- to_str(range.end, false)?- ))-}--pub(crate) fn range_to_key(range: &Range<u64>, field_type: &ColumnType) -> crate::Result<Key> {- Ok(Key::Str(range_to_string(range, field_type)?))-}--#[cfg(test)]-mod tests {-- use serde_json::Value;-- use super::*;- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::tests::{- exec_request, exec_request_with_query, get_test_index_2_segments,- get_test_index_with_num_docs,- };-- pub fn get_collector_from_ranges(- ranges: Vec<RangeAggregationRange>,- field_type: ColumnType,- ) -> SegmentRangeCollector {- let req = RangeAggregation {- field: "dummy".to_string(),- ranges,- ..Default::default()- };-- SegmentRangeCollector::from_req_and_validate(- &req,- &mut Default::default(),- &AggregationLimits::default().new_guard(),- field_type,- 0,- )- .expect("unexpected error")- }-- #[test]- fn range_fraction_test() -> crate::Result<()> {- let index = get_test_index_with_num_docs(false, 100)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "range": {- "range": {- "field": "fraction_f64",- "ranges": [- {"from": 0.0, "to": 0.1},- {"from": 0.1, "to": 0.2},- ]- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(res["range"]["buckets"][0]["key"], "*-0");- assert_eq!(res["range"]["buckets"][0]["doc_count"], 0);- assert_eq!(res["range"]["buckets"][1]["key"], "0-0.1");- assert_eq!(res["range"]["buckets"][1]["doc_count"], 10);- assert_eq!(res["range"]["buckets"][2]["key"], "0.1-0.2");- assert_eq!(res["range"]["buckets"][2]["doc_count"], 10);- assert_eq!(res["range"]["buckets"][3]["key"], "0.2-*");- assert_eq!(res["range"]["buckets"][3]["doc_count"], 80);-- Ok(())- }-- #[test]- fn range_fraction_test_with_sub_agg() -> crate::Result<()> {- let index = get_test_index_with_num_docs(false, 100)?;-- let sub_agg_req: Aggregations = serde_json::from_value(json!({- "avg": { "avg": { "field": "score_f64", } }-- }))- .unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "range": {- "range": {- "field": "fraction_f64",- "ranges": [- {"from": 0.0, "to": 0.1},- {"from": 0.1, "to": 0.2},- ]- },- "aggs": sub_agg_req- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(res["range"]["buckets"][0]["key"], "*-0");- assert_eq!(res["range"]["buckets"][0]["doc_count"], 0);- assert_eq!(res["range"]["buckets"][1]["key"], "0-0.1");- assert_eq!(res["range"]["buckets"][1]["doc_count"], 10);- assert_eq!(res["range"]["buckets"][2]["key"], "0.1-0.2");- assert_eq!(res["range"]["buckets"][2]["doc_count"], 10);- assert_eq!(res["range"]["buckets"][3]["key"], "0.2-*");- assert_eq!(res["range"]["buckets"][3]["doc_count"], 80);-- Ok(())- }-- #[test]- fn range_keyed_buckets_test() -> crate::Result<()> {- let index = get_test_index_with_num_docs(false, 100)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "range": {- "range": {- "field": "fraction_f64",- "ranges": [- {"from": 0.0, "to": 0.1},- {"from": 0.1, "to": 0.2},- ],- "keyed": true- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res,- json!({- "range": {- "buckets": {- "*-0": { "key": "*-0", "doc_count": 0, "to": 0.0},- "0-0.1": {"key": "0-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},- "0.1-0.2": {"key": "0.1-0.2", "doc_count": 10, "from": 0.1, "to": 0.2},- "0.2-*": {"key": "0.2-*", "doc_count": 80, "from": 0.2},- }- }- })- );-- Ok(())- }-- #[test]- fn range_custom_key_test() -> crate::Result<()> {- let index = get_test_index_with_num_docs(false, 100)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "range": {- "range": {- "field": "fraction_f64",- "ranges": [- {"key": "custom-key-0-to-0.1", "from": 0.0, "to": 0.1},- {"from": 0.1, "to": 0.2},- ],- "keyed": false- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res,- json!({- "range": {- "buckets": [- {"key": "*-0", "doc_count": 0, "to": 0.0},- {"key": "custom-key-0-to-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},- {"key": "0.1-0.2", "doc_count": 10, "from": 0.1, "to": 0.2},- {"key": "0.2-*", "doc_count": 80, "from": 0.2}- ]- }- })- );-- Ok(())- }-- #[test]- fn range_date_test_single_segment() -> crate::Result<()> {- range_date_test_with_opt(true)- }-- #[test]- fn range_date_test_multi_segment() -> crate::Result<()> {- range_date_test_with_opt(false)- }-- fn range_date_test_with_opt(merge_segments: bool) -> crate::Result<()> {- let index = get_test_index_2_segments(merge_segments)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "date_ranges": {- "range": {- "field": "date",- "ranges": [- {"to": 1546300800000000000i64},- {"from": 1546300800000000000i64, "to": 1546387200000000000i64},- ],- "keyed": false- },- }- }))- .unwrap();-- let agg_res = exec_request(agg_req, &index)?;-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;-- assert_eq!(- res["date_ranges"]["buckets"][0]["from_as_string"],- Value::Null- );- assert_eq!(- res["date_ranges"]["buckets"][0]["key"],- "*-2019-01-01T00:00:00Z"- );- assert_eq!(- res["date_ranges"]["buckets"][1]["from_as_string"],- "2019-01-01T00:00:00Z"- );- assert_eq!(- res["date_ranges"]["buckets"][1]["to_as_string"],- "2019-01-02T00:00:00Z"- );-- assert_eq!(- res["date_ranges"]["buckets"][2]["from_as_string"],- "2019-01-02T00:00:00Z"- );- assert_eq!(- res["date_ranges"]["buckets"][2]["to_as_string"],- Value::Null- );-- Ok(())- }-- #[test]- fn range_custom_key_keyed_buckets_test() -> crate::Result<()> {- let index = get_test_index_with_num_docs(false, 100)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "range": {- "range": {- "field": "fraction_f64",- "ranges": [- {"key": "custom-key-0-to-0.1", "from": 0.0, "to": 0.1},- ],- "keyed": true- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res,- json!({- "range": {- "buckets": {- "*-0": { "key": "*-0", "doc_count": 0, "to": 0.0},- "custom-key-0-to-0.1": {"key": "custom-key-0-to-0.1", "doc_count": 10, "from": 0.0, "to": 0.1},- "0.1-*": {"key": "0.1-*", "doc_count": 90, "from": 0.1},- }- }- })- );-- Ok(())- }-- #[test]- fn bucket_test_extend_range_hole() {- let buckets = vec![(10f64..20f64).into(), (30f64..40f64).into()];- let collector = get_collector_from_ranges(buckets, ColumnType::F64);-- let buckets = collector.buckets;- assert_eq!(buckets[0].range.start, u64::MIN);- assert_eq!(buckets[0].range.end, 10f64.to_u64());- assert_eq!(buckets[1].range.start, 10f64.to_u64());- assert_eq!(buckets[1].range.end, 20f64.to_u64());- // Added bucket to fill hole- assert_eq!(buckets[2].range.start, 20f64.to_u64());- assert_eq!(buckets[2].range.end, 30f64.to_u64());- assert_eq!(buckets[3].range.start, 30f64.to_u64());- assert_eq!(buckets[3].range.end, 40f64.to_u64());- }-- #[test]- fn bucket_test_range_conversion_special_case() {- // the monotonic conversion between f64 and u64, does not map f64::MIN.to_u64() ==- // u64::MIN, but the into trait converts f64::MIN/MAX to None- let buckets = vec![- (f64::MIN..10f64).into(),- (10f64..20f64).into(),- (20f64..f64::MAX).into(),- ];- let collector = get_collector_from_ranges(buckets, ColumnType::F64);-- let buckets = collector.buckets;- assert_eq!(buckets[0].range.start, u64::MIN);- assert_eq!(buckets[0].range.end, 10f64.to_u64());- assert_eq!(buckets[1].range.start, 10f64.to_u64());- assert_eq!(buckets[1].range.end, 20f64.to_u64());- assert_eq!(buckets[2].range.start, 20f64.to_u64());- assert_eq!(buckets[2].range.end, u64::MAX);- assert_eq!(buckets.len(), 3);- }-- #[test]- fn bucket_range_test_negative_vals() {- let buckets = vec![(-10f64..-1f64).into()];- let collector = get_collector_from_ranges(buckets, ColumnType::F64);-- let buckets = collector.buckets;- assert_eq!(&buckets[0].bucket.key.to_string(), "*--10");- assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "-1-*");- }- #[test]- fn bucket_range_test_positive_vals() {- let buckets = vec![(0f64..10f64).into()];- let collector = get_collector_from_ranges(buckets, ColumnType::F64);-- let buckets = collector.buckets;- assert_eq!(&buckets[0].bucket.key.to_string(), "*-0");- assert_eq!(&buckets[buckets.len() - 1].bucket.key.to_string(), "10-*");- }-- #[test]- fn range_binary_search_test_u64() {- let check_ranges = |ranges: Vec<RangeAggregationRange>| {- let collector = get_collector_from_ranges(ranges, ColumnType::U64);- let search = |val: u64| collector.get_bucket_pos(val);-- assert_eq!(search(u64::MIN), 0);- assert_eq!(search(9), 0);- assert_eq!(search(10), 1);- assert_eq!(search(11), 1);- assert_eq!(search(99), 1);- assert_eq!(search(100), 2);- assert_eq!(search(u64::MAX - 1), 2); // Since the end range is never included, the max- // value- };-- let ranges = vec![(10.0..100.0).into()];- check_ranges(ranges);-- let ranges = vec![- RangeAggregationRange {- key: None,- to: Some(10.0),- from: None,- },- (10.0..100.0).into(),- ];- check_ranges(ranges);-- let ranges = vec![- RangeAggregationRange {- key: None,- to: Some(10.0),- from: None,- },- (10.0..100.0).into(),- RangeAggregationRange {- key: None,- to: None,- from: Some(100.0),- },- ];- check_ranges(ranges);- }-- #[test]- fn range_binary_search_test_f64() {- let ranges = vec![(10.0..100.0).into()];-- let collector = get_collector_from_ranges(ranges, ColumnType::F64);- let search = |val: u64| collector.get_bucket_pos(val);-- assert_eq!(search(u64::MIN), 0);- assert_eq!(search(9f64.to_u64()), 0);- assert_eq!(search(10f64.to_u64()), 1);- assert_eq!(search(11f64.to_u64()), 1);- assert_eq!(search(99f64.to_u64()), 1);- assert_eq!(search(100f64.to_u64()), 2);- assert_eq!(search(u64::MAX - 1), 2); // Since the end range is never included,- // the max value- }-}diff --git a/crates/tantivy/src/aggregation/bucket/term_agg.rs b/crates/tantivy/src/aggregation/bucket/term_agg.rsdeleted file mode 100644index 7c8000b7..00000000--- a/crates/tantivy/src/aggregation/bucket/term_agg.rs+++ /dev/null@@ -1,2002 +0,0 @@-use std::fmt::Debug;-use std::net::Ipv6Addr;--use crate::columnar::column_values::CompactSpaceU64Accessor;-use crate::columnar::{- BytesColumn, ColumnType, MonotonicallyMappableToU128, MonotonicallyMappableToU64, StrColumn,-};-use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};--use super::{CustomOrder, Order, OrderTarget};-use crate::aggregation::agg_limits::MemoryConsumption;-use crate::aggregation::agg_req_with_accessor::{- AggregationWithAccessor, AggregationsWithAccessor,-};-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,- IntermediateKey, IntermediateTermBucketEntry, IntermediateTermBucketResult,-};-use crate::aggregation::segment_agg_result::{- build_segment_agg_collector, SegmentAggregationCollector,-};-use crate::aggregation::{f64_from_fastfield_u64, format_date, Key};-use crate::error::DataCorruption;-use crate::TantivyError;--/// Creates a bucket for every unique term and counts the number of occurences.-/// Note that doc_count in the response buckets equals term count here.-///-/// If the text is untokenized and single value, that means one term per document and therefore it-/// is in fact doc count.-///-/// ## Prerequisite-/// Term aggregations work only on [fast fields](`crate::fastfield`) of type `u64`, `f64`, `i64` and-/// text.-///-/// ## Document count error-/// To improve performance, results from one segment are cut off at `segment_size`. On a index with-/// a single segment this is fine. When combining results of multiple segments, terms that-/// don't make it in the top n of a shard increase the theoretical upper bound error by lowest-/// term-count.-///-/// Even with a larger `segment_size` value, doc_count values for a terms aggregation may be-/// approximate. As a result, any sub-aggregations on the terms aggregation may also be approximate.-/// `sum_other_doc_count` is the number of documents that didn’t make it into the top size-/// terms. If this is greater than 0, you can be sure that the terms agg had to throw away some-/// buckets, either because they didn’t fit into size on the root node or they didn’t fit into-/// `segment_size` on the segment node.-///-/// ## Per bucket document count error-/// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will include-/// doc_count_error_upper_bound, which is an upper bound to the error on the doc_count returned by-/// each segment. It’s the sum of the size of the largest bucket on each segment that didn’t fit-/// into segment_size.-///-/// Result type is [`BucketResult`](crate::aggregation::agg_result::BucketResult) with-/// [`BucketEntry`](crate::aggregation::agg_result::BucketEntry) on the-/// `AggregationCollector`.-///-/// Result type is-/// [`IntermediateBucketResult`](crate::aggregation::intermediate_agg_result::IntermediateBucketResult) with-/// [`IntermediateTermBucketEntry`](crate::aggregation::intermediate_agg_result::IntermediateTermBucketEntry) on the-/// `DistributedAggregationCollector`.-///-/// # Limitations/Compatibility-///-/// Each segment returns up to [segment_size](TermsAggregation::segment_size) results. This-/// differences to elasticsearch behaviour.-///-/// # Request JSON Format-/// ```json-/// {-/// "genres": {-/// "terms":{ "field": "genre" }-/// }-/// }-/// ```-///-/// /// # Response JSON Format-/// ```json-/// {-/// ...-/// "aggregations": {-/// "genres": {-/// "doc_count_error_upper_bound": 0,-/// "sum_other_doc_count": 0,-/// "buckets": [-/// { "key": "drumnbass", "doc_count": 6 },-/// { "key": "raggae", "doc_count": 4 },-/// { "key": "jazz", "doc_count": 2 }-/// ]-/// }-/// }-/// }-/// ```--#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]-pub struct TermsAggregation {- /// The field to aggregate on.- pub field: String,- /// By default, the top 10 terms with the most documents are returned.- /// Larger values for size are more expensive.- #[serde(skip_serializing_if = "Option::is_none", default)]- pub size: Option<u32>,-- /// To get more accurate results, we fetch more than `size` from each segment.- ///- /// Increasing this value is will increase the cost for more accuracy.- ///- /// Defaults to 10 * size.- #[serde(skip_serializing_if = "Option::is_none", default)]- #[serde(alias = "shard_size")]- #[serde(alias = "split_size")]- pub segment_size: Option<u32>,-- /// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will- /// include doc_count_error_upper_bound, which is an upper bound to the error on the- /// doc_count returned by each shard. It’s the sum of the size of the largest bucket on- /// each segment that didn’t fit into `shard_size`.- ///- /// Defaults to true when ordering by count desc.- #[serde(skip_serializing_if = "Option::is_none", default)]- pub show_term_doc_count_error: Option<bool>,-- /// Filter all terms that are lower than `min_doc_count`. Defaults to 1.- ///- /// **Expensive**: When set to 0, this will return all terms in the field.- #[serde(skip_serializing_if = "Option::is_none", default)]- pub min_doc_count: Option<u64>,-- /// Set the order. `String` is here a target, which is either "_count", "_key", or the name of- /// a metric sub_aggregation.- ///- /// Single value metrics like average can be addressed by its name.- /// Multi value metrics like stats are required to address their field by name e.g.- /// "stats.avg"- ///- /// Examples in JSON format:- /// { "_count": "asc" }- /// { "_key": "asc" }- /// { "average_price": "asc" }- #[serde(skip_serializing_if = "Option::is_none", default)]- pub order: Option<CustomOrder>,-- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "missing": "NO_DATA" }- ///- /// # Internal- ///- /// Internally, `missing` requires some specialized handling in some scenarios.- ///- /// Simple Case:- /// In the simplest case, we can just put the missing value in the termmap use that. In case of- /// text we put a special u64::MAX and replace it at the end with the actual missing value,- /// when loading the text.- /// Special Case 1:- /// If we have multiple columns on one field, we need to have a union on the indices on both- /// columns, to find docids without a value. That requires a special missing aggreggation.- /// Special Case 2: if the key is of type text and the column is numerical, we also need to use- /// the special missing aggregation, since there is no mechanism in the numerical column to- /// add text.- #[serde(skip_serializing_if = "Option::is_none", default)]- pub missing: Option<Key>,-}--/// Same as TermsAggregation, but with populated defaults.-#[derive(Clone, Debug, PartialEq)]-pub(crate) struct TermsAggregationInternal {- /// The field to aggregate on.- pub field: String,- /// By default, the top 10 terms with the most documents are returned.- /// Larger values for size are more expensive.- ///- /// Defaults to 10.- pub size: u32,-- /// If you set the `show_term_doc_count_error` parameter to true, the terms aggregation will- /// include doc_count_error_upper_bound, which is an upper bound to the error on the- /// doc_count returned by each shard. It’s the sum of the size of the largest bucket on- /// each segment that didn’t fit into `segment_size`.- pub show_term_doc_count_error: bool,-- /// The get more accurate results, we fetch more than `size` from each segment.- ///- /// Increasing this value is will increase the cost for more accuracy.- pub segment_size: u32,-- /// Filter all terms that are lower than `min_doc_count`. Defaults to 1.- ///- /// *Expensive*: When set to 0, this will return all terms in the field.- pub min_doc_count: u64,-- pub order: CustomOrder,- pub missing: Option<Key>,-}--impl TermsAggregationInternal {- pub(crate) fn from_req(req: &TermsAggregation) -> Self {- let size = req.size.unwrap_or(10);-- let mut segment_size = req.segment_size.unwrap_or(size * 10);-- let order = req.order.clone().unwrap_or_default();- segment_size = segment_size.max(size);- TermsAggregationInternal {- field: req.field.to_string(),- size,- segment_size,- show_term_doc_count_error: req- .show_term_doc_count_error- .unwrap_or_else(|| order == CustomOrder::default()),- min_doc_count: req.min_doc_count.unwrap_or(1),- order,- missing: req.missing.clone(),- }- }-}--#[derive(Clone, Debug, Default)]-/// Container to store term_ids/or u64 values and their buckets.-struct TermBuckets {- pub(crate) entries: FxHashMap<u64, u32>,- pub(crate) sub_aggs: FxHashMap<u64, Box<dyn SegmentAggregationCollector>>,-}--impl TermBuckets {- fn get_memory_consumption(&self) -> usize {- let sub_aggs_mem = self.sub_aggs.memory_consumption();- let buckets_mem = self.entries.memory_consumption();- sub_aggs_mem + buckets_mem- }-- fn force_flush(- &mut self,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- for sub_aggregations in &mut self.sub_aggs.values_mut() {- sub_aggregations.as_mut().flush(agg_with_accessor)?;- }- Ok(())- }-}--/// The collector puts values from the fast field into the correct buckets and does a conversion to-/// the correct datatype.-#[derive(Clone, Debug)]-pub struct SegmentTermCollector {- /// The buckets containing the aggregation data.- term_buckets: TermBuckets,- req: TermsAggregationInternal,- blueprint: Option<Box<dyn SegmentAggregationCollector>>,- column_type: ColumnType,- accessor_idx: usize,-}--pub(crate) fn get_agg_name_and_property(name: &str) -> (&str, &str) {- let (agg_name, agg_property) = name.split_once('.').unwrap_or((name, ""));- (agg_name, agg_property)-}--impl SegmentAggregationCollector for SegmentTermCollector {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];-- let bucket = self.into_intermediate_bucket_result(agg_with_accessor)?;- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.collect_block(&[doc], agg_with_accessor)- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let bucket_agg_accessor = &mut agg_with_accessor.aggs.values[self.accessor_idx];-- let mem_pre = self.get_memory_consumption();-- if let Some(missing) = bucket_agg_accessor.missing_value_for_accessor {- bucket_agg_accessor- .column_block_accessor- .fetch_block_with_missing(docs, &bucket_agg_accessor.accessor, missing);- } else {- bucket_agg_accessor- .column_block_accessor- .fetch_block(docs, &bucket_agg_accessor.accessor);- }-- for term_id in bucket_agg_accessor.column_block_accessor.iter_vals() {- let entry = self.term_buckets.entries.entry(term_id).or_default();- *entry += 1;- }- // has subagg- if let Some(blueprint) = self.blueprint.as_ref() {- for (doc, term_id) in bucket_agg_accessor- .column_block_accessor- .iter_docid_vals(docs, &bucket_agg_accessor.accessor)- {- let sub_aggregations = self- .term_buckets- .sub_aggs- .entry(term_id)- .or_insert_with(|| blueprint.clone());- sub_aggregations.collect(doc, &mut bucket_agg_accessor.sub_aggregation)?;- }- }-- let mem_delta = self.get_memory_consumption() - mem_pre;- if mem_delta > 0 {- bucket_agg_accessor- .limits- .add_memory_consumed(mem_delta as u64)?;- }-- Ok(())- }-- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- let sub_aggregation_accessor =- &mut agg_with_accessor.aggs.values[self.accessor_idx].sub_aggregation;-- self.term_buckets.force_flush(sub_aggregation_accessor)?;- Ok(())- }-}--impl SegmentTermCollector {- fn get_memory_consumption(&self) -> usize {- let self_mem = std::mem::size_of::<Self>();- let term_buckets_mem = self.term_buckets.get_memory_consumption();- self_mem + term_buckets_mem- }-- pub(crate) fn from_req_and_validate(- req: &TermsAggregation,- sub_aggregations: &mut AggregationsWithAccessor,- field_type: ColumnType,- accessor_idx: usize,- ) -> crate::Result<Self> {- if field_type == ColumnType::Bytes {- return Err(TantivyError::InvalidArgument(format!(- "terms aggregation is not supported for column type {field_type:?}"- )));- }- let term_buckets = TermBuckets::default();-- if let Some(custom_order) = req.order.as_ref() {- // Validate sub aggregtion exists- if let OrderTarget::SubAggregation(sub_agg_name) = &custom_order.target {- let (agg_name, _agg_property) = get_agg_name_and_property(sub_agg_name);-- sub_aggregations.aggs.get(agg_name).ok_or_else(|| {- TantivyError::InvalidArgument(format!(- "could not find aggregation with name {agg_name} in metric \- sub_aggregations"- ))- })?;- }- }-- let has_sub_aggregations = !sub_aggregations.is_empty();- let blueprint = if has_sub_aggregations {- let sub_aggregation = build_segment_agg_collector(sub_aggregations)?;- Some(sub_aggregation)- } else {- None- };-- Ok(SegmentTermCollector {- req: TermsAggregationInternal::from_req(req),- term_buckets,- blueprint,- column_type: field_type,- accessor_idx,- })- }-- #[inline]- pub(crate) fn into_intermediate_bucket_result(- mut self,- agg_with_accessor: &AggregationWithAccessor,- ) -> crate::Result<IntermediateBucketResult> {- let mut entries: Vec<(u64, u32)> = self.term_buckets.entries.into_iter().collect();-- let order_by_sub_aggregation =- matches!(self.req.order.target, OrderTarget::SubAggregation(_));-- match self.req.order.target {- OrderTarget::Key => {- // We rely on the fact, that term ordinals match the order of the strings- // TODO: We could have a special collector, that keeps only TOP n results at any- // time.- if self.req.order.order == Order::Desc {- entries.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.0));- } else {- entries.sort_unstable_by_key(|bucket| bucket.0);- }- }- OrderTarget::SubAggregation(_name) => {- // don't sort and cut off since it's hard to make assumptions on the quality of the- // results when cutting off du to unknown nature of the sub_aggregation (possible- // to check).- }- OrderTarget::Count => {- if self.req.order.order == Order::Desc {- entries.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.1));- } else {- entries.sort_unstable_by_key(|bucket| bucket.1);- }- }- }-- let (term_doc_count_before_cutoff, sum_other_doc_count) = if order_by_sub_aggregation {- (0, 0)- } else {- cut_off_buckets(&mut entries, self.req.segment_size as usize)- };-- let mut dict: FxHashMap<IntermediateKey, IntermediateTermBucketEntry> = Default::default();- dict.reserve(entries.len());-- let mut into_intermediate_bucket_entry =- |id, doc_count| -> crate::Result<IntermediateTermBucketEntry> {- let intermediate_entry = if self.blueprint.as_ref().is_some() {- let mut sub_aggregation_res = IntermediateAggregationResults::default();- self.term_buckets- .sub_aggs- .remove(&id)- .unwrap_or_else(|| {- panic!("Internal Error: could not find subaggregation for id {id}")- })- .add_intermediate_aggregation_result(- &agg_with_accessor.sub_aggregation,- &mut sub_aggregation_res,- )?;-- IntermediateTermBucketEntry {- doc_count,- sub_aggregation: sub_aggregation_res,- }- } else {- IntermediateTermBucketEntry {- doc_count,- sub_aggregation: Default::default(),- }- };- Ok(intermediate_entry)- };-- if self.column_type == ColumnType::Str {- let term_dict = agg_with_accessor- .str_dict_column- .as_ref()- .cloned()- .unwrap_or_else(|| {- StrColumn::wrap(BytesColumn::empty(agg_with_accessor.accessor.num_docs()))- });- let mut buffer = String::new();- for (term_id, doc_count) in entries {- let intermediate_entry = into_intermediate_bucket_entry(term_id, doc_count)?;- // Special case for missing key- if term_id == u64::MAX {- let missing_key = self- .req- .missing- .as_ref()- .expect("Found placeholder term_id but `missing` is None");- match missing_key {- Key::Str(missing) => {- buffer.clear();- buffer.push_str(missing);- dict.insert(- IntermediateKey::Str(buffer.to_string()),- intermediate_entry,- );- }- Key::F64(val) => {- buffer.push_str(&val.to_string());- dict.insert(IntermediateKey::F64(*val), intermediate_entry);- }- }- } else {- if !term_dict.ord_to_str(term_id, &mut buffer)? {- return Err(TantivyError::InternalError(format!(- "Couldn't find term_id {term_id} in dict"- )));- }- dict.insert(IntermediateKey::Str(buffer.to_string()), intermediate_entry);- }- }- if self.req.min_doc_count == 0 {- // TODO: Handle rev streaming for descending sorting by keys- let mut stream = term_dict.dictionary().stream()?;- let empty_sub_aggregation = IntermediateAggregationResults::empty_from_req(- agg_with_accessor.agg.sub_aggregation(),- );- while let Some((key, _ord)) = stream.next() {- if dict.len() >= self.req.segment_size as usize {- break;- }-- let key = IntermediateKey::Str(- std::str::from_utf8(key)- .map_err(|utf8_err| DataCorruption::comment_only(utf8_err.to_string()))?- .to_string(),- );-- dict.entry(key.clone())- .or_insert_with(|| IntermediateTermBucketEntry {- doc_count: 0,- sub_aggregation: empty_sub_aggregation.clone(),- });- }- }- } else if self.column_type == ColumnType::DateTime {- for (val, doc_count) in entries {- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;- let val = i64::from_u64(val);- let date = format_date(val)?;- dict.insert(IntermediateKey::Str(date), intermediate_entry);- }- } else if self.column_type == ColumnType::Bool {- for (val, doc_count) in entries {- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;- let val = bool::from_u64(val);- dict.insert(IntermediateKey::Bool(val), intermediate_entry);- }- } else if self.column_type == ColumnType::IpAddr {- let compact_space_accessor = agg_with_accessor- .accessor- .values- .clone()- .downcast_arc::<CompactSpaceU64Accessor>()- .map_err(|_| {- TantivyError::AggregationError(- crate::aggregation::AggregationError::InternalError(- "Type mismatch: Could not downcast to CompactSpaceU64Accessor"- .to_string(),- ),- )- })?;-- for (val, doc_count) in entries {- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;- let val: u128 = compact_space_accessor.compact_to_u128(val as u32);- let val = Ipv6Addr::from_u128(val);- dict.insert(IntermediateKey::IpAddr(val), intermediate_entry);- }- } else {- for (val, doc_count) in entries {- let intermediate_entry = into_intermediate_bucket_entry(val, doc_count)?;- let val = f64_from_fastfield_u64(val, &self.column_type);- dict.insert(IntermediateKey::F64(val), intermediate_entry);- }- };-- Ok(IntermediateBucketResult::Terms {- buckets: IntermediateTermBucketResult {- entries: dict,- sum_other_doc_count,- doc_count_error_upper_bound: term_doc_count_before_cutoff,- },- })- }-}--pub(crate) trait GetDocCount {- fn doc_count(&self) -> u64;-}-impl GetDocCount for (u64, u32) {- fn doc_count(&self) -> u64 {- self.1 as u64- }-}-impl GetDocCount for (String, IntermediateTermBucketEntry) {- fn doc_count(&self) -> u64 {- self.1.doc_count as u64- }-}--pub(crate) fn cut_off_buckets<T: GetDocCount + Debug>(- entries: &mut Vec<T>,- num_elem: usize,-) -> (u64, u64) {- let term_doc_count_before_cutoff = entries- .get(num_elem)- .map(|entry| entry.doc_count())- .unwrap_or(0);-- let sum_other_doc_count = entries- .get(num_elem..)- .map(|cut_off_range| cut_off_range.iter().map(|entry| entry.doc_count()).sum())- .unwrap_or(0);-- entries.truncate(num_elem);- (term_doc_count_before_cutoff, sum_other_doc_count)-}--#[cfg(test)]-mod tests {- use std::net::IpAddr;- use std::str::FromStr;-- use crate::common::DateTime;- use time::{Date, Month};-- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::tests::{- exec_request, exec_request_with_query, exec_request_with_query_and_memory_limit,- get_test_index_from_terms, get_test_index_from_values_and_terms,- };- use crate::aggregation::AggregationLimits;- use crate::indexer::NoMergePolicy;- use crate::schema::{IntoIpv6Addr, Schema, FAST, STRING};- use crate::{Index, IndexWriter};-- #[test]- fn terms_aggregation_test_single_segment() -> crate::Result<()> {- terms_aggregation_test_merge_segment(true)- }- #[test]- fn terms_aggregation_test() -> crate::Result<()> {- terms_aggregation_test_merge_segment(false)- }- fn terms_aggregation_test_merge_segment(merge_segments: bool) -> crate::Result<()> {- let segment_and_terms = vec![- vec!["terma"],- vec!["termb"],- vec!["termc"],- vec!["terma"],- vec!["terma"],- vec!["terma"],- vec!["termb"],- vec!["terma"],- ];- let index = get_test_index_from_terms(merge_segments, &segment_and_terms)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 1);- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "size": 2,- "segment_size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 1);-- // test min_doc_count- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "size": 2,- "min_doc_count": 3,- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(- res["my_texts"]["buckets"][1]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0); // TODO sum_other_doc_count with min_doc_count- Ok(())- }-- #[test]- fn terms_aggregation_test_order_count_single_segment() -> crate::Result<()> {- terms_aggregation_test_order_count_merge_segment(true)- }- #[test]- fn terms_aggregation_test_count_order() -> crate::Result<()> {- terms_aggregation_test_order_count_merge_segment(false)- }- fn terms_aggregation_test_order_count_merge_segment(merge_segments: bool) -> crate::Result<()> {- let segment_and_terms = vec![- vec![(5.0, "terma".to_string())],- vec![(2.0, "termb".to_string())],- vec![(2.0, "terma".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(8.0, "termb".to_string())],- vec![(5.0, "terma".to_string())],- ];- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;-- let sub_agg: Aggregations = serde_json::from_value(json!({- "avg_score": {- "avg": {- "field": "score",- }- },- "stats_score": {- "stats": {- "field": "score",- }- }- }))- .unwrap();-- // sub agg desc- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_count": "asc",- },- },- "aggs": sub_agg,- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 5.0);-- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_texts"]["buckets"][2]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 6);- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 4.5);-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // Agg on non string- //- let agg_req: Aggregations = serde_json::from_value(json!({- "my_scores1": {- "terms": {- "field": "score",- "order": {- "_count": "asc",- },- },- "aggs": sub_agg,- },- "my_scores2": {- "terms": {- "field": "score_f64",- "order": {- "_count": "asc",- },- },- "aggs": sub_agg,- },- "my_scores3": {- "terms": {- "field": "score_i64",- "order": {- "_count": "asc",- },- },- "aggs": sub_agg,- }-- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_scores1"]["buckets"][0]["key"], 8.0);- assert_eq!(res["my_scores1"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["my_scores1"]["buckets"][0]["avg_score"]["value"], 8.0);-- assert_eq!(res["my_scores1"]["buckets"][1]["key"], 2.0);- assert_eq!(res["my_scores1"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_scores1"]["buckets"][1]["avg_score"]["value"], 2.0);-- assert_eq!(res["my_scores1"]["buckets"][2]["key"], 1.0);- assert_eq!(res["my_scores1"]["buckets"][2]["doc_count"], 3);- assert_eq!(res["my_scores1"]["buckets"][2]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_scores1"]["buckets"][3]["key"], 5.0);- assert_eq!(res["my_scores1"]["buckets"][3]["doc_count"], 5);- assert_eq!(res["my_scores1"]["buckets"][3]["avg_score"]["value"], 5.0);-- assert_eq!(res["my_scores1"]["sum_other_doc_count"], 0);-- assert_eq!(res["my_scores2"]["buckets"][0]["key"], 8.0);- assert_eq!(res["my_scores2"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["my_scores2"]["buckets"][0]["avg_score"]["value"], 8.0);-- assert_eq!(res["my_scores2"]["buckets"][1]["key"], 2.0);- assert_eq!(res["my_scores2"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_scores2"]["buckets"][1]["avg_score"]["value"], 2.0);-- assert_eq!(res["my_scores2"]["buckets"][2]["key"], 1.0);- assert_eq!(res["my_scores2"]["buckets"][2]["doc_count"], 3);- assert_eq!(res["my_scores2"]["buckets"][2]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_scores2"]["sum_other_doc_count"], 0);-- assert_eq!(res["my_scores3"]["buckets"][0]["key"], 8.0);- assert_eq!(res["my_scores3"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["my_scores3"]["buckets"][0]["avg_score"]["value"], 8.0);-- assert_eq!(res["my_scores3"]["buckets"][1]["key"], 2.0);- assert_eq!(res["my_scores3"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_scores3"]["buckets"][1]["avg_score"]["value"], 2.0);-- assert_eq!(res["my_scores3"]["buckets"][2]["key"], 1.0);- assert_eq!(res["my_scores3"]["buckets"][2]["doc_count"], 3);- assert_eq!(res["my_scores3"]["buckets"][2]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_scores3"]["sum_other_doc_count"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_test_order_sub_agg_single_segment() -> crate::Result<()> {- terms_aggregation_test_order_sub_agg_merge_segment(true)- }- #[test]- fn terms_aggregation_test_sub_agg_order() -> crate::Result<()> {- terms_aggregation_test_order_sub_agg_merge_segment(false)- }- fn terms_aggregation_test_order_sub_agg_merge_segment(- merge_segments: bool,- ) -> crate::Result<()> {- let segment_and_terms = vec![- vec![(5.0, "terma".to_string())],- vec![(4.0, "termb".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(8.0, "termb".to_string())],- vec![(5.0, "terma".to_string())],- ];- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;-- let sub_agg: Aggregations = serde_json::from_value(json!({- "avg_score": {- "avg": {- "field": "score",- }- },- "stats_score": {- "stats": {- "field": "score",- }- }- }))- .unwrap();-- // sub agg desc- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "avg_score": "desc"- }- },- "aggs": sub_agg,- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 6.0);-- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);-- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // sub agg asc- //- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "avg_score": "asc"- }- },- "aggs": sub_agg,- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);-- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 6.0);-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // sub agg multi value asc- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "stats_score.avg": "asc"- }- },- "aggs": sub_agg,- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][0]["avg_score"]["value"], 1.0);-- assert_eq!(res["my_texts"]["buckets"][1]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["avg_score"]["value"], 5.0);-- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][2]["avg_score"]["value"], 6.0);-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // sub agg invalid request- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "doesnotexist": "asc"- }- },- "aggs": sub_agg,- }- }))- .unwrap();-- let res = exec_request(agg_req, &index);- assert!(res.is_err());-- Ok(())- }-- #[test]- fn terms_aggregation_test_order_key_single_segment() -> crate::Result<()> {- terms_aggregation_test_order_key_merge_segment(true)- }- #[test]- fn terms_aggregation_test_key_order() -> crate::Result<()> {- terms_aggregation_test_order_key_merge_segment(false)- }- fn terms_aggregation_test_order_key_merge_segment(merge_segments: bool) -> crate::Result<()> {- let segment_and_terms = vec![- vec![(5.0, "terma".to_string())],- vec![(4.0, "termb".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(1.0, "termc".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(5.0, "terma".to_string())],- vec![(8.0, "termb".to_string())],- vec![(5.0, "terma".to_string())],- ];- let index = get_test_index_from_values_and_terms(merge_segments, &segment_and_terms)?;-- // key asc- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "asc"- }- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 3);- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // key desc and size cut_off- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "asc"- },- "size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["doc_count"],- serde_json::Value::Null- );-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 3);-- // key asc and segment_size cut_off- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "asc"- },- "size": 2,- "segment_size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["doc_count"],- serde_json::Value::Null- );-- // key desc- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "desc"- },- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_texts"]["buckets"][2]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 5);- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);-- // key desc, size cut_off- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "desc"- },- "size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["doc_count"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 5);-- // key desc, segment_size cut_off- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "order": {- "_key": "desc"- },- "size": 2,- "segment_size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;- assert_eq!(res["my_texts"]["buckets"][0]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["doc_count"],- serde_json::Value::Null- );-- Ok(())- }-- #[test]- fn terms_aggregation_min_doc_count_special_case() -> crate::Result<()> {- let terms_per_segment = vec![- vec!["terma", "terma", "termb", "termb", "termb", "termc"],- vec!["terma", "terma", "termb", "termc", "termc"],- ];-- let index = get_test_index_from_terms(false, &terms_per_segment)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "min_doc_count": 0,- },- }- }))- .unwrap();-- // searching for terma, but min_doc_count will return all terms- let res = exec_request_with_query(agg_req, &index, Some(("string_id", "terma")))?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 0);- assert_eq!(res["my_texts"]["buckets"][2]["key"], "termc");- assert_eq!(res["my_texts"]["buckets"][2]["doc_count"], 0);- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_min_doc_count_special_case_with_sub_agg_empty_merge() -> crate::Result<()>- {- let mut schema_builder = Schema::builder();- let string_field_1 = schema_builder.add_text_field("string1", STRING | FAST);- let string_field_2 = schema_builder.add_text_field("string2", STRING | FAST);- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();- let score_field = schema_builder.add_u64_field("score", score_fieldtype);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- // writing the segment- index_writer.add_document(doc!(- string_field_1 => "A".to_string(),- string_field_2 => "hit".to_string(),- score_field => 1u64,- ))?;- index_writer.add_document(doc!(- string_field_1 => "B".to_string(),- string_field_2 => "nohit".to_string(), // this doc gets filtered in this segment,- // but the term will still be loaded because- // min_doc_count == 0- score_field => 2u64,- ))?;- index_writer.commit()?;-- index_writer.add_document(doc!(- string_field_1 => "A".to_string(),- string_field_2 => "hit".to_string(),- score_field => 2u64,- ))?;- index_writer.add_document(doc!(- string_field_1 => "B".to_string(),- string_field_2 => "hit".to_string(),- score_field => 4u64,- ))?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string1",- "min_doc_count": 0,- },- "aggs":{- "elhistogram": {- "histogram": {- "field": "score",- "interval": 1- }- }- }- }- }))- .unwrap();-- // searching for terma, but min_doc_count will return all terms- let res = exec_request_with_query(agg_req, &index, Some(("string2", "hit")))?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "A");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][0]["elhistogram"]["buckets"],- json!([{ "doc_count": 1, "key": 1.0 }, { "doc_count": 1, "key": 2.0 } ])- );- assert_eq!(res["my_texts"]["buckets"][1]["key"], "B");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);- assert_eq!(- res["my_texts"]["buckets"][1]["elhistogram"]["buckets"],- json!([ { "doc_count": 1, "key": 4.0 } ])- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_error_count_test() -> crate::Result<()> {- let terms_per_segment = vec![- vec!["terma", "terma", "termb", "termb", "termb", "termc"], /* termc doesn't make it- * from this segment */- vec!["terma", "terma", "termb", "termc", "termc"], /* termb doesn't make it from- * this segment */- ];-- let index = get_test_index_from_terms(false, &terms_per_segment)?;- assert_eq!(index.searchable_segments().unwrap().len(), 2);-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "size": 2,- "segment_size": 2- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "terma");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "termb");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 3);- assert_eq!(- res["my_texts"]["buckets"][2]["doc_count"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 4);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 2);-- // disable doc_count_error_upper_bound-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "size": 2,- "segment_size": 2,- "show_term_doc_count_error": false- },- }- }))- .unwrap();-- let res = exec_request(agg_req, &index)?;-- assert_eq!(res["my_texts"]["sum_other_doc_count"], 4);- assert_eq!(- res["my_texts"]["doc_count_error_upper_bound"],- serde_json::Value::Null- );-- Ok(())- }-- #[test]- fn terms_aggregation_term_bucket_limit() -> crate::Result<()> {- let terms: Vec<String> = (0..20_000).map(|el| el.to_string()).collect();- let terms_per_segment = vec![terms.iter().map(|el| el.as_str()).collect()];-- let index = get_test_index_from_terms(true, &terms_per_segment)?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "string_id",- "min_doc_count": 0,- },- }- }))- .unwrap();-- let res = exec_request_with_query_and_memory_limit(- agg_req,- &index,- None,- AggregationLimits::new(Some(50_000), None),- )- .unwrap_err();- assert!(res- .to_string()- .contains("Aborting aggregation because memory limit was exceeded. Limit: 50.00 KB"));-- Ok(())- }-- #[test]- fn terms_aggregation_different_tokenizer_on_ff_test() -> crate::Result<()> {- let terms = vec!["Hello Hello", "Hallo Hallo", "Hallo Hallo"];-- let index = get_test_index_from_terms(true, &[terms])?;-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "text_id",- "min_doc_count": 0,- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None).unwrap();- println!("{}", serde_json::to_string_pretty(&res).unwrap());-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hallo Hallo");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 2);-- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Hello Hello");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 1);-- Ok(())- }-- #[test]- fn test_json_format() -> crate::Result<()> {- let agg_req: Aggregations = serde_json::from_value(json!({- "term_agg_test": {- "terms": {- "field": "string_id",- "size": 2,- "segment_size": 2,- "order": {- "_key": "desc"- }- },- }- }))- .unwrap();-- let elasticsearch_compatible_json = json!(- {- "term_agg_test":{- "terms": {- "field": "string_id",- "size": 2u64,- "segment_size": 2u64,- "order": {"_key": "desc"}- }- }- });-- let agg_req_deser: Aggregations =- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())- .unwrap();- assert_eq!(agg_req, agg_req_deser);-- let elasticsearch_compatible_json = json!(- {- "term_agg_test":{- "terms": {- "field": "string_id",- "split_size": 2u64,- }- }- });-- // test alias shard_size, split_size- let agg_req: Aggregations = serde_json::from_value(json!({- "term_agg_test": {- "terms": {- "field": "string_id",- "split_size": 2,- },- }- }))- .unwrap();-- let agg_req_deser: Aggregations =- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())- .unwrap();- assert_eq!(agg_req, agg_req_deser);-- let elasticsearch_compatible_json = json!(- {- "term_agg_test":{- "terms": {- "field": "string_id",- "shard_size": 2u64,- }- }- });-- let agg_req_deser: Aggregations =- serde_json::from_str(&serde_json::to_string(&elasticsearch_compatible_json).unwrap())- .unwrap();- assert_eq!(agg_req, agg_req_deser);-- Ok(())- }- #[test]- fn terms_empty_json() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with empty json- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();- // => Segment with json, but no field partially_empty- index_writer- .add_document(doc!(json => json!({"different_field": "blue"})))- .unwrap();- index_writer.commit().unwrap();- //// => Segment with field partially_empty- index_writer- .add_document(doc!(json => json!({"partially_empty": "blue"})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "json.partially_empty"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(res["my_texts"]["buckets"][0]["key"], "blue");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["my_texts"]["buckets"][1], serde_json::Value::Null);- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_bytes() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let bytes_field = schema_builder.add_bytes_field("bytes", FAST);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer.add_document(doc!(- bytes_field => vec![1,2,3],- ))?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "bytes"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // TODO: Returning an error would be better instead of an empty result, since this is not a- // JSON field- assert_eq!(- res["my_texts"]["buckets"][0]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_multi_value() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("text", FAST);- let id_field = schema_builder.add_u64_field("id", FAST);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer.add_document(doc!(- text_field => "Hello Hello",- text_field => "Hello Hello",- id_field => 1u64,- id_field => 1u64,- ))?;- // Missing- index_writer.add_document(doc!())?;- index_writer.add_document(doc!(- text_field => "Hello Hello",- ))?;- index_writer.add_document(doc!(- text_field => "Hello Hello",- ))?;- index_writer.commit()?;- // Empty segment special case- index_writer.add_document(doc!())?;- index_writer.commit()?;- // Full segment special case- index_writer.add_document(doc!(- text_field => "Hello Hello",- id_field => 1u64,- ))?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "text",- "missing": "Empty"- },- },- "my_texts2": {- "terms": {- "field": "text",- "missing": 1337- },- },- "my_ids": {- "terms": {- "field": "id",- "missing": 1337- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hello Hello");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Empty");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["key"],- serde_json::Value::Null- );- // text field with numner as missing fallback- assert_eq!(res["my_texts2"]["buckets"][0]["key"], "Hello Hello");- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 5);- assert_eq!(res["my_texts2"]["buckets"][1]["key"], 1337.0);- assert_eq!(res["my_texts2"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts2"]["buckets"][2]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- // id field- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 3);- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }- #[test]- fn terms_aggregation_missing_simple_id() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let id_field = schema_builder.add_u64_field("id", FAST);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer.add_document(doc!(- id_field => 1u64,- ))?;- // Missing- index_writer.add_document(doc!())?;- index_writer.add_document(doc!())?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_ids": {- "terms": {- "field": "id",- "missing": 1337- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // id field- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 1);- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }-- #[test]- fn terms_aggregation_missing1() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("text", FAST);- let id_field = schema_builder.add_u64_field("id", FAST);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- index_writer.add_document(doc!(- text_field => "Hello Hello",- id_field => 1u64,- ))?;- // Missing- index_writer.add_document(doc!())?;- index_writer.add_document(doc!(- text_field => "Hello Hello",- ))?;- index_writer.add_document(doc!(- text_field => "Hello Hello",- ))?;- index_writer.commit()?;- // Empty segment special case- index_writer.add_document(doc!())?;- index_writer.commit()?;- // Full segment special case- index_writer.add_document(doc!(- text_field => "Hello Hello",- id_field => 1u64,- ))?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "text",- "missing": "Empty"- },- },- "my_texts2": {- "terms": {- "field": "text",- "missing": 1337- },- },- "my_ids": {- "terms": {- "field": "id",- "missing": 1337- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Hello Hello");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_texts"]["buckets"][1]["key"], "Empty");- assert_eq!(res["my_texts"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts"]["buckets"][2]["key"],- serde_json::Value::Null- );- // text field with numner as missing fallback- assert_eq!(res["my_texts2"]["buckets"][0]["key"], "Hello Hello");- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_texts2"]["buckets"][1]["key"], 1337.0);- assert_eq!(res["my_texts2"]["buckets"][1]["doc_count"], 2);- assert_eq!(- res["my_texts2"]["buckets"][2]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- // id field- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 4);- assert_eq!(res["my_ids"]["buckets"][1]["key"], 1.0);- assert_eq!(res["my_ids"]["buckets"][1]["doc_count"], 2);- assert_eq!(res["my_ids"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }- #[test]- fn terms_aggregation_missing_empty() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- schema_builder.add_text_field("text", FAST);- schema_builder.add_u64_field("id", FAST);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- // Empty segment special case- index_writer.add_document(doc!())?;- index_writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_texts": {- "terms": {- "field": "text",- "missing": "Empty"- },- },- "my_texts2": {- "terms": {- "field": "text",- "missing": 1337- },- },- "my_ids": {- "terms": {- "field": "id",- "missing": 1337- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["my_texts"]["buckets"][0]["key"], "Empty");- assert_eq!(res["my_texts"]["buckets"][0]["doc_count"], 1);- assert_eq!(- res["my_texts"]["buckets"][1]["key"],- serde_json::Value::Null- );- // text field with number as missing fallback- assert_eq!(res["my_texts2"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["my_texts2"]["buckets"][0]["doc_count"], 1);- assert_eq!(- res["my_texts2"]["buckets"][1]["key"],- serde_json::Value::Null- );- assert_eq!(res["my_texts"]["sum_other_doc_count"], 0);- assert_eq!(res["my_texts"]["doc_count_error_upper_bound"], 0);-- // id field- assert_eq!(res["my_ids"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["my_ids"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["my_ids"]["buckets"][1]["key"], serde_json::Value::Null);-- Ok(())- }-- #[test]- fn terms_aggregation_date() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let date_field = schema_builder.add_date_field("date_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- {- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;- writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_date": {- "terms": {- "field": "date_field"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // date_field field- assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");- assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");- assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);- assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }- #[test]- fn terms_aggregation_date_missing() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let date_field = schema_builder.add_date_field("date_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- {- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1982, Month::September, 17)?.with_hms(0, 0, 0)?)))?;- writer.add_document(doc!(date_field=>DateTime::from_primitive(Date::from_calendar_date(1983, Month::September, 27)?.with_hms(0, 0, 0)?)))?;- writer.add_document(doc!())?;- writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_date": {- "terms": {- "field": "date_field",- "missing": "1982-09-17T00:00:00Z"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // date_field field- assert_eq!(res["my_date"]["buckets"][0]["key"], "1982-09-17T00:00:00Z");- assert_eq!(res["my_date"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["my_date"]["buckets"][1]["key"], "1983-09-27T00:00:00Z");- assert_eq!(res["my_date"]["buckets"][1]["doc_count"], 1);- assert_eq!(res["my_date"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }-- #[test]- fn terms_aggregation_bool() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let field = schema_builder.add_bool_field("bool_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- {- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;- writer.add_document(doc!(field=>true))?;- writer.add_document(doc!(field=>false))?;- writer.add_document(doc!(field=>true))?;- writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_bool": {- "terms": {- "field": "bool_field"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(res["my_bool"]["buckets"][0]["key"], 1.0);- assert_eq!(res["my_bool"]["buckets"][0]["key_as_string"], "true");- assert_eq!(res["my_bool"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_bool"]["buckets"][1]["key"], 0.0);- assert_eq!(res["my_bool"]["buckets"][1]["key_as_string"], "false");- assert_eq!(res["my_bool"]["buckets"][1]["doc_count"], 1);- assert_eq!(res["my_bool"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }-- #[test]- fn terms_aggregation_ip_addr() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let field = schema_builder.add_ip_addr_field("ip_field", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- {- let mut writer = index.writer_with_num_threads(1, 15_000_000)?;- // IpV6 loopback- writer.add_document(doc!(field=>IpAddr::from_str("::1").unwrap().into_ipv6_addr()))?;- writer.add_document(doc!(field=>IpAddr::from_str("::1").unwrap().into_ipv6_addr()))?;- // IpV4- writer.add_document(- doc!(field=>IpAddr::from_str("127.0.0.1").unwrap().into_ipv6_addr()),- )?;- writer.commit()?;- }-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_bool": {- "terms": {- "field": "ip_field"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;- // print as json- // println!("{}", serde_json::to_string_pretty(&res).unwrap());-- assert_eq!(res["my_bool"]["buckets"][0]["key"], "::1");- assert_eq!(res["my_bool"]["buckets"][0]["doc_count"], 2);- assert_eq!(res["my_bool"]["buckets"][1]["key"], "127.0.0.1");- assert_eq!(res["my_bool"]["buckets"][1]["doc_count"], 1);- assert_eq!(res["my_bool"]["buckets"][2]["key"], serde_json::Value::Null);-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/bucket/term_missing_agg.rs b/crates/tantivy/src/aggregation/bucket/term_missing_agg.rsdeleted file mode 100644index bb8b295b..00000000--- a/crates/tantivy/src/aggregation/bucket/term_missing_agg.rs+++ /dev/null@@ -1,481 +0,0 @@-use rustc_hash::FxHashMap;--use crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor;-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateBucketResult,- IntermediateKey, IntermediateTermBucketEntry, IntermediateTermBucketResult,-};-use crate::aggregation::segment_agg_result::{- build_segment_agg_collector, SegmentAggregationCollector,-};--/// The specialized missing term aggregation.-#[derive(Default, Debug, Clone)]-pub struct TermMissingAgg {- missing_count: u32,- accessor_idx: usize,- sub_agg: Option<Box<dyn SegmentAggregationCollector>>,-}-impl TermMissingAgg {- pub(crate) fn new(- accessor_idx: usize,- sub_aggregations: &mut AggregationsWithAccessor,- ) -> crate::Result<Self> {- let has_sub_aggregations = !sub_aggregations.is_empty();- let sub_agg = if has_sub_aggregations {- let sub_aggregation = build_segment_agg_collector(sub_aggregations)?;- Some(sub_aggregation)- } else {- None- };-- Ok(Self {- accessor_idx,- sub_agg,- ..Default::default()- })- }-}--impl SegmentAggregationCollector for TermMissingAgg {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- let agg_with_accessor = &agg_with_accessor.aggs.values[self.accessor_idx];- let term_agg = agg_with_accessor- .agg- .agg- .as_term()- .expect("TermMissingAgg collector must be term agg req");- let missing = term_agg- .missing- .as_ref()- .expect("TermMissingAgg collector, but no missing found in agg req")- .clone();- let mut entries: FxHashMap<IntermediateKey, IntermediateTermBucketEntry> =- Default::default();-- let mut missing_entry = IntermediateTermBucketEntry {- doc_count: self.missing_count,- sub_aggregation: Default::default(),- };- if let Some(sub_agg) = self.sub_agg {- let mut res = IntermediateAggregationResults::default();- sub_agg.add_intermediate_aggregation_result(- &agg_with_accessor.sub_aggregation,- &mut res,- )?;- missing_entry.sub_aggregation = res;- }-- entries.insert(missing.into(), missing_entry);-- let bucket = IntermediateBucketResult::Terms {- buckets: IntermediateTermBucketResult {- entries,- sum_other_doc_count: 0,- doc_count_error_upper_bound: 0,- },- };-- results.push(name, IntermediateAggregationResult::Bucket(bucket))?;-- Ok(())- }-- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let agg = &mut agg_with_accessor.aggs.values[self.accessor_idx];- let has_value = agg- .accessors- .iter()- .any(|(acc, _)| acc.index.has_value(doc));- if !has_value {- self.missing_count += 1;- if let Some(sub_agg) = self.sub_agg.as_mut() {- sub_agg.collect(doc, &mut agg.sub_aggregation)?;- }- }- Ok(())- }-- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- for doc in docs {- self.collect(*doc, agg_with_accessor)?;- }- Ok(())- }-}--#[cfg(test)]-mod tests {- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::tests::exec_request_with_query;- use crate::schema::{Schema, FAST};- use crate::{Index, IndexWriter};-- #[test]- fn terms_aggregation_missing_mixed_type_mult_seg_sub_agg() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let score = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with all values numeric- index_writer- .add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))- .unwrap();- index_writer.add_document(doc!(score => 5.0))?;- // index_writer.commit().unwrap();- //// => Segment with all values text- index_writer- .add_document(doc!(score => 1.0, json => json!({"mixed_type": "blue"})))- .unwrap();- index_writer.add_document(doc!(score => 5.0))?;- // index_writer.commit().unwrap();-- // => Segment with mixed values- index_writer.add_document(doc!(json => json!({"mixed_type": "red"})))?;- index_writer.add_document(doc!(json => json!({"mixed_type": -20.5})))?;- index_writer.add_document(doc!(json => json!({"mixed_type": true})))?;- index_writer.add_document(doc!(score => 5.0))?;-- index_writer.commit().unwrap();- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- "aggs": {- "sum_score": {- "sum": {- "field": "score"- }- }- }- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(- res["replace_null"]["buckets"][0]["sum_score"]["value"],- 15.0- );- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_mixed_type_sub_agg_reg1() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let score = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with all values numeric- index_writer.add_document(doc!(score => 1.0, json => json!({"mixed_type": 10.0})))?;- index_writer.add_document(doc!(score => 5.0))?;- index_writer.add_document(doc!(score => 5.0))?;-- index_writer.commit().unwrap();- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- "aggs": {- "sum_score": {- "sum": {- "field": "score"- }- }- }- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 2);- assert_eq!(- res["replace_null"]["buckets"][0]["sum_score"]["value"],- 10.0- );- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_mult_seg_empty() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let score = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();-- index_writer.add_document(doc!(score => 5.0))?;- index_writer.commit().unwrap();- index_writer.add_document(doc!(score => 5.0))?;- index_writer.commit().unwrap();- index_writer.add_document(doc!(score => 5.0))?;-- index_writer.commit().unwrap();- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- "aggs": {- "sum_score": {- "sum": {- "field": "score"- }- }- }- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(- res["replace_null"]["buckets"][0]["sum_score"]["value"],- 15.0- );- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_single_seg_empty() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let score = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();-- index_writer.add_document(doc!(score => 5.0))?;- index_writer.add_document(doc!(score => 5.0))?;- index_writer.add_document(doc!(score => 5.0))?;-- index_writer.commit().unwrap();- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- "aggs": {- "sum_score": {- "sum": {- "field": "score"- }- }- }- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(- res["replace_null"]["buckets"][0]["sum_score"]["value"],- 15.0- );- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_mixed_type_mult_seg() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with all values numeric- index_writer- .add_document(doc!(json => json!({"mixed_type": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();- //// => Segment with all values text- index_writer- .add_document(doc!(json => json!({"mixed_type": "blue"})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();-- // => Segment with mixed values- index_writer- .add_document(doc!(json => json!({"mixed_type": "red"})))- .unwrap();- index_writer- .add_document(doc!(json => json!({"mixed_type": -20.5})))- .unwrap();- index_writer- .add_document(doc!(json => json!({"mixed_type": true})))- .unwrap();- index_writer.add_document(doc!())?;-- index_writer.commit().unwrap();- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- },- "replace_num": {- "terms": {- "field": "json.mixed_type",- "missing": 1337- },- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["replace_num"]["buckets"][0]["key"], 1337.0);- assert_eq!(res["replace_num"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_str_on_numeric_field() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with all values numeric- index_writer- .add_document(doc!(json => json!({"mixed_type": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.add_document(doc!())?;-- index_writer- .add_document(doc!(json => json!({"mixed_type": -20.5})))- .unwrap();- index_writer.add_document(doc!())?;-- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-- #[test]- fn terms_aggregation_missing_mixed_type_one_seg() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with all values numeric- index_writer- .add_document(doc!(json => json!({"mixed_type": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- //// => Segment with all values text- index_writer- .add_document(doc!(json => json!({"mixed_type": "blue"})))- .unwrap();- index_writer.add_document(doc!())?;-- // => Segment with mixed values- index_writer- .add_document(doc!(json => json!({"mixed_type": "red"})))- .unwrap();- index_writer- .add_document(doc!(json => json!({"mixed_type": -20.5})))- .unwrap();- index_writer- .add_document(doc!(json => json!({"mixed_type": true})))- .unwrap();- index_writer.add_document(doc!())?;-- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "replace_null": {- "terms": {- "field": "json.mixed_type",- "missing": "NULL"- },- },- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- // text field- assert_eq!(res["replace_null"]["buckets"][0]["key"], "NULL");- assert_eq!(res["replace_null"]["buckets"][0]["doc_count"], 3);- assert_eq!(res["replace_null"]["sum_other_doc_count"], 0);- assert_eq!(res["replace_null"]["doc_count_error_upper_bound"], 0);-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/buf_collector.rs b/crates/tantivy/src/aggregation/buf_collector.rsdeleted file mode 100644index 15be6281..00000000--- a/crates/tantivy/src/aggregation/buf_collector.rs+++ /dev/null@@ -1,83 +0,0 @@-use super::agg_req_with_accessor::AggregationsWithAccessor;-use super::intermediate_agg_result::IntermediateAggregationResults;-use super::segment_agg_result::SegmentAggregationCollector;-use crate::DocId;--pub(crate) const DOC_BLOCK_SIZE: usize = 64;-pub(crate) type DocBlock = [DocId; DOC_BLOCK_SIZE];--/// BufAggregationCollector buffers documents before calling collect_block().-#[derive(Clone)]-pub(crate) struct BufAggregationCollector {- pub(crate) collector: Box<dyn SegmentAggregationCollector>,- staged_docs: DocBlock,- num_staged_docs: usize,-}--impl std::fmt::Debug for BufAggregationCollector {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("SegmentAggregationResultsCollector")- .field("staged_docs", &&self.staged_docs[..self.num_staged_docs])- .field("num_staged_docs", &self.num_staged_docs)- .finish()- }-}--impl BufAggregationCollector {- pub fn new(collector: Box<dyn SegmentAggregationCollector>) -> Self {- Self {- collector,- num_staged_docs: 0,- staged_docs: [0; DOC_BLOCK_SIZE],- }- }-}--impl SegmentAggregationCollector for BufAggregationCollector {- #[inline]- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- Box::new(self.collector).add_intermediate_aggregation_result(agg_with_accessor, results)- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.staged_docs[self.num_staged_docs] = doc;- self.num_staged_docs += 1;- if self.num_staged_docs == self.staged_docs.len() {- self.collector- .collect_block(&self.staged_docs[..self.num_staged_docs], agg_with_accessor)?;- self.num_staged_docs = 0;- }- Ok(())- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.collector.collect_block(docs, agg_with_accessor)?;-- Ok(())- }-- #[inline]- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- self.collector- .collect_block(&self.staged_docs[..self.num_staged_docs], agg_with_accessor)?;- self.num_staged_docs = 0;-- self.collector.flush(agg_with_accessor)?;-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/collector.rs b/crates/tantivy/src/aggregation/collector.rsdeleted file mode 100644index 2b9ee6f6..00000000--- a/crates/tantivy/src/aggregation/collector.rs+++ /dev/null@@ -1,209 +0,0 @@-use super::agg_req::Aggregations;-use super::agg_req_with_accessor::AggregationsWithAccessor;-use super::agg_result::AggregationResults;-use super::buf_collector::BufAggregationCollector;-use super::intermediate_agg_result::IntermediateAggregationResults;-use super::segment_agg_result::{- build_segment_agg_collector, AggregationLimits, SegmentAggregationCollector,-};-use crate::aggregation::agg_req_with_accessor::get_aggs_with_segment_accessor_and_validate;-use crate::collector::{Collector, SegmentCollector};-use crate::index::SegmentReader;-use crate::{DocId, SegmentOrdinal, TantivyError};--/// The default max bucket count, before the aggregation fails.-pub const DEFAULT_BUCKET_LIMIT: u32 = 65000;--/// The default memory limit in bytes before the aggregation fails. 500MB-pub const DEFAULT_MEMORY_LIMIT: u64 = 500_000_000;--/// Collector for aggregations.-///-/// The collector collects all aggregations by the underlying aggregation request.-pub struct AggregationCollector {- agg: Aggregations,- limits: AggregationLimits,-}--impl AggregationCollector {- /// Create collector from aggregation request.- ///- /// Aggregation fails when the limits in `AggregationLimits` is exceeded. (memory limit and- /// bucket limit)- pub fn from_aggs(agg: Aggregations, limits: AggregationLimits) -> Self {- Self { agg, limits }- }-}--/// Collector for distributed aggregations.-///-/// The collector collects all aggregations by the underlying aggregation request.-///-/// # Purpose-/// AggregationCollector returns `IntermediateAggregationResults` and not the final-/// `AggregationResults`, so that results from different indices can be merged and then converted-/// into the final `AggregationResults` via the `into_final_result()` method.-pub struct DistributedAggregationCollector {- agg: Aggregations,- limits: AggregationLimits,-}--impl DistributedAggregationCollector {- /// Create collector from aggregation request.- ///- /// Aggregation fails when the limits in `AggregationLimits` is exceeded. (memory limit and- /// bucket limit)- pub fn from_aggs(agg: Aggregations, limits: AggregationLimits) -> Self {- Self { agg, limits }- }-}--impl Collector for DistributedAggregationCollector {- type Fruit = IntermediateAggregationResults;-- type Child = AggregationSegmentCollector;-- fn for_segment(- &self,- segment_local_id: crate::SegmentOrdinal,- reader: &crate::SegmentReader,- ) -> crate::Result<Self::Child> {- AggregationSegmentCollector::from_agg_req_and_reader(- &self.agg,- reader,- segment_local_id,- &self.limits,- )- }-- fn requires_scoring(&self) -> bool {- false- }-- fn merge_fruits(- &self,- segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,- ) -> crate::Result<Self::Fruit> {- merge_fruits(segment_fruits)- }-}--impl Collector for AggregationCollector {- type Fruit = AggregationResults;-- type Child = AggregationSegmentCollector;-- fn for_segment(- &self,- segment_local_id: crate::SegmentOrdinal,- reader: &crate::SegmentReader,- ) -> crate::Result<Self::Child> {- AggregationSegmentCollector::from_agg_req_and_reader(- &self.agg,- reader,- segment_local_id,- &self.limits,- )- }-- fn requires_scoring(&self) -> bool {- false- }-- fn merge_fruits(- &self,- segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,- ) -> crate::Result<Self::Fruit> {- let res = merge_fruits(segment_fruits)?;- res.into_final_result(self.agg.clone(), &self.limits)- }-}--fn merge_fruits(- mut segment_fruits: Vec<crate::Result<IntermediateAggregationResults>>,-) -> crate::Result<IntermediateAggregationResults> {- if let Some(fruit) = segment_fruits.pop() {- let mut fruit = fruit?;- for next_fruit in segment_fruits {- fruit.merge_fruits(next_fruit?)?;- }- Ok(fruit)- } else {- Ok(IntermediateAggregationResults::default())- }-}--/// `AggregationSegmentCollector` does the aggregation collection on a segment.-pub struct AggregationSegmentCollector {- aggs_with_accessor: AggregationsWithAccessor,- agg_collector: BufAggregationCollector,- error: Option<TantivyError>,-}--impl AggregationSegmentCollector {- /// Creates an `AggregationSegmentCollector from` an [`Aggregations`] request and a segment- /// reader. Also includes validation, e.g. checking field types and existence.- pub fn from_agg_req_and_reader(- agg: &Aggregations,- reader: &SegmentReader,- segment_ordinal: SegmentOrdinal,- limits: &AggregationLimits,- ) -> crate::Result<Self> {- let mut aggs_with_accessor =- get_aggs_with_segment_accessor_and_validate(agg, reader, segment_ordinal, limits)?;- let result =- BufAggregationCollector::new(build_segment_agg_collector(&mut aggs_with_accessor)?);- Ok(AggregationSegmentCollector {- aggs_with_accessor,- agg_collector: result,- error: None,- })- }-}--impl SegmentCollector for AggregationSegmentCollector {- type Fruit = crate::Result<IntermediateAggregationResults>;-- #[inline]- fn collect(&mut self, doc: DocId, _score: crate::Score) {- if self.error.is_some() {- return;- }- if let Err(err) = self- .agg_collector- .collect(doc, &mut self.aggs_with_accessor)- {- self.error = Some(err);- }- }-- /// The query pushes the documents to the collector via this method.- ///- /// Only valid for Collectors that ignore docs- fn collect_block(&mut self, docs: &[DocId]) {- if self.error.is_some() {- return;- }- if let Err(err) = self- .agg_collector- .collect_block(docs, &mut self.aggs_with_accessor)- {- self.error = Some(err);- }- }-- fn harvest(mut self) -> Self::Fruit {- if let Some(err) = self.error {- return Err(err);- }- self.agg_collector.flush(&mut self.aggs_with_accessor)?;-- let mut sub_aggregation_res = IntermediateAggregationResults::default();- Box::new(self.agg_collector).add_intermediate_aggregation_result(- &self.aggs_with_accessor,- &mut sub_aggregation_res,- )?;-- Ok(sub_aggregation_res)- }-}diff --git a/crates/tantivy/src/aggregation/date.rs b/crates/tantivy/src/aggregation/date.rsdeleted file mode 100644index 97befe7b..00000000--- a/crates/tantivy/src/aggregation/date.rs+++ /dev/null@@ -1,16 +0,0 @@-use time::format_description::well_known::Rfc3339;-use time::OffsetDateTime;--use crate::TantivyError;--pub(crate) fn format_date(val: i64) -> crate::Result<String> {- let datetime = OffsetDateTime::from_unix_timestamp_nanos(val as i128).map_err(|err| {- TantivyError::InvalidArgument(format!(- "Could not convert {val:?} to OffsetDateTime, err {err:?}"- ))- })?;- let key_as_string = datetime- .format(&Rfc3339)- .map_err(|_err| TantivyError::InvalidArgument("Could not serialize date".to_string()))?;- Ok(key_as_string)-}diff --git a/crates/tantivy/src/aggregation/error.rs b/crates/tantivy/src/aggregation/error.rsdeleted file mode 100644index 2034d5de..00000000--- a/crates/tantivy/src/aggregation/error.rs+++ /dev/null@@ -1,39 +0,0 @@-use crate::common::ByteCount;--use super::bucket::DateHistogramParseError;--/// Error that may occur when opening a directory-#[derive(Debug, Clone, PartialEq, Eq, Error)]-pub enum AggregationError {- /// InternalError Aggregation Request- #[error("InternalError: {0:?}")]- InternalError(String),- /// Invalid Aggregation Request- #[error("InvalidRequest: {0:?}")]- InvalidRequest(String),- /// Date histogram parse error- #[error("Date histogram parse error: {0:?}")]- DateHistogramParseError(#[from] DateHistogramParseError),- /// Memory limit exceeded- #[error(- "Aborting aggregation because memory limit was exceeded. Limit: {limit:?}, Current: \- {current:?}"- )]- MemoryExceeded {- /// Memory consumption limit- limit: ByteCount,- /// Current memory consumption- current: ByteCount,- },- /// Bucket limit exceeded- #[error(- "Aborting aggregation because bucket limit was exceeded. Limit: {limit:?}, Current: \- {current:?}"- )]- BucketLimitExceeded {- /// Bucket limit- limit: u32,- /// Current num buckets- current: u32,- },-}diff --git a/crates/tantivy/src/aggregation/intermediate_agg_result.rs b/crates/tantivy/src/aggregation/intermediate_agg_result.rsdeleted file mode 100644index b62852de..00000000--- a/crates/tantivy/src/aggregation/intermediate_agg_result.rs+++ /dev/null@@ -1,930 +0,0 @@-//! Contains the intermediate aggregation tree, that can be merged.-//! Intermediate aggregation results can be used to merge results between segments or between-//! indices.--use std::cmp::Ordering;-use std::collections::hash_map::Entry;-use std::hash::Hash;-use std::net::Ipv6Addr;--use crate::columnar::ColumnType;-use itertools::Itertools;-use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};--use super::agg_req::{Aggregation, AggregationVariants, Aggregations};-use super::agg_result::{AggregationResult, BucketResult, MetricResult, RangeBucketEntry};-use super::bucket::{- cut_off_buckets, get_agg_name_and_property, intermediate_histogram_buckets_to_final_buckets,- GetDocCount, Order, OrderTarget, RangeAggregation, TermsAggregation,-};-use super::metric::{- IntermediateAverage, IntermediateCount, IntermediateExtendedStats, IntermediateMax,- IntermediateMin, IntermediateStats, IntermediateSum, PercentilesCollector, TopHitsTopNComputer,-};-use super::segment_agg_result::AggregationLimits;-use super::{format_date, AggregationError, Key, SerializedKey};-use crate::aggregation::agg_result::{AggregationResults, BucketEntries, BucketEntry};-use crate::aggregation::bucket::TermsAggregationInternal;-use crate::TantivyError;--/// Contains the intermediate aggregation result, which is optimized to be merged with other-/// intermediate results.-///-/// Notice: This struct should not be de/serialized via JSON format.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateAggregationResults {- pub(crate) aggs_res: FxHashMap<String, IntermediateAggregationResult>,-}--#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)]-/// The key to identify a bucket.-/// This might seem redundant with `Key`, but the point is to have a different-/// Serialize implementation.-pub enum IntermediateKey {- /// Ip Addr key- IpAddr(Ipv6Addr),- /// Bool key- Bool(bool),- /// String key- Str(String),- /// `f64` key- F64(f64),-}-impl From<Key> for IntermediateKey {- fn from(value: Key) -> Self {- match value {- Key::Str(s) => Self::Str(s),- Key::F64(f) => Self::F64(f),- }- }-}-impl From<IntermediateKey> for Key {- fn from(value: IntermediateKey) -> Self {- match value {- IntermediateKey::Str(s) => Self::Str(s),- IntermediateKey::IpAddr(s) => {- // Prefer to use the IPv4 representation if possible- if let Some(ip) = s.to_ipv4_mapped() {- Self::Str(ip.to_string())- } else {- Self::Str(s.to_string())- }- }- IntermediateKey::F64(f) => Self::F64(f),- IntermediateKey::Bool(f) => Self::F64(f as u64 as f64),- }- }-}--impl Eq for IntermediateKey {}--impl std::hash::Hash for IntermediateKey {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- core::mem::discriminant(self).hash(state);- match self {- IntermediateKey::Str(text) => text.hash(state),- IntermediateKey::F64(val) => val.to_bits().hash(state),- IntermediateKey::Bool(val) => val.hash(state),- IntermediateKey::IpAddr(val) => val.hash(state),- }- }-}--impl IntermediateAggregationResults {- /// Add a result- pub fn push(&mut self, key: String, value: IntermediateAggregationResult) -> crate::Result<()> {- let entry = self.aggs_res.entry(key);- match entry {- Entry::Occupied(mut e) => {- // In case of term aggregation over different types, we need to merge the results.- e.get_mut().merge_fruits(value)?;- }- Entry::Vacant(e) => {- e.insert(value);- }- }- Ok(())- }-- /// Convert intermediate result and its aggregation request to the final result.- pub fn into_final_result(- self,- req: Aggregations,- limits: &AggregationLimits,- ) -> crate::Result<AggregationResults> {- let res = self.into_final_result_internal(&req, limits)?;- let bucket_count = res.get_bucket_count() as u32;- if bucket_count > limits.get_bucket_limit() {- return Err(TantivyError::AggregationError(- AggregationError::BucketLimitExceeded {- limit: limits.get_bucket_limit(),- current: bucket_count,- },- ));- }- Ok(res)- }-- /// Convert intermediate result and its aggregation request to the final result.- pub(crate) fn into_final_result_internal(- self,- req: &Aggregations,- limits: &AggregationLimits,- ) -> crate::Result<AggregationResults> {- let mut results: FxHashMap<String, AggregationResult> = FxHashMap::default();- for (key, agg_res) in self.aggs_res.into_iter() {- let req = req.get(key.as_str()).unwrap_or_else(|| {- panic!(- "Could not find key {:?} in request keys {:?}. This probably means that \- add_intermediate_aggregation_result passed the wrong agg object.",- key,- req.keys().collect::<Vec<_>>()- )- });- results.insert(key, agg_res.into_final_result(req, limits)?);- }- // Handle empty results- if results.len() != req.len() {- for (key, req) in req.iter() {- if !results.contains_key(key) {- let empty_res = empty_from_req(req);- results.insert(key.to_string(), empty_res.into_final_result(req, limits)?);- }- }- }-- Ok(AggregationResults(results))- }-- pub(crate) fn empty_from_req(req: &Aggregations) -> Self {- let mut aggs_res: FxHashMap<String, IntermediateAggregationResult> = FxHashMap::default();- for (key, req) in req.iter() {- let empty_res = empty_from_req(req);- aggs_res.insert(key.to_string(), empty_res);- }-- Self { aggs_res }- }-- /// Merge another intermediate aggregation result into this result.- ///- /// The order of the values need to be the same on both results. This is ensured when the same- /// (key values) are present on the underlying `VecWithNames` struct.- pub fn merge_fruits(&mut self, other: IntermediateAggregationResults) -> crate::Result<()> {- for (left, right) in self.aggs_res.values_mut().zip(other.aggs_res.into_values()) {- left.merge_fruits(right)?;- }- Ok(())- }-}--pub(crate) fn empty_from_req(req: &Aggregation) -> IntermediateAggregationResult {- use AggregationVariants::*;- match req.agg {- Terms(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Terms {- buckets: Default::default(),- }),- Range(_) => IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(- Default::default(),- )),- Histogram(_) => {- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {- buckets: Vec::new(),- is_date_agg: false,- })- }- DateHistogram(_) => {- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Histogram {- buckets: Vec::new(),- is_date_agg: true,- })- }- Average(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Average(- IntermediateAverage::default(),- )),- Count(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Count(- IntermediateCount::default(),- )),- Max(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Max(- IntermediateMax::default(),- )),- Min(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Min(- IntermediateMin::default(),- )),- Stats(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Stats(- IntermediateStats::default(),- )),- ExtendedStats(_) => IntermediateAggregationResult::Metric(- IntermediateMetricResult::ExtendedStats(IntermediateExtendedStats::default()),- ),- Sum(_) => IntermediateAggregationResult::Metric(IntermediateMetricResult::Sum(- IntermediateSum::default(),- )),- Percentiles(_) => IntermediateAggregationResult::Metric(- IntermediateMetricResult::Percentiles(PercentilesCollector::default()),- ),- TopHits(ref req) => IntermediateAggregationResult::Metric(- IntermediateMetricResult::TopHits(TopHitsTopNComputer::new(req)),- ),- }-}--/// An aggregation is either a bucket or a metric.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub enum IntermediateAggregationResult {- /// Bucket variant- Bucket(IntermediateBucketResult),- /// Metric variant- Metric(IntermediateMetricResult),-}--impl IntermediateAggregationResult {- pub(crate) fn into_final_result(- self,- req: &Aggregation,- limits: &AggregationLimits,- ) -> crate::Result<AggregationResult> {- let res = match self {- IntermediateAggregationResult::Bucket(bucket) => {- AggregationResult::BucketResult(bucket.into_final_bucket_result(req, limits)?)- }- IntermediateAggregationResult::Metric(metric) => {- AggregationResult::MetricResult(metric.into_final_metric_result(req))- }- };- Ok(res)- }- fn merge_fruits(&mut self, other: IntermediateAggregationResult) -> crate::Result<()> {- match (self, other) {- (- IntermediateAggregationResult::Bucket(b1),- IntermediateAggregationResult::Bucket(b2),- ) => b1.merge_fruits(b2),- (- IntermediateAggregationResult::Metric(m1),- IntermediateAggregationResult::Metric(m2),- ) => m1.merge_fruits(m2),- _ => panic!("aggregation result type mismatch (mixed metric and buckets)"),- }- }-}--/// Holds the intermediate data for metric results-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub enum IntermediateMetricResult {- /// Intermediate average result.- Percentiles(PercentilesCollector),- /// Intermediate average result.- Average(IntermediateAverage),- /// Intermediate count result.- Count(IntermediateCount),- /// Intermediate max result.- Max(IntermediateMax),- /// Intermediate min result.- Min(IntermediateMin),- /// Intermediate stats result.- Stats(IntermediateStats),- /// Intermediate stats result.- ExtendedStats(IntermediateExtendedStats),- /// Intermediate sum result.- Sum(IntermediateSum),- /// Intermediate top_hits result- TopHits(TopHitsTopNComputer),-}--impl IntermediateMetricResult {- fn into_final_metric_result(self, req: &Aggregation) -> MetricResult {- match self {- IntermediateMetricResult::Average(intermediate_avg) => {- MetricResult::Average(intermediate_avg.finalize().into())- }- IntermediateMetricResult::Count(intermediate_count) => {- MetricResult::Count(intermediate_count.finalize().into())- }- IntermediateMetricResult::Max(intermediate_max) => {- MetricResult::Max(intermediate_max.finalize().into())- }- IntermediateMetricResult::Min(intermediate_min) => {- MetricResult::Min(intermediate_min.finalize().into())- }- IntermediateMetricResult::Stats(intermediate_stats) => {- MetricResult::Stats(intermediate_stats.finalize())- }- IntermediateMetricResult::ExtendedStats(intermediate_stats) => {- MetricResult::ExtendedStats(intermediate_stats.finalize())- }- IntermediateMetricResult::Sum(intermediate_sum) => {- MetricResult::Sum(intermediate_sum.finalize().into())- }- IntermediateMetricResult::Percentiles(percentiles) => MetricResult::Percentiles(- percentiles- .into_final_result(req.agg.as_percentile().expect("unexpected metric type")),- ),- IntermediateMetricResult::TopHits(top_hits) => {- MetricResult::TopHits(top_hits.into_final_result())- }- }- }-- // TODO: this is our top-of-the-chain fruit merge mech- fn merge_fruits(&mut self, other: IntermediateMetricResult) -> crate::Result<()> {- match (self, other) {- (- IntermediateMetricResult::Average(avg_left),- IntermediateMetricResult::Average(avg_right),- ) => {- avg_left.merge_fruits(avg_right);- }- (- IntermediateMetricResult::Count(count_left),- IntermediateMetricResult::Count(count_right),- ) => {- count_left.merge_fruits(count_right);- }- (IntermediateMetricResult::Max(max_left), IntermediateMetricResult::Max(max_right)) => {- max_left.merge_fruits(max_right);- }- (IntermediateMetricResult::Min(min_left), IntermediateMetricResult::Min(min_right)) => {- min_left.merge_fruits(min_right);- }- (- IntermediateMetricResult::Stats(stats_left),- IntermediateMetricResult::Stats(stats_right),- ) => {- stats_left.merge_fruits(stats_right);- }- (- IntermediateMetricResult::ExtendedStats(extended_stats_left),- IntermediateMetricResult::ExtendedStats(extended_stats_right),- ) => {- extended_stats_left.merge_fruits(extended_stats_right);- }- (IntermediateMetricResult::Sum(sum_left), IntermediateMetricResult::Sum(sum_right)) => {- sum_left.merge_fruits(sum_right);- }- (- IntermediateMetricResult::Percentiles(left),- IntermediateMetricResult::Percentiles(right),- ) => {- left.merge_fruits(right)?;- }- (IntermediateMetricResult::TopHits(left), IntermediateMetricResult::TopHits(right)) => {- left.merge_fruits(right)?;- }- _ => {- panic!("incompatible fruit types in tree or missing merge_fruits handler");- }- }-- Ok(())- }-}--/// The intermediate bucket results. Internally they can be easily merged via the keys of the-/// buckets.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub enum IntermediateBucketResult {- /// This is the range entry for a bucket, which contains a key, count, from, to, and optionally- /// sub_aggregations.- Range(IntermediateRangeBucketResult),- /// This is the histogram entry for a bucket, which contains a key, count, and optionally- /// sub_aggregations.- Histogram {- /// The column_type of the underlying `Column` is DateTime- is_date_agg: bool,- /// The histogram buckets- buckets: Vec<IntermediateHistogramBucketEntry>,- },- /// Term aggregation- Terms {- /// The term buckets- buckets: IntermediateTermBucketResult,- },-}--impl IntermediateBucketResult {- pub(crate) fn into_final_bucket_result(- self,- req: &Aggregation,- limits: &AggregationLimits,- ) -> crate::Result<BucketResult> {- match self {- IntermediateBucketResult::Range(range_res) => {- let mut buckets: Vec<RangeBucketEntry> = range_res- .buckets- .into_values()- .map(|bucket| {- bucket.into_final_bucket_entry(- req.sub_aggregation(),- req.agg- .as_range()- .expect("unexpected aggregation, expected histogram aggregation"),- range_res.column_type,- limits,- )- })- .collect::<crate::Result<Vec<_>>>()?;-- buckets.sort_by(|left, right| {- left.from- .unwrap_or(f64::MIN)- .total_cmp(&right.from.unwrap_or(f64::MIN))- });-- let is_keyed = req- .agg- .as_range()- .expect("unexpected aggregation, expected range aggregation")- .keyed;- let buckets = if is_keyed {- let mut bucket_map =- FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());- for bucket in buckets {- bucket_map.insert(bucket.key.to_string(), bucket);- }- BucketEntries::HashMap(bucket_map)- } else {- BucketEntries::Vec(buckets)- };- Ok(BucketResult::Range { buckets })- }- IntermediateBucketResult::Histogram {- is_date_agg,- buckets,- } => {- let histogram_req = &req- .agg- .as_histogram()?- .expect("unexpected aggregation, expected histogram aggregation");- let buckets = intermediate_histogram_buckets_to_final_buckets(- buckets,- is_date_agg,- histogram_req,- req.sub_aggregation(),- limits,- )?;-- let buckets = if histogram_req.keyed {- let mut bucket_map =- FxHashMap::with_capacity_and_hasher(buckets.len(), Default::default());- for bucket in buckets {- bucket_map.insert(bucket.key.to_string(), bucket);- }- BucketEntries::HashMap(bucket_map)- } else {- BucketEntries::Vec(buckets)- };- Ok(BucketResult::Histogram { buckets })- }- IntermediateBucketResult::Terms { buckets: terms } => terms.into_final_result(- req.agg- .as_term()- .expect("unexpected aggregation, expected term aggregation"),- req.sub_aggregation(),- limits,- ),- }- }-- fn merge_fruits(&mut self, other: IntermediateBucketResult) -> crate::Result<()> {- match (self, other) {- (- IntermediateBucketResult::Terms {- buckets: term_res_left,- },- IntermediateBucketResult::Terms {- buckets: term_res_right,- },- ) => {- merge_maps(&mut term_res_left.entries, term_res_right.entries)?;- term_res_left.sum_other_doc_count += term_res_right.sum_other_doc_count;- term_res_left.doc_count_error_upper_bound +=- term_res_right.doc_count_error_upper_bound;- }-- (- IntermediateBucketResult::Range(range_res_left),- IntermediateBucketResult::Range(range_res_right),- ) => {- merge_maps(&mut range_res_left.buckets, range_res_right.buckets)?;- }- (- IntermediateBucketResult::Histogram {- buckets: buckets_left,- is_date_agg: _,- },- IntermediateBucketResult::Histogram {- buckets: buckets_right,- is_date_agg: _,- },- ) => {- let buckets: Result<Vec<IntermediateHistogramBucketEntry>, TantivyError> =- buckets_left- .drain(..)- .merge_join_by(buckets_right, |left, right| {- left.key.partial_cmp(&right.key).unwrap_or(Ordering::Equal)- })- .map(|either| match either {- itertools::EitherOrBoth::Both(mut left, right) => {- left.merge_fruits(right)?;- Ok(left)- }- itertools::EitherOrBoth::Left(left) => Ok(left),- itertools::EitherOrBoth::Right(right) => Ok(right),- })- .collect::<Result<_, _>>();-- *buckets_left = buckets?;- }- (IntermediateBucketResult::Range(_), _) => {- panic!("try merge on different types")- }- (IntermediateBucketResult::Histogram { .. }, _) => {- panic!("try merge on different types")- }- (IntermediateBucketResult::Terms { .. }, _) => {- panic!("try merge on different types")- }- }- Ok(())- }-}--#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-/// Range aggregation including error counts-pub struct IntermediateRangeBucketResult {- pub(crate) buckets: FxHashMap<SerializedKey, IntermediateRangeBucketEntry>,- pub(crate) column_type: Option<ColumnType>,-}--#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-/// Term aggregation including error counts-pub struct IntermediateTermBucketResult {- pub(crate) entries: FxHashMap<IntermediateKey, IntermediateTermBucketEntry>,- pub(crate) sum_other_doc_count: u64,- pub(crate) doc_count_error_upper_bound: u64,-}--impl IntermediateTermBucketResult {- pub(crate) fn into_final_result(- self,- req: &TermsAggregation,- sub_aggregation_req: &Aggregations,- limits: &AggregationLimits,- ) -> crate::Result<BucketResult> {- let req = TermsAggregationInternal::from_req(req);- let mut buckets: Vec<BucketEntry> = self- .entries- .into_iter()- .filter(|bucket| bucket.1.doc_count as u64 >= req.min_doc_count)- .map(|(key, entry)| {- let key_as_string = match key {- IntermediateKey::Bool(key) => {- let val = if key { "true" } else { "false" };- Some(val.to_string())- }- _ => None,- };- Ok(BucketEntry {- key_as_string,- key: key.into(),- doc_count: entry.doc_count as u64,- sub_aggregation: entry- .sub_aggregation- .into_final_result_internal(sub_aggregation_req, limits)?,- })- })- .collect::<crate::Result<_>>()?;-- let order = req.order.order;- match req.order.target {- OrderTarget::Key => {- buckets.sort_by(|left, right| {- if req.order.order == Order::Asc {- left.key.partial_cmp(&right.key)- } else {- right.key.partial_cmp(&left.key)- }- .expect("expected type string, which is always sortable")- });- }- OrderTarget::Count => {- if req.order.order == Order::Desc {- buckets.sort_unstable_by_key(|bucket| std::cmp::Reverse(bucket.doc_count()));- } else {- buckets.sort_unstable_by_key(|bucket| bucket.doc_count());- }- }- OrderTarget::SubAggregation(name) => {- let (agg_name, agg_property) = get_agg_name_and_property(&name);- let mut buckets_with_val = buckets- .into_iter()- .map(|bucket| {- let val = bucket- .sub_aggregation- .get_value_from_aggregation(agg_name, agg_property)?- .unwrap_or(f64::MIN);- Ok((bucket, val))- })- .collect::<crate::Result<Vec<_>>>()?;-- buckets_with_val.sort_by(|(_, val1), (_, val2)| match &order {- Order::Desc => val2.total_cmp(val1),- Order::Asc => val1.total_cmp(val2),- });- buckets = buckets_with_val- .into_iter()- .map(|(bucket, _val)| bucket)- .collect_vec();- }- }-- // We ignore _term_doc_count_before_cutoff here, because it increases the upperbound error- // only for terms that didn't make it into the top N.- //- // This can be interesting, as a value of quality of the results, but not good to check the- // actual error count for the returned terms.- let (_term_doc_count_before_cutoff, sum_other_doc_count) =- cut_off_buckets(&mut buckets, req.size as usize);-- let doc_count_error_upper_bound = if req.show_term_doc_count_error {- Some(self.doc_count_error_upper_bound)- } else {- None- };-- Ok(BucketResult::Terms {- buckets,- sum_other_doc_count: self.sum_other_doc_count + sum_other_doc_count,- doc_count_error_upper_bound,- })- }-}--trait MergeFruits {- fn merge_fruits(&mut self, other: Self) -> crate::Result<()>;-}--fn merge_maps<V: MergeFruits + Clone, T: Eq + PartialEq + Hash>(- entries_left: &mut FxHashMap<T, V>,- mut entries_right: FxHashMap<T, V>,-) -> crate::Result<()> {- for (name, entry_left) in entries_left.iter_mut() {- if let Some(entry_right) = entries_right.remove(name) {- entry_left.merge_fruits(entry_right)?;- }- }-- for (key, res) in entries_right.into_iter() {- entries_left.entry(key).or_insert(res);- }- Ok(())-}--/// This is the histogram entry for a bucket, which contains a key, count, and optionally-/// sub_aggregations.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateHistogramBucketEntry {- /// The unique the bucket is identified.- pub key: f64,- /// The number of documents in the bucket.- pub doc_count: u64,- /// The sub_aggregation in this bucket.- pub sub_aggregation: IntermediateAggregationResults,-}--impl IntermediateHistogramBucketEntry {- pub(crate) fn into_final_bucket_entry(- self,- req: &Aggregations,- limits: &AggregationLimits,- ) -> crate::Result<BucketEntry> {- Ok(BucketEntry {- key_as_string: None,- key: Key::F64(self.key),- doc_count: self.doc_count,- sub_aggregation: self- .sub_aggregation- .into_final_result_internal(req, limits)?,- })- }-}--/// This is the range entry for a bucket, which contains a key, count, and optionally-/// sub_aggregations.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateRangeBucketEntry {- /// The unique key the bucket is identified with.- pub key: IntermediateKey,- /// The number of documents in the bucket.- pub doc_count: u64,- /// The sub_aggregation in this bucket.- pub sub_aggregation: IntermediateAggregationResults,- /// The from range of the bucket. Equals `f64::MIN` when `None`.- pub from: Option<f64>,- /// The to range of the bucket. Equals `f64::MAX` when `None`.- pub to: Option<f64>,-}--impl IntermediateRangeBucketEntry {- pub(crate) fn into_final_bucket_entry(- self,- req: &Aggregations,- _range_req: &RangeAggregation,- column_type: Option<ColumnType>,- limits: &AggregationLimits,- ) -> crate::Result<RangeBucketEntry> {- let mut range_bucket_entry = RangeBucketEntry {- key: self.key.into(),- doc_count: self.doc_count,- sub_aggregation: self- .sub_aggregation- .into_final_result_internal(req, limits)?,- to: self.to,- from: self.from,- to_as_string: None,- from_as_string: None,- };-- // If we have a date type on the histogram buckets, we add the `key_as_string` field as- // rfc339- if column_type == Some(ColumnType::DateTime) {- if let Some(val) = range_bucket_entry.to {- let key_as_string = format_date(val as i64)?;- range_bucket_entry.to_as_string = Some(key_as_string);- }- if let Some(val) = range_bucket_entry.from {- let key_as_string = format_date(val as i64)?;- range_bucket_entry.from_as_string = Some(key_as_string);- }- }-- Ok(range_bucket_entry)- }-}--/// This is the term entry for a bucket, which contains a count, and optionally-/// sub_aggregations.-#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateTermBucketEntry {- /// The number of documents in the bucket.- pub doc_count: u32,- /// The sub_aggregation in this bucket.- pub sub_aggregation: IntermediateAggregationResults,-}--impl MergeFruits for IntermediateTermBucketEntry {- fn merge_fruits(&mut self, other: IntermediateTermBucketEntry) -> crate::Result<()> {- self.doc_count += other.doc_count;- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;- Ok(())- }-}--impl MergeFruits for IntermediateRangeBucketEntry {- fn merge_fruits(&mut self, other: IntermediateRangeBucketEntry) -> crate::Result<()> {- self.doc_count += other.doc_count;- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;- Ok(())- }-}--impl MergeFruits for IntermediateHistogramBucketEntry {- fn merge_fruits(&mut self, other: IntermediateHistogramBucketEntry) -> crate::Result<()> {- self.doc_count += other.doc_count;- self.sub_aggregation.merge_fruits(other.sub_aggregation)?;- Ok(())- }-}--#[cfg(test)]-mod tests {- use std::collections::HashMap;-- use pretty_assertions::assert_eq;-- use super::*;-- fn get_sub_test_tree(data: &[(String, u64)]) -> IntermediateAggregationResults {- let mut map = HashMap::new();- let mut buckets = FxHashMap::default();- for (key, doc_count) in data {- buckets.insert(- key.to_string(),- IntermediateRangeBucketEntry {- key: IntermediateKey::Str(key.to_string()),- doc_count: *doc_count,- sub_aggregation: Default::default(),- from: None,- to: None,- },- );- }- map.insert(- "my_agg_level2".to_string(),- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(- IntermediateRangeBucketResult {- buckets,- column_type: None,- },- )),- );- IntermediateAggregationResults {- aggs_res: map.into_iter().collect(),- }- }-- fn get_intermediat_tree_with_ranges(- data: &[(String, u64, String, u64)],- ) -> IntermediateAggregationResults {- let mut map = HashMap::new();- let mut buckets: FxHashMap<_, _> = Default::default();- for (key, doc_count, sub_aggregation_key, sub_aggregation_count) in data {- buckets.insert(- key.to_string(),- IntermediateRangeBucketEntry {- key: IntermediateKey::Str(key.to_string()),- doc_count: *doc_count,- from: None,- to: None,- sub_aggregation: get_sub_test_tree(&[(- sub_aggregation_key.to_string(),- *sub_aggregation_count,- )]),- },- );- }- map.insert(- "my_agg_level1".to_string(),- IntermediateAggregationResult::Bucket(IntermediateBucketResult::Range(- IntermediateRangeBucketResult {- buckets,- column_type: None,- },- )),- );- IntermediateAggregationResults {- aggs_res: map.into_iter().collect(),- }- }-- #[test]- fn test_merge_fruits_tree_1() {- let mut tree_left = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 50, "1900".to_string(), 25),- ("blue".to_string(), 30, "1900".to_string(), 30),- ]);- let tree_right = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 60, "1900".to_string(), 30),- ("blue".to_string(), 25, "1900".to_string(), 50),- ]);-- tree_left.merge_fruits(tree_right).unwrap();-- let tree_expected = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 110, "1900".to_string(), 55),- ("blue".to_string(), 55, "1900".to_string(), 80),- ]);-- assert_eq!(tree_left, tree_expected);- }-- #[test]- fn test_merge_fruits_tree_2() {- let mut tree_left = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 50, "1900".to_string(), 25),- ("blue".to_string(), 30, "1900".to_string(), 30),- ]);- let tree_right = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 60, "1900".to_string(), 30),- ("green".to_string(), 25, "1900".to_string(), 50),- ]);-- tree_left.merge_fruits(tree_right).unwrap();-- let tree_expected = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 110, "1900".to_string(), 55),- ("blue".to_string(), 30, "1900".to_string(), 30),- ("green".to_string(), 25, "1900".to_string(), 50),- ]);-- assert_eq!(tree_left, tree_expected);- }-- #[test]- fn test_merge_fruits_tree_empty() {- let mut tree_left = get_intermediat_tree_with_ranges(&[- ("red".to_string(), 50, "1900".to_string(), 25),- ("blue".to_string(), 30, "1900".to_string(), 30),- ]);-- let orig = tree_left.clone();-- tree_left- .merge_fruits(IntermediateAggregationResults::default())- .unwrap();-- assert_eq!(tree_left, orig);- }-}diff --git a/crates/tantivy/src/aggregation/metric/average.rs b/crates/tantivy/src/aggregation/metric/average.rsdeleted file mode 100644index e707f2b0..00000000--- a/crates/tantivy/src/aggregation/metric/average.rs+++ /dev/null@@ -1,136 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::*;--/// A single-value metric aggregation that computes the average of numeric values that are-/// extracted from the aggregated documents.-/// See [super::SingleMetricResult] for return value.-///-/// # JSON Format-/// ```json-/// {-/// "avg": {-/// "field": "score"-/// }-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct AverageAggregation {- /// The field name to compute the average on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl AverageAggregation {- /// Creates a new [`AverageAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- Self {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Intermediate result of the average aggregation that can be combined with other intermediate-/// results.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateAverage {- stats: IntermediateStats,-}--impl IntermediateAverage {- /// Creates a new [`IntermediateAverage`] instance from a [`SegmentStatsCollector`].- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {- Self {- stats: collector.stats,- }- }- /// Merges the other intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateAverage) {- self.stats.merge_fruits(other.stats);- }- /// Computes the final average value.- pub fn finalize(&self) -> Option<f64> {- self.stats.finalize().avg- }-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn deserialization_with_missing_test1() {- let json = r#"{- "field": "score",- "missing": "10.0"- }"#;- let avg: AverageAggregation = serde_json::from_str(json).unwrap();- assert_eq!(avg.field, "score");- assert_eq!(avg.missing, Some(10.0));- // no dot- let json = r#"{- "field": "score",- "missing": "10"- }"#;- let avg: AverageAggregation = serde_json::from_str(json).unwrap();- assert_eq!(avg.field, "score");- assert_eq!(avg.missing, Some(10.0));-- // from value- let avg: AverageAggregation = serde_json::from_value(json!({- "field": "score_f64",- "missing": 10u64,- }))- .unwrap();- assert_eq!(avg.missing, Some(10.0));- // from value- let avg: AverageAggregation = serde_json::from_value(json!({- "field": "score_f64",- "missing": 10u32,- }))- .unwrap();- assert_eq!(avg.missing, Some(10.0));- let avg: AverageAggregation = serde_json::from_value(json!({- "field": "score_f64",- "missing": 10i8,- }))- .unwrap();- assert_eq!(avg.missing, Some(10.0));- }-- #[test]- fn deserialization_with_missing_test_fail() {- let json = r#"{- "field": "score",- "missing": "a"- }"#;- let avg: Result<AverageAggregation, _> = serde_json::from_str(json);- assert!(avg.is_err());- assert!(avg- .unwrap_err()- .to_string()- .contains("Failed to parse f64 from string: \"a\""));-- // Disallow NaN- let json = r#"{- "field": "score",- "missing": "NaN"- }"#;- let avg: Result<AverageAggregation, _> = serde_json::from_str(json);- assert!(avg.is_err());- assert!(avg.unwrap_err().to_string().contains("NaN"));- }-}diff --git a/crates/tantivy/src/aggregation/metric/count.rs b/crates/tantivy/src/aggregation/metric/count.rsdeleted file mode 100644index ac550a38..00000000--- a/crates/tantivy/src/aggregation/metric/count.rs+++ /dev/null@@ -1,68 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::*;--/// A single-value metric aggregation that counts the number of values that are-/// extracted from the aggregated documents.-/// See [super::SingleMetricResult] for return value.-///-/// # JSON Format-/// ```json-/// {-/// "value_count": {-/// "field": "score"-/// }-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct CountAggregation {- /// The field name to compute the count on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl CountAggregation {- /// Creates a new [`CountAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- Self {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Intermediate result of the count aggregation that can be combined with other intermediate-/// results.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateCount {- stats: IntermediateStats,-}--impl IntermediateCount {- /// Creates a new [`IntermediateCount`] instance from a [`SegmentStatsCollector`].- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {- Self {- stats: collector.stats,- }- }- /// Merges the other intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateCount) {- self.stats.merge_fruits(other.stats);- }- /// Computes the final count value.- pub fn finalize(&self) -> Option<f64> {- Some(self.stats.finalize().count as f64)- }-}diff --git a/crates/tantivy/src/aggregation/metric/extended_stats.rs b/crates/tantivy/src/aggregation/metric/extended_stats.rsdeleted file mode 100644index d3cd59d9..00000000--- a/crates/tantivy/src/aggregation/metric/extended_stats.rs+++ /dev/null@@ -1,1181 +0,0 @@-use std::fmt::Debug;-use std::mem;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::agg_req_with_accessor::{- AggregationWithAccessor, AggregationsWithAccessor,-};-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,-};-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;-use crate::aggregation::*;-use crate::{DocId, TantivyError};--/// A multi-value metric aggregation that computes a collection of extended statistics-/// on numeric values that are extracted-/// from the aggregated documents.-/// See [`ExtendedStats`] for returned statistics.-///-/// # JSON Format-/// ```json-/// {-/// "extended_stats": {-/// "field": "score"-/// }-/// }-/// ```--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct ExtendedStatsAggregation {- /// The field name to compute the stats on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default)]- pub missing: Option<f64>,- /// The sigma parameter defines how standard_deviation_bound_are_calculated.- /// This can be a useful way to visualize variance of your data.- /// The default value is 2. Examples in JSON format:- /// { "field": "my_numbers", "sigma": "3.0" }- #[serde(default)]- pub sigma: Option<f64>,-}--impl ExtendedStatsAggregation {- /// Creates a new [`ExtendedStatsAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- ExtendedStatsAggregation {- field: field_name,- missing: None,- sigma: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Extended stats contains a collection of statistics-/// they extends stats adding variance, standard deviation-/// and bound informations-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct ExtendedStats {- /// The number of documents.- pub count: u64,- /// The sum of the fast field values.- pub sum: f64,- /// The min value of the fast field values.- pub min: Option<f64>,- /// The max value of the fast field values.- pub max: Option<f64>,- /// The average of the fast field values. `None` if count equals zero.- pub avg: Option<f64>,- /// The sum of squares of the fast field values. `None` if count equals zero.- pub sum_of_squares: Option<f64>,- /// The variance of the fast field values. `None` if count is is 0 or 1.- pub variance: Option<f64>,- /// The variance population of the fast field values, always equal to variance. `None` if count- /// is is 0 or 1.- pub variance_population: Option<f64>,- /// The variance sampling of the fast field values, always equal to variance. `None` if count- /// is is 0 or 1.- pub variance_sampling: Option<f64>,- /// The standard deviation of the fast field values. `None` if count is is 0 or 1.- pub std_deviation: Option<f64>,- /// The standard deviation of the fast field values, always equal to variance. `None` if count- /// is is 0 or 1.- pub std_deviation_population: Option<f64>,- /// The standard deviation sampling of the fast field values. `None`- /// if count is is 0 or 1.- pub std_deviation_sampling: Option<f64>,- /// The standard deviation bounds of the fast field values, always equal to variance. `None`- /// if count is is 0 or 1.- pub std_deviation_bounds: Option<StandardDeviationBounds>,-}--/// A sub struct for ExtendedStat containing deviation bounds-/// the values depend on sigma and represent-/// the bounds from the average with a distance of-/// std_deviation*sigma-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct StandardDeviationBounds {- /// upper bound -> avg + std_dev*sigma- pub upper: f64,- /// lower bound -> avg - std_dev*sigma- pub lower: f64,- /// upper bound sampling -> avg + std_dev_sampling*sigma- pub upper_sampling: f64,- /// lower bound sampling -> avg - std_dev_sampling*sigma- pub lower_sampling: f64,- /// same as upper- pub upper_population: f64,- /// same as lower- pub lower_population: f64,-}--impl ExtendedStats {- pub(crate) fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {- match agg_property {- "count" => Ok(Some(self.count as f64)),- "sum" => Ok(Some(self.sum)),- "min" => Ok(self.min),- "max" => Ok(self.max),- "avg" => Ok(self.avg),- "variance" => Ok(self.variance),- "variance_sampling" => Ok(self.variance_sampling),- "variance_population" => Ok(self.variance_population),- "sum_of_squares" => Ok(self.sum_of_squares),- "std_deviation" => Ok(self.std_deviation),- "std_deviation_sampling" => Ok(self.std_deviation_sampling),- "std_deviation_population" => Ok(self.std_deviation_population),- "std_deviation_bounds.lower" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.lower)),- "std_deviation_bounds.lower_population" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.lower_population)),- "std_deviation_bounds.lower_sampling" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.lower_sampling)),- "std_deviation_bounds.upper" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.upper)),- "std_deviation_bounds.upper_population" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.upper_population)),- "std_deviation_bounds.upper_sampling" => Ok(self- .std_deviation_bounds- .as_ref()- .map(|bounds| bounds.upper_sampling)),- _ => Err(TantivyError::InvalidArgument(format!(- "Unknown property {agg_property} on stats metric aggregation"- ))),- }- }-}--/// Intermediate result of the extended stats aggregation that can be combined with other-/// intermediate results.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateExtendedStats {- intermediate_stats: IntermediateStats,- /// The number of extracted values.- /// The sum of square values, it's referred as M2 in Welford's online algorithm- sum_of_squares: f64,- /// The sum of square values as computed by elastic search- sum_of_squares_elastic: f64,- /// The delta for sum of squares as computed by elastic search needed for the Kahan algorithm- delta_sum_for_squares_elastic: f64,- /// The mean is an intermediate value need for calculating the variance- /// as per [Welford's online algorithm](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm)- mean: f64,- /// The value used for computing standard deviation bounds- sigma: f64,-}--impl Default for IntermediateExtendedStats {- fn default() -> Self {- Self {- intermediate_stats: IntermediateStats::default(),- sum_of_squares: 0.0,- sum_of_squares_elastic: 0.0,- delta_sum_for_squares_elastic: 0.0,- mean: 0.0,- // The default value is the same of ElasticSearch- sigma: 2.0,- }- }-}--impl IntermediateExtendedStats {- /// Creates a new IntermediateExtendedStats using an option- /// containing the sigma to be used for calculating bound values.- pub fn with_sigma(sigma: Option<f64>) -> Self {- Self {- intermediate_stats: IntermediateStats::default(),- sum_of_squares: 0.0,- sum_of_squares_elastic: 0.0,- delta_sum_for_squares_elastic: 0.0,- mean: 0.0,- // The default value is the same of ElasticSearch- sigma: sigma.unwrap_or(2.0),- }- }- /// Merges the other stats intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateExtendedStats) {- if other.intermediate_stats.count == 0 {- return;- }- if self.intermediate_stats.count == 0 {- let _ = mem::replace(self, other);- return;- }- let new_count = self.intermediate_stats.count + other.intermediate_stats.count;- let delta = other.mean - self.mean;- self.sum_of_squares += other.sum_of_squares- + delta- * delta- * self.intermediate_stats.count as f64- * other.intermediate_stats.count as f64- / new_count as f64;- self.mean = (self.intermediate_stats.sum + other.intermediate_stats.sum) / new_count as f64;- self.sum_of_squares_elastic += other.sum_of_squares_elastic;- self.delta_sum_for_squares_elastic += other.delta_sum_for_squares_elastic;- self.intermediate_stats- .merge_fruits(other.intermediate_stats);- }-- /// Computes the final stats value.- pub fn finalize(&self) -> Box<ExtendedStats> {- let (min, max, avg, sum_of_squares) = if self.intermediate_stats.count == 0 {- (None, None, None, None)- } else {- (- Some(self.intermediate_stats.min),- Some(self.intermediate_stats.max),- Some(self.mean),- Some(self.sum_of_squares_elastic),- )- };- let (variance, variance_sampling) = if self.intermediate_stats.count <= 1 {- (None, None)- } else {- (- Some(self.sum_of_squares / self.intermediate_stats.count as f64),- Some(self.sum_of_squares / (self.intermediate_stats.count - 1) as f64),- )- };- let std_deviation = variance.map(|v| v.sqrt());- let std_deviation_sampling = variance_sampling.map(|v| v.sqrt());- let std_deviation_bounds =- if let (Some(std_deviation_val), Some(std_deviation_sampling_val)) =- (std_deviation, std_deviation_sampling)- {- let upper = self.mean + std_deviation_val * self.sigma;- let lower = self.mean - std_deviation_val * self.sigma;- let upper_sampling = self.mean + std_deviation_sampling_val * self.sigma;- let lower_sampling = self.mean - std_deviation_sampling_val * self.sigma;- Some(StandardDeviationBounds {- upper,- lower,- upper_sampling,- lower_sampling,- upper_population: upper,- lower_population: lower,- })- } else {- None- };- Box::new(ExtendedStats {- count: self.intermediate_stats.count,- sum: self.intermediate_stats.sum,- min,- max,- avg,- sum_of_squares,- variance,- variance_population: variance,- variance_sampling,- std_deviation,- std_deviation_population: std_deviation,- std_deviation_sampling,- std_deviation_bounds,- })- }-- fn update_variance(&mut self, value: f64) {- let delta = value - self.mean;- // this is not what the Welford's online algorithm prescribes but- // using the pseudo code from wikipedia there was a small rounding- // error (in 15th decimal place) that caused a test- //(test_aggregation_level1 in agg_test.rs)- // failure- self.mean = self.intermediate_stats.sum / self.intermediate_stats.count as f64;- // self.mean += delta / self.count as f64;- let delta2 = value - self.mean;- self.sum_of_squares += delta * delta2;- }-- #[inline]- fn collect(&mut self, value: f64) {- self.intermediate_stats.collect(value);- // kahan algorithm for sum_of_squares_elastic- let y = value * value - self.delta_sum_for_squares_elastic;- let t = self.sum_of_squares_elastic + y;- self.delta_sum_for_squares_elastic = (t - self.sum_of_squares_elastic) - y;- self.sum_of_squares_elastic = t;- self.update_variance(value);- }-}--#[derive(Clone, Debug, PartialEq)]-pub(crate) struct SegmentExtendedStatsCollector {- missing: Option<u64>,- field_type: ColumnType,- pub(crate) extended_stats: IntermediateExtendedStats,- pub(crate) accessor_idx: usize,- val_cache: Vec<u64>,-}--impl SegmentExtendedStatsCollector {- pub fn from_req(- field_type: ColumnType,- sigma: Option<f64>,- accessor_idx: usize,- missing: Option<f64>,- ) -> Self {- let missing = missing.and_then(|val| f64_to_fastfield_u64(val, &field_type));- Self {- field_type,- extended_stats: IntermediateExtendedStats::with_sigma(sigma),- accessor_idx,- missing,- val_cache: Default::default(),- }- }- #[inline]- pub(crate) fn collect_block_with_field(- &mut self,- docs: &[DocId],- agg_accessor: &mut AggregationWithAccessor,- ) {- if let Some(missing) = self.missing.as_ref() {- agg_accessor.column_block_accessor.fetch_block_with_missing(- docs,- &agg_accessor.accessor,- *missing,- );- } else {- agg_accessor- .column_block_accessor- .fetch_block(docs, &agg_accessor.accessor);- }- for val in agg_accessor.column_block_accessor.iter_vals() {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.extended_stats.collect(val1);- }- }-}--impl SegmentAggregationCollector for SegmentExtendedStatsCollector {- #[inline]- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- results.push(- name,- IntermediateAggregationResult::Metric(IntermediateMetricResult::ExtendedStats(- self.extended_stats,- )),- )?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;- if let Some(missing) = self.missing {- let mut has_val = false;- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.extended_stats.collect(val1);- has_val = true;- }- if !has_val {- self.extended_stats- .collect(f64_from_fastfield_u64(missing, &self.field_type));- }- } else {- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.extended_stats.collect(val1);- }- }-- Ok(())- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];- self.collect_block_with_field(docs, field);- Ok(())- }-}--#[cfg(test)]-mod tests {-- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::metric::IntermediateExtendedStats;- use crate::aggregation::tests::get_test_index_from_values;- use crate::aggregation::AggregationCollector;- use crate::assert_nearly_equals;- use crate::query::AllQuery;-- const EPSILON_FOR_TEST: f64 = 0.000000000002;-- #[test]- fn test_aggregation_extended_stats_no_variance() -> crate::Result<()> {- let values = vec![1.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "my_stats": {- "extended_stats": {- "field": "score_f64",- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "count")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "min")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "max")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "avg")?- .unwrap(),- 1.0- );-- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_population")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?- .is_none());- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum_of_squares")?- .unwrap(),- 1.0- );- assert!(agg_res- .get_value_from_aggregation("my_stats", "variance_population")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "variance")?- .is_none());- assert!(agg_res- .get_value_from_aggregation("my_stats", "variance_sampling")?- .is_none());-- Ok(())- }-- #[test]- fn test_aggregation_extended_stats() -> crate::Result<()> {- let values = vec![1.0, 3.0, 4.0, 5.0, 8.0, 10.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "my_stats": {- "extended_stats": {- "field": "score_f64",- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- const EXPECTED_VARIANCE: f64 = 9.138888888888888;- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "count")?- .unwrap(),- 6.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "min")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "max")?- .unwrap(),- 10.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum")?- .unwrap(),- 31.0- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "avg")?- .unwrap(),- 5.166666666666667,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_population")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?- .unwrap(),- 3.311595788538611,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?- .unwrap(),- -0.8794523824056837,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?- .unwrap(),- -0.8794523824056837,- 0.00000000000001- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?- .unwrap(),- -1.4565249104105549,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?- .unwrap(),- 11.212785715739017,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?- .unwrap(),- 11.212785715739017,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?- .unwrap(),- 11.78985824374389,- EPSILON_FOR_TEST- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum_of_squares")?- .unwrap(),- 215.0- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_population")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_sampling")?- .unwrap(),- 10.966666666666663,- EPSILON_FOR_TEST- );-- Ok(())- }-- #[test]- fn test_aggregation_extended_stats_with_sigma() -> crate::Result<()> {- let values = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "my_stats": {- "extended_stats": {- "field": "score_f64",- "sigma": 1.5- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- const EXPECTED_VARIANCE: f64 = 2.9166666666666665;- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "count")?- .unwrap(),- 6.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "min")?- .unwrap(),- 1.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "max")?- .unwrap(),- 6.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum")?- .unwrap(),- 21.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "avg")?- .unwrap(),- 3.5- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_population")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?- .unwrap(),- 1.8708286933869709,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?- .unwrap(),- 0.9382623085101005,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?- .unwrap(),- 0.9382623085101005,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?- .unwrap(),- 0.6937569599195434,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?- .unwrap(),- 6.061737691489899,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?- .unwrap(),- 6.061737691489899,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?- .unwrap(),- 6.3062430400804566,- EPSILON_FOR_TEST- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum_of_squares")?- .unwrap(),- 91.0- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_population")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_sampling")?- .unwrap(),- 3.5,- EPSILON_FOR_TEST- );-- Ok(())- }-- #[test]- fn test_aggregation_extended_stats_with_variance_similar_to_mean() -> crate::Result<()> {- let values = vec![50.01, 50.02, 50.01, 50.03, 50.01, 50.02];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "my_stats": {- "extended_stats": {- "field": "score_f64",- "sigma": 1.5- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- const EXPECTED_VARIANCE: f64 = 5.5555555555608854e-5;- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "count")?- .unwrap(),- 6.0- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "min")?- .unwrap(),- 50.01- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "max")?- .unwrap(),- 50.03- );- assert_eq!(- agg_res- .get_value_from_aggregation("my_stats", "sum")?- .unwrap(),- 300.1- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "avg")?- .unwrap(),- 50.01666666666667,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_population")?- .unwrap(),- EXPECTED_VARIANCE.sqrt(),- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_sampling")?- .unwrap(),- 0.008164965809279263,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower")?- .unwrap(),- 50.00548632677917,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_population")?- .unwrap(),- 50.00548632677917,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.lower_sampling")?- .unwrap(),- 50.00441921795275,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper")?- .unwrap(),- 50.027847006554175,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_population")?- .unwrap(),- 50.027847006554175,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "std_deviation_bounds.upper_sampling")?- .unwrap(),- 50.028914115380594,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "sum_of_squares")?- .unwrap(),- 15010.002,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_population")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance")?- .unwrap(),- EXPECTED_VARIANCE,- EPSILON_FOR_TEST- );- assert_nearly_equals!(- agg_res- .get_value_from_aggregation("my_stats", "variance_sampling")?- .unwrap(),- 6.666666666670718e-5,- EPSILON_FOR_TEST- );-- Ok(())- }-- #[test]- fn extended_stat_zero_value() {- let intermediate_extend_stats = IntermediateExtendedStats::default();- let extended_stats = intermediate_extend_stats.finalize();- assert!(extended_stats.variance.is_none());- assert!(extended_stats.variance_population.is_none());- assert!(extended_stats.variance_sampling.is_none());- assert!(extended_stats.sum_of_squares.is_none());- assert!(extended_stats.std_deviation.is_none());- assert!(extended_stats.std_deviation_population.is_none());- assert!(extended_stats.std_deviation_sampling.is_none());- assert!(extended_stats.std_deviation_bounds.is_none());- }-- #[test]- fn extended_stat_one_value() {- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.collect(1.0f64);- let extended_stats = intermediate_extend_stats.finalize();- assert!(extended_stats.variance.is_none());- assert!(extended_stats.variance_population.is_none());- assert!(extended_stats.variance_sampling.is_none());- assert!(extended_stats.std_deviation.is_none());- assert!(extended_stats.std_deviation_population.is_none());- assert!(extended_stats.std_deviation_sampling.is_none());- assert!(extended_stats.std_deviation_bounds.is_none());- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(1.0f64, sum_of_squares);- }-- #[test]- fn extended_stat_multiple_values() {- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.collect(1.0f64);- intermediate_extend_stats.collect(3.0f64);- intermediate_extend_stats.collect(4.0f64);- intermediate_extend_stats.collect(5.0f64);- intermediate_extend_stats.collect(8.0f64);- intermediate_extend_stats.collect(10.0f64);- let extended_stats = intermediate_extend_stats.finalize();- let variance = extended_stats.variance.unwrap();- const EXPECTED_VARIANCE: f64 = 9.138888888888888;- assert_eq!(EXPECTED_VARIANCE, variance);- let variance_population = extended_stats.variance_population.unwrap();- assert_eq!(EXPECTED_VARIANCE, variance_population);- let variance_sampling = extended_stats.variance_sampling.unwrap();- assert_eq!(10.966666666666665f64, variance_sampling);- let std_deviation = extended_stats.std_deviation.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);- let std_deviation_population = extended_stats.std_deviation_population.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();- assert_eq!(10.966666666666665f64.sqrt(), std_deviation_sampling);- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(215.0, sum_of_squares);- let avg = extended_stats.avg.unwrap();- assert_eq!(5.166666666666667, avg);- }-- #[test]- fn merge_empty_with_one_value() {- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();- intermediate_extend_stats1.collect(1.0f64);- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);- let extended_stats = intermediate_extend_stats.finalize();- assert!(extended_stats.variance.is_none());- assert!(extended_stats.variance_population.is_none());- assert!(extended_stats.variance_sampling.is_none());- assert!(extended_stats.std_deviation.is_none());- assert!(extended_stats.std_deviation_population.is_none());- assert!(extended_stats.std_deviation_sampling.is_none());- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(1.0f64, sum_of_squares);- }-- #[test]- fn merge_empty_with_multiple_values() {- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();- intermediate_extend_stats1.collect(1.0f64);- intermediate_extend_stats1.collect(2.0f64);- intermediate_extend_stats1.collect(3.0f64);- intermediate_extend_stats1.collect(4.0f64);- intermediate_extend_stats1.collect(5.0f64);-- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);- let extended_stats = intermediate_extend_stats.finalize();- const EXPECTED_VARIANCE: f64 = 2.0;- let variance = extended_stats.variance.unwrap();- assert_eq!(EXPECTED_VARIANCE, variance);- let variance_population = extended_stats.variance_population.unwrap();- assert_eq!(EXPECTED_VARIANCE, variance_population);- let variance_sampling = extended_stats.variance_sampling.unwrap();- assert_eq!(2.5f64, variance_sampling);- let std_deviation = extended_stats.std_deviation.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);- let std_deviation_population = extended_stats.std_deviation_population.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();- assert_eq!(2.5f64.sqrt(), std_deviation_sampling);- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(55f64, sum_of_squares);- }-- #[test]- fn merge_non_empty_extended_stats() {- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();- intermediate_extend_stats1.collect(3.0f64);- intermediate_extend_stats1.collect(4.0f64);- intermediate_extend_stats1.collect(5.0f64);-- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.collect(1.0f64);- intermediate_extend_stats.collect(2.0f64);- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);- let extended_stats = intermediate_extend_stats.finalize();-- let variance = extended_stats.variance.unwrap();- assert_eq!(2.0f64, variance);- let variance_population = extended_stats.variance_population.unwrap();- assert_eq!(2.0f64, variance_population);- let variance_sampling = extended_stats.variance_sampling.unwrap();- assert_eq!(2.5f64, variance_sampling);- let std_deviation = extended_stats.std_deviation.unwrap();- assert_eq!(2.0f64.sqrt(), std_deviation);- let std_deviation_population = extended_stats.std_deviation_population.unwrap();- assert_eq!(2.0f64.sqrt(), std_deviation_population);- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();- assert_eq!(2.5f64.sqrt(), std_deviation_sampling);- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(55f64, sum_of_squares);-- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.collect(1.0f64);- intermediate_extend_stats.collect(3.0f64);- intermediate_extend_stats.collect(4.0f64);- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();- intermediate_extend_stats1.collect(5.0f64);- intermediate_extend_stats1.collect(8.0f64);- intermediate_extend_stats1.collect(10.0f64);- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);- let extended_stats = intermediate_extend_stats.finalize();- const EXPECTED_VARIANCE: f64 = 9.138888888888888;- let variance = extended_stats.variance.unwrap();- assert_eq!(EXPECTED_VARIANCE, variance);- let variance_population = extended_stats.variance_population.unwrap();- assert_eq!(EXPECTED_VARIANCE, variance_population);- let variance_sampling = extended_stats.variance_sampling.unwrap();- assert_eq!(10.966666666666665f64, variance_sampling);- let std_deviation = extended_stats.std_deviation.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation);- let std_deviation_population = extended_stats.std_deviation_population.unwrap();- assert_eq!(EXPECTED_VARIANCE.sqrt(), std_deviation_population);- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();- assert_eq!(10.966666666666665f64.sqrt(), std_deviation_sampling);- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(215f64, sum_of_squares);- let avg = extended_stats.avg.unwrap();- assert_eq!(5.166666666666667, avg);- }-- #[test]- fn merge_and_then_collect_non_empty_extended_stats() {- let mut intermediate_extend_stats = IntermediateExtendedStats::default();- intermediate_extend_stats.collect(1.0f64);- intermediate_extend_stats.collect(3.0f64);-- let mut intermediate_extend_stats1 = IntermediateExtendedStats::default();- intermediate_extend_stats1.collect(5.0f64);- intermediate_extend_stats1.collect(8.0f64);- intermediate_extend_stats1.collect(10.0f64);- intermediate_extend_stats.merge_fruits(intermediate_extend_stats1);- intermediate_extend_stats.collect(4.0f64);- let extended_stats = intermediate_extend_stats.finalize();- const EXPECTED_VARIANCE: f64 = 9.138888888888888;- let variance = extended_stats.variance.unwrap();- assert_nearly_equals!(EXPECTED_VARIANCE, variance, EPSILON_FOR_TEST);- let variance_population = extended_stats.variance_population.unwrap();- assert_nearly_equals!(EXPECTED_VARIANCE, variance_population, EPSILON_FOR_TEST);- let variance_sampling = extended_stats.variance_sampling.unwrap();- assert_nearly_equals!(10.966666666666665, variance_sampling, EPSILON_FOR_TEST);- let std_deviation = extended_stats.std_deviation.unwrap();- assert_nearly_equals!(EXPECTED_VARIANCE.sqrt(), std_deviation, EPSILON_FOR_TEST);- let std_deviation_population = extended_stats.std_deviation_population.unwrap();- assert_nearly_equals!(- EXPECTED_VARIANCE.sqrt(),- std_deviation_population,- EPSILON_FOR_TEST- );- let std_deviation_sampling = extended_stats.std_deviation_sampling.unwrap();- assert_nearly_equals!(- 10.966666666666665_f64.sqrt(),- std_deviation_sampling,- EPSILON_FOR_TEST- );- let sum_of_squares = extended_stats.sum_of_squares.unwrap();- assert_eq!(215.0, sum_of_squares);- let avg = extended_stats.avg.unwrap();- assert_eq!(5.166666666666667, avg);- }-}diff --git a/crates/tantivy/src/aggregation/metric/max.rs b/crates/tantivy/src/aggregation/metric/max.rsdeleted file mode 100644index 89c6e445..00000000--- a/crates/tantivy/src/aggregation/metric/max.rs+++ /dev/null@@ -1,120 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::*;--/// A single-value metric aggregation that computes the maximum of numeric values that are-/// extracted from the aggregated documents.-/// See [super::SingleMetricResult] for return value.-///-/// # JSON Format-/// ```json-/// {-/// "max": {-/// "field": "score"-/// }-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct MaxAggregation {- /// The field name to compute the maximum on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl MaxAggregation {- /// Creates a new [`MaxAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- Self {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Intermediate result of the maximum aggregation that can be combined with other intermediate-/// results.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateMax {- stats: IntermediateStats,-}--impl IntermediateMax {- /// Creates a new [`IntermediateMax`] instance from a [`SegmentStatsCollector`].- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {- Self {- stats: collector.stats,- }- }- /// Merges the other intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateMax) {- self.stats.merge_fruits(other.stats);- }- /// Computes the final maximum value.- pub fn finalize(&self) -> Option<f64> {- self.stats.finalize().max- }-}--#[cfg(test)]-mod tests {- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::tests::exec_request_with_query;- use crate::schema::{Schema, FAST};- use crate::{Index, IndexWriter};-- #[test]- fn test_max_agg_with_missing() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with empty json- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();- // => Segment with json, but no field partially_empty- index_writer- .add_document(doc!(json => json!({"different_field": "blue"})))- .unwrap();- index_writer.commit().unwrap();- //// => Segment with field partially_empty- index_writer- .add_document(doc!(json => json!({"partially_empty": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_stats": {- "max": {- "field": "json.partially_empty",- "missing": 100.0,- }- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res["my_stats"],- json!({- "value": 100.0,- })- );-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/metric/min.rs b/crates/tantivy/src/aggregation/metric/min.rsdeleted file mode 100644index 61fd2ecd..00000000--- a/crates/tantivy/src/aggregation/metric/min.rs+++ /dev/null@@ -1,68 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::*;--/// A single-value metric aggregation that computes the minimum of numeric values that are-/// extracted from the aggregated documents.-/// See [super::SingleMetricResult] for return value.-///-/// # JSON Format-/// ```json-/// {-/// "min": {-/// "field": "score"-/// }-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct MinAggregation {- /// The field name to compute the minimum on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl MinAggregation {- /// Creates a new [`MinAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- Self {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Intermediate result of the minimum aggregation that can be combined with other intermediate-/// results.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateMin {- stats: IntermediateStats,-}--impl IntermediateMin {- /// Creates a new [`IntermediateMin`] instance from a [`SegmentStatsCollector`].- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {- Self {- stats: collector.stats,- }- }- /// Merges the other intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateMin) {- self.stats.merge_fruits(other.stats);- }- /// Computes the final minimum value.- pub fn finalize(&self) -> Option<f64> {- self.stats.finalize().min- }-}diff --git a/crates/tantivy/src/aggregation/metric/mod.rs b/crates/tantivy/src/aggregation/metric/mod.rsdeleted file mode 100644index 4c64737b..00000000--- a/crates/tantivy/src/aggregation/metric/mod.rs+++ /dev/null@@ -1,171 +0,0 @@-//! Module for all metric aggregations.-//!-//! The aggregations in this family compute metrics based on values extracted-//! from the documents that are being aggregated. Values are extracted from the fast field of-//! the document.-//! Some aggregations output a single numeric metric (e.g. Average) and are called-//! single-value numeric metrics aggregation, others generate multiple metrics (e.g. Stats) and are-//! called multi-value numeric metrics aggregation.-//!-//! ## Supported Metric Aggregations-//! - [Average](AverageAggregation)-//! - [Stats](StatsAggregation)-//! - [Min](MinAggregation)-//! - [Max](MaxAggregation)-//! - [Sum](SumAggregation)-//! - [Count](CountAggregation)-//! - [Percentiles](PercentilesAggregationReq)--mod average;-mod count;-mod extended_stats;-mod max;-mod min;-mod percentiles;-mod stats;-mod sum;-mod top_hits;--use std::collections::HashMap;--pub use average::*;-pub use count::*;-pub use extended_stats::*;-pub use max::*;-pub use min::*;-pub use percentiles::*;-use rustc_hash::FxHashMap;-use serde::{Deserialize, Serialize};-pub use stats::*;-pub use sum::*;-pub use top_hits::*;--use crate::schema::OwnedValue;--/// Single-metric aggregations use this common result structure.-///-/// Main reason to wrap it in value is to match elasticsearch output structure.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct SingleMetricResult {- /// The value of the single value metric.- pub value: Option<f64>,-}--impl From<f64> for SingleMetricResult {- fn from(value: f64) -> Self {- Self { value: Some(value) }- }-}--impl From<Option<f64>> for SingleMetricResult {- fn from(value: Option<f64>) -> Self {- Self { value }- }-}--/// This is the wrapper of percentile entries, which can be vector or hashmap-/// depending on if it's keyed or not.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-#[serde(untagged)]-pub enum PercentileValues {- /// Vector format percentile entries- Vec(Vec<PercentileValuesVecEntry>),- /// HashMap format percentile entries. Key is the serialized percentile- HashMap(FxHashMap<String, f64>),-}--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-/// The entry when requesting percentiles with keyed: false-pub struct PercentileValuesVecEntry {- key: f64,- value: f64,-}--/// Single-metric aggregations use this common result structure.-///-/// Main reason to wrap it in value is to match elasticsearch output structure.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct PercentilesMetricResult {- /// The result of the percentile metric.- pub values: PercentileValues,-}--/// The top_hits metric results entry-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct TopHitsVecEntry {- /// The sort values of the document, depending on the sort criteria in the request.- pub sort: Vec<Option<u64>>,-- /// Search results, for queries that include field retrieval requests- /// (`docvalue_fields`).- #[serde(rename = "docvalue_fields")]- #[serde(skip_serializing_if = "HashMap::is_empty")]- pub doc_value_fields: HashMap<String, OwnedValue>,-}--/// The top_hits metric aggregation results a list of top hits by sort criteria.-///-/// The main reason for wrapping it in `hits` is to match elasticsearch output structure.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct TopHitsMetricResult {- /// The result of the top_hits metric.- pub hits: Vec<TopHitsVecEntry>,-}--#[cfg(test)]-mod tests {- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::AggregationCollector;- use crate::query::AllQuery;- use crate::schema::{NumericOptions, Schema};- use crate::{Index, IndexWriter};-- #[test]- fn test_metric_aggregations() {- let mut schema_builder = Schema::builder();- let field_options = NumericOptions::default().set_fast();- let field = schema_builder.add_f64_field("price", field_options);- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();-- for i in 0..3 {- index_writer- .add_document(doc!(- field => i as f64,- ))- .unwrap();- }- index_writer.commit().unwrap();-- for i in 3..6 {- index_writer- .add_document(doc!(- field => i as f64,- ))- .unwrap();- }- index_writer.commit().unwrap();-- let aggregations_json = r#"{- "price_avg": { "avg": { "field": "price" } },- "price_count": { "value_count": { "field": "price" } },- "price_max": { "max": { "field": "price" } },- "price_min": { "min": { "field": "price" } },- "price_stats": { "stats": { "field": "price" } },- "price_sum": { "sum": { "field": "price" } }- }"#;- let aggregations: Aggregations = serde_json::from_str(aggregations_json).unwrap();- let collector = AggregationCollector::from_aggs(aggregations, Default::default());- let reader = index.reader().unwrap();- let searcher = reader.searcher();- let aggregations_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let aggregations_res_json = serde_json::to_value(aggregations_res).unwrap();-- assert_eq!(aggregations_res_json["price_avg"]["value"], 2.5);- assert_eq!(aggregations_res_json["price_count"]["value"], 6.0);- assert_eq!(aggregations_res_json["price_max"]["value"], 5.0);- assert_eq!(aggregations_res_json["price_min"]["value"], 0.0);- assert_eq!(aggregations_res_json["price_sum"]["value"], 15.0);- }-}diff --git a/crates/tantivy/src/aggregation/metric/percentiles.rs b/crates/tantivy/src/aggregation/metric/percentiles.rsdeleted file mode 100644index 4a6bac3f..00000000--- a/crates/tantivy/src/aggregation/metric/percentiles.rs+++ /dev/null@@ -1,691 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::agg_req_with_accessor::{- AggregationWithAccessor, AggregationsWithAccessor,-};-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,-};-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;-use crate::aggregation::*;-use crate::{DocId, TantivyError};--/// # Percentiles-///-/// The percentiles aggregation is a useful tool for understanding the distribution-/// of a data set. It calculates the values below which a given percentage of the-/// data falls. For instance, the 95th percentile indicates the value below which-/// 95% of the data points can be found.-///-/// This aggregation can be particularly interesting for analyzing website or service response-/// times. For example, if the 95th percentile website load time is significantly higher than the-/// median, this indicates that a small percentage of users are experiencing much slower load times-/// than the majority.-///-/// To use the percentiles aggregation, you'll need to provide a field to-/// aggregate on. In the case of website load times, this would typically be a-/// field containing the duration of time it takes for the site to load.-///-/// The following example demonstrates a request for the percentiles of the "load_time"-/// field:-///-/// ```JSON-/// {-/// "percentiles": {-/// "field": "load_time"-/// }-/// }-/// ```-///-/// This request will return an object containing the default percentiles (1, 5,-/// 25, 50 (median), 75, 95, and 99). You can also customize the percentiles you want to-/// calculate by providing an array of values in the "percents" parameter:-///-/// ```JSON-/// {-/// "percentiles": {-/// "field": "load_time",-/// "percents": [10, 20, 30, 40, 50, 60, 70, 80, 90]-/// }-/// }-/// ```-///-/// In this example, the aggregation will return the 10th, 20th, 30th, 40th, 50th,-/// 60th, 70th, 80th, and 90th percentiles of the "load_time" field.-///-/// Analyzing the percentiles of website load times can help you understand the-/// user experience and identify areas for optimization. For example, if the 95th-/// percentile load time is significantly higher than the median, this indicates-/// that a small percentage of users are experiencing much slower load times than-/// the majority.-///-/// # Estimating Percentiles-///-/// While percentiles provide valuable insights into the distribution of data, it's-/// important to understand that they are often estimates. This is because-/// calculating exact percentiles for large data sets can be computationally-/// expensive and time-consuming. As a result, many percentile aggregation-/// algorithms use approximation techniques to provide faster results.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct PercentilesAggregationReq {- /// The field name to compute the percentiles on.- pub field: String,- /// The percentiles to compute.- /// Defaults to [1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0]- pub percents: Option<Vec<f64>>,- /// Whether to return the percentiles as a hash map- #[serde(default = "default_as_true")]- pub keyed: bool,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(- skip_serializing_if = "Option::is_none",- default,- deserialize_with = "deserialize_option_f64"- )]- pub missing: Option<f64>,-}-fn default_percentiles() -> &'static [f64] {- &[1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0]-}-fn default_as_true() -> bool {- true-}--impl PercentilesAggregationReq {- /// Creates a new [`PercentilesAggregationReq`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- PercentilesAggregationReq {- field: field_name,- percents: None,- keyed: default_as_true(),- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-- fn validate(&self) -> crate::Result<()> {- if let Some(percents) = self.percents.as_ref() {- let all_in_range = percents- .iter()- .cloned()- .all(|percent| (0.0..=100.0).contains(&percent));- if !all_in_range {- return Err(TantivyError::AggregationError(- AggregationError::InvalidRequest(- "All percentiles have to be between 0.0 and 100.0".to_string(),- ),- ));- }- }-- Ok(())- }-}--#[derive(Clone, Debug, PartialEq)]-pub(crate) struct SegmentPercentilesCollector {- field_type: ColumnType,- pub(crate) percentiles: PercentilesCollector,- pub(crate) accessor_idx: usize,- missing: Option<u64>,-}--#[derive(Clone, Serialize, Deserialize)]-/// The percentiles collector used during segment collection and for merging results.-pub struct PercentilesCollector {- sketch: sketches_ddsketch::DDSketch,-}-impl Default for PercentilesCollector {- fn default() -> Self {- Self::new()- }-}--impl Debug for PercentilesCollector {- fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {- f.debug_struct("IntermediatePercentiles")- .field("sketch_len", &self.sketch.length())- .finish()- }-}-impl PartialEq for PercentilesCollector {- fn eq(&self, _other: &Self) -> bool {- false- }-}--fn format_percentil(percentil: f64) -> String {- let mut out = percentil.to_string();- // Slightly silly way to format trailing decimals- if !out.contains('.') {- out.push_str(".0");- }- out-}--impl PercentilesCollector {- /// Convert result into final result. This will query the quantils from the underlying quantil- /// collector.- pub fn into_final_result(self, req: &PercentilesAggregationReq) -> PercentilesMetricResult {- let percentiles: &[f64] = req- .percents- .as_ref()- .map(|el| el.as_ref())- .unwrap_or(default_percentiles());- let iter_quantile_and_values = percentiles.iter().cloned().map(|percentile| {- (- percentile,- self.sketch- .quantile(percentile / 100.0)- .expect(- "quantil out of range. This error should have been caught during \- validation phase",- )- .unwrap_or(f64::NAN),- )- });-- let values = if req.keyed {- PercentileValues::HashMap(- iter_quantile_and_values- .map(|(val, quantil)| (format_percentil(val), quantil))- .collect(),- )- } else {- PercentileValues::Vec(- iter_quantile_and_values- .map(|(key, value)| PercentileValuesVecEntry { key, value })- .collect(),- )- };- PercentilesMetricResult { values }- }-- fn new() -> Self {- let ddsketch_config = sketches_ddsketch::Config::defaults();- let sketch = sketches_ddsketch::DDSketch::new(ddsketch_config);- Self { sketch }- }- fn collect(&mut self, val: f64) {- self.sketch.add(val);- }-- pub(crate) fn merge_fruits(&mut self, right: PercentilesCollector) -> crate::Result<()> {- self.sketch.merge(&right.sketch).map_err(|err| {- TantivyError::AggregationError(AggregationError::InternalError(format!(- "Error while merging percentiles {err:?}"- )))- })?;-- Ok(())- }-}--impl SegmentPercentilesCollector {- pub fn from_req_and_validate(- req: &PercentilesAggregationReq,- field_type: ColumnType,- accessor_idx: usize,- ) -> crate::Result<Self> {- req.validate()?;- let missing = req- .missing- .and_then(|val| f64_to_fastfield_u64(val, &field_type));-- Ok(Self {- field_type,- percentiles: PercentilesCollector::new(),- accessor_idx,- missing,- })- }- #[inline]- pub(crate) fn collect_block_with_field(- &mut self,- docs: &[DocId],- agg_accessor: &mut AggregationWithAccessor,- ) {- if let Some(missing) = self.missing.as_ref() {- agg_accessor.column_block_accessor.fetch_block_with_missing(- docs,- &agg_accessor.accessor,- *missing,- );- } else {- agg_accessor- .column_block_accessor- .fetch_block(docs, &agg_accessor.accessor);- }-- for val in agg_accessor.column_block_accessor.iter_vals() {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.percentiles.collect(val1);- }- }-}--impl SegmentAggregationCollector for SegmentPercentilesCollector {- #[inline]- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();- let intermediate_metric_result = IntermediateMetricResult::Percentiles(self.percentiles);-- results.push(- name,- IntermediateAggregationResult::Metric(intermediate_metric_result),- )?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;-- if let Some(missing) = self.missing {- let mut has_val = false;- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.percentiles.collect(val1);- has_val = true;- }- if !has_val {- self.percentiles- .collect(f64_from_fastfield_u64(missing, &self.field_type));- }- } else {- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.percentiles.collect(val1);- }- }-- Ok(())- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];- self.collect_block_with_field(docs, field);- Ok(())- }-}--#[cfg(test)]-mod tests {-- use itertools::Itertools;- use more_asserts::{assert_ge, assert_le};- use rand::rngs::StdRng;- use rand::SeedableRng;- use serde_json::Value;-- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::tests::{- exec_request_with_query, get_test_index_from_values, get_test_index_from_values_and_terms,- };- use crate::aggregation::AggregationCollector;- use crate::query::AllQuery;- use crate::schema::{Schema, FAST};- use crate::Index;-- #[test]- fn test_aggregation_percentiles_empty_index() -> crate::Result<()> {- // test index without segments- let values = vec![];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "percentiles": {- "percentiles": {- "field": "score",- }- },- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- assert_eq!(- res["percentiles"]["values"],- json!({- "1.0": Value::Null,- "5.0": Value::Null,- "25.0": Value::Null,- "50.0": Value::Null,- "75.0": Value::Null,- "95.0": Value::Null,- "99.0": Value::Null,- })- );-- Ok(())- }-- #[test]- fn test_aggregation_percentile_simple() -> crate::Result<()> {- let values = vec![10.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "percentiles": {- "percentiles": {- "field": "score",- }- },- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;-- let percents = vec!["1.0", "5.0", "25.0", "50.0", "75.0", "95.0", "99.0"];- let range = 9.9..10.1;- for percent in percents {- let val = res["percentiles"]["values"][percent].as_f64().unwrap();- assert!(range.contains(&val));- }-- Ok(())- }-- #[test]- fn test_aggregation_percentile_parameters() -> crate::Result<()> {- let values = vec![10.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_str = r#"- {- "mypercentiles": {- "percentiles": {- "field": "score",- "percents": [ 95, 99, 99.9 ]- }- }- } "#;- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;-- let percents = vec!["95.0", "99.0", "99.9"];- let expected_range = 9.9..10.1;- for percent in percents {- let val = res["mypercentiles"]["values"][percent].as_f64().unwrap();- assert!(expected_range.contains(&val));- }- // Keyed false- //- let agg_req_str = r#"- {- "mypercentiles": {- "percentiles": {- "field": "score",- "percents": [ 95, 99, 99.9 ],- "keyed": false- }- }- } "#;- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;-- let vals = &res["mypercentiles"]["values"];- assert_eq!(vals[0]["key"].as_f64().unwrap(), 95.0);- assert_eq!(vals[1]["key"].as_f64().unwrap(), 99.0);- assert_eq!(vals[2]["key"].as_f64().unwrap(), 99.9);- assert_eq!(vals[3]["key"], serde_json::Value::Null);- assert!(expected_range.contains(&vals[0]["value"].as_f64().unwrap()));- assert!(expected_range.contains(&vals[1]["value"].as_f64().unwrap()));- assert!(expected_range.contains(&vals[2]["value"].as_f64().unwrap()));-- Ok(())- }-- #[test]- fn test_aggregation_percentiles_single_seg() -> crate::Result<()> {- test_aggregation_percentiles(true)- }-- #[test]- fn test_aggregation_percentiles_multi_seg() -> crate::Result<()> {- test_aggregation_percentiles(false)- }-- fn test_aggregation_percentiles(merge_segments: bool) -> crate::Result<()> {- use rand_distr::Distribution;- let num_values_in_segment = [100, 30_000, 8000];- let lg_norm = rand_distr::LogNormal::new(2.996f64, 0.979f64).unwrap();- let mut rng = StdRng::from_seed([1u8; 32]);-- let segment_data = |i| {- (0..num_values_in_segment[i])- .map(|_| lg_norm.sample(&mut rng))- .collect_vec()- };-- let values = (0..=2).map(segment_data).collect_vec();-- let mut all_values = values- .iter()- .flat_map(|el| el.iter().cloned())- .collect_vec();- all_values.sort_unstable_by(|a, b| a.total_cmp(b));-- fn get_exact_quantil(q: f64, all_values: &[f64]) -> f64 {- let q = q / 100.0;- assert!((0f64..=1f64).contains(&q));-- let index = (all_values.len() as f64 * q).ceil() as usize;- let index = index.min(all_values.len() - 1);- all_values[index]- }-- let segment_and_values = values- .into_iter()- .map(|segment_data| {- segment_data- .into_iter()- .map(|val| (val, val.to_string()))- .collect_vec()- })- .collect_vec();-- let index =- get_test_index_from_values_and_terms(merge_segments, &segment_and_values).unwrap();-- let reader = index.reader()?;-- let agg_req_str = r#"- {- "mypercentiles": {- "percentiles": {- "field": "score_f64",- "percents": [ 95, 99, 99.9 ]- }- }- } "#;- let agg_req_1: Aggregations = serde_json::from_str(agg_req_str).unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- let vals = &res["mypercentiles"]["values"];-- let check_quantil = |exact_quantil: f64, val: f64| {- let lower = exact_quantil - exact_quantil * 0.02;- let upper = exact_quantil + exact_quantil * 0.02;- assert_le!(val, upper);- assert_ge!(val, lower);- };-- let val = vals["95.0"].as_f64().unwrap();- let exact_quantil = get_exact_quantil(95.0, &all_values);- check_quantil(exact_quantil, val);-- let val = vals["99.0"].as_f64().unwrap();- let exact_quantil = get_exact_quantil(99.0, &all_values);- check_quantil(exact_quantil, val);-- let val = vals["99.9"].as_f64().unwrap();- let exact_quantil = get_exact_quantil(99.9, &all_values);- check_quantil(exact_quantil, val);-- Ok(())- }-- #[test]- fn test_percentiles_missing_sub_agg() -> crate::Result<()> {- // This test verifies the `collect` method (in contrast to `collect_block`), which is- // called when the sub-aggregations are flushed.- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("texts", FAST);- let score_field_f64 = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- let mut index_writer = index.writer_for_tests()?;- // writing the segment- index_writer.add_document(doc!(- score_field_f64 => 10.0f64,- text_field => "a"- ))?;- index_writer.add_document(doc!(- score_field_f64 => 10.0f64,- text_field => "a"- ))?;-- index_writer.add_document(doc!(text_field => "a"))?;-- index_writer.commit()?;- }-- let agg_req: Aggregations = {- serde_json::from_value(json!({- "range_with_stats": {- "terms": {- "field": "texts"- },- "aggs": {- "percentiles": {- "percentiles": {- "field": "score",- "missing": 5.0- }- }- }- }- }))- .unwrap()- };-- let res = exec_request_with_query(agg_req, &index, None)?;- assert_eq!(res["range_with_stats"]["buckets"][0]["doc_count"], 3);-- assert_eq!(- res["range_with_stats"]["buckets"][0]["percentiles"]["values"]["1.0"],- 5.0028295751107414- );- assert_eq!(- res["range_with_stats"]["buckets"][0]["percentiles"]["values"]["99.0"],- 10.07469668951144- );-- Ok(())- }-- #[test]- fn test_percentiles_missing() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("texts", FAST);- let score_field_f64 = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- let mut index_writer = index.writer_for_tests()?;- // writing the segment- index_writer.add_document(doc!(- score_field_f64 => 10.0f64,- text_field => "a"- ))?;- index_writer.add_document(doc!(- score_field_f64 => 10.0f64,- text_field => "a"- ))?;-- index_writer.add_document(doc!(text_field => "a"))?;-- index_writer.commit()?;- }-- let agg_req: Aggregations = {- serde_json::from_value(json!({- "percentiles": {- "percentiles": {- "field": "score",- "missing": 5.0- }- }- }))- .unwrap()- };-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(res["percentiles"]["values"]["1.0"], 5.0028295751107414);- assert_eq!(res["percentiles"]["values"]["99.0"], 10.07469668951144);-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/metric/stats.rs b/crates/tantivy/src/aggregation/metric/stats.rsdeleted file mode 100644index 8b8be399..00000000--- a/crates/tantivy/src/aggregation/metric/stats.rs+++ /dev/null@@ -1,684 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::agg_req_with_accessor::{- AggregationWithAccessor, AggregationsWithAccessor,-};-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateAggregationResults, IntermediateMetricResult,-};-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;-use crate::aggregation::*;-use crate::{DocId, TantivyError};--/// A multi-value metric aggregation that computes a collection of statistics on numeric values that-/// are extracted from the aggregated documents.-/// See [`Stats`] for returned statistics.-///-/// # JSON Format-/// ```json-/// {-/// "stats": {-/// "field": "score"-/// }-/// }-/// ```--#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct StatsAggregation {- /// The field name to compute the stats on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl StatsAggregation {- /// Creates a new [`StatsAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- StatsAggregation {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Stats contains a collection of statistics.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct Stats {- /// The number of documents.- pub count: u64,- /// The sum of the fast field values.- pub sum: f64,- /// The min value of the fast field values.- pub min: Option<f64>,- /// The max value of the fast field values.- pub max: Option<f64>,- /// The average of the fast field values. `None` if count equals zero.- pub avg: Option<f64>,-}--impl Stats {- pub(crate) fn get_value(&self, agg_property: &str) -> crate::Result<Option<f64>> {- match agg_property {- "count" => Ok(Some(self.count as f64)),- "sum" => Ok(Some(self.sum)),- "min" => Ok(self.min),- "max" => Ok(self.max),- "avg" => Ok(self.avg),- _ => Err(TantivyError::InvalidArgument(format!(- "Unknown property {agg_property} on stats metric aggregation"- ))),- }- }-}--/// Intermediate result of the stats aggregation that can be combined with other intermediate-/// results.-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateStats {- /// The number of extracted values.- pub(crate) count: u64,- /// The sum of the extracted values.- pub(crate) sum: f64,- /// delta for sum needed for [Kahan algorithm for summation](https://en.wikipedia.org/wiki/Kahan_summation_algorithm)- pub(crate) delta: f64,- /// The min value.- pub(crate) min: f64,- /// The max value.- pub(crate) max: f64,-}--impl Default for IntermediateStats {- fn default() -> Self {- Self {- count: 0,- sum: 0.0,- delta: 0.0,- min: f64::MAX,- max: f64::MIN,- }- }-}--impl IntermediateStats {- /// Merges the other stats intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateStats) {- self.count += other.count;-- // kahan algorithm for sum- let y = other.sum - (self.delta + other.delta);- let t = self.sum + y;- self.delta = (t - self.sum) - y;- self.sum = t;-- self.min = self.min.min(other.min);- self.max = self.max.max(other.max);- }-- /// Computes the final stats value.- pub fn finalize(&self) -> Stats {- let min = if self.count == 0 {- None- } else {- Some(self.min)- };- let max = if self.count == 0 {- None- } else {- Some(self.max)- };- let avg = if self.count == 0 {- None- } else {- Some(self.sum / (self.count as f64))- };- Stats {- count: self.count,- sum: self.sum,- min,- max,- avg,- }- }-- #[inline]- pub(in crate::aggregation::metric) fn collect(&mut self, value: f64) {- self.count += 1;-- // kahan algorithm for sum- let y = value - self.delta;- let t = self.sum + y;- self.delta = (t - self.sum) - y;- self.sum = t;-- self.min = self.min.min(value);- self.max = self.max.max(value);- }-}--#[derive(Clone, Debug, PartialEq)]-pub(crate) enum SegmentStatsType {- Average,- Count,- Max,- Min,- Stats,- Sum,-}--#[derive(Clone, Debug, PartialEq)]-pub(crate) struct SegmentStatsCollector {- missing: Option<u64>,- field_type: ColumnType,- pub(crate) collecting_for: SegmentStatsType,- pub(crate) stats: IntermediateStats,- pub(crate) accessor_idx: usize,- val_cache: Vec<u64>,-}--impl SegmentStatsCollector {- pub fn from_req(- field_type: ColumnType,- collecting_for: SegmentStatsType,- accessor_idx: usize,- missing: Option<f64>,- ) -> Self {- let missing = missing.and_then(|val| f64_to_fastfield_u64(val, &field_type));- Self {- field_type,- collecting_for,- stats: IntermediateStats::default(),- accessor_idx,- missing,- val_cache: Default::default(),- }- }- #[inline]- pub(crate) fn collect_block_with_field(- &mut self,- docs: &[DocId],- agg_accessor: &mut AggregationWithAccessor,- ) {- if let Some(missing) = self.missing.as_ref() {- agg_accessor.column_block_accessor.fetch_block_with_missing(- docs,- &agg_accessor.accessor,- *missing,- );- } else {- agg_accessor- .column_block_accessor- .fetch_block(docs, &agg_accessor.accessor);- }- for val in agg_accessor.column_block_accessor.iter_vals() {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.stats.collect(val1);- }- }-}--impl SegmentAggregationCollector for SegmentStatsCollector {- #[inline]- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();-- let intermediate_metric_result = match self.collecting_for {- SegmentStatsType::Average => {- IntermediateMetricResult::Average(IntermediateAverage::from_collector(*self))- }- SegmentStatsType::Count => {- IntermediateMetricResult::Count(IntermediateCount::from_collector(*self))- }- SegmentStatsType::Max => {- IntermediateMetricResult::Max(IntermediateMax::from_collector(*self))- }- SegmentStatsType::Min => {- IntermediateMetricResult::Min(IntermediateMin::from_collector(*self))- }- SegmentStatsType::Stats => IntermediateMetricResult::Stats(self.stats),- SegmentStatsType::Sum => {- IntermediateMetricResult::Sum(IntermediateSum::from_collector(*self))- }- };-- results.push(- name,- IntermediateAggregationResult::Metric(intermediate_metric_result),- )?;-- Ok(())- }-- #[inline]- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &agg_with_accessor.aggs.values[self.accessor_idx].accessor;- if let Some(missing) = self.missing {- let mut has_val = false;- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.stats.collect(val1);- has_val = true;- }- if !has_val {- self.stats- .collect(f64_from_fastfield_u64(missing, &self.field_type));- }- } else {- for val in field.values_for_doc(doc) {- let val1 = f64_from_fastfield_u64(val, &self.field_type);- self.stats.collect(val1);- }- }-- Ok(())- }-- #[inline]- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- let field = &mut agg_with_accessor.aggs.values[self.accessor_idx];- self.collect_block_with_field(docs, field);- Ok(())- }-}--#[cfg(test)]-mod tests {- use serde_json::Value;-- use crate::aggregation::agg_req::{Aggregation, Aggregations};- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::tests::{- exec_request_with_query, get_test_index_2_segments, get_test_index_from_values,- };- use crate::aggregation::AggregationCollector;- use crate::query::{AllQuery, TermQuery};- use crate::schema::{IndexRecordOption, Schema, FAST};- use crate::{Index, IndexWriter, Term};-- #[test]- fn test_aggregation_stats_empty_index() -> crate::Result<()> {- // test index without segments- let values = vec![];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "stats": {- "stats": {- "field": "score",- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- assert_eq!(- res["stats"],- json!({- "avg": Value::Null,- "count": 0,- "max": Value::Null,- "min": Value::Null,- "sum": 0.0- })- );-- Ok(())- }-- #[test]- fn test_aggregation_stats_simple() -> crate::Result<()> {- let values = vec![10.0];-- let index = get_test_index_from_values(false, &values)?;-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "stats": {- "stats": {- "field": "score",- },- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- assert_eq!(- res["stats"],- json!({- "avg": 10.0,- "count": 1,- "max": 10.0,- "min": 10.0,- "sum": 10.0- })- );-- Ok(())- }-- #[test]- fn test_aggregation_stats() -> crate::Result<()> {- let index = get_test_index_2_segments(false)?;-- let reader = index.reader()?;- let text_field = reader.searcher().schema().get_field("text").unwrap();-- let term_query = TermQuery::new(- Term::from_field_text(text_field, "cool"),- IndexRecordOption::Basic,- );-- let range_agg: Aggregation = {- serde_json::from_value(json!({- "range": {- "field": "score",- "ranges": [ { "from": 3.0f64, "to": 7.0f64 }, { "from": 7.0f64, "to": 19.0f64 }, { "from": 19.0f64, "to": 20.0f64 } ]- },- "aggs": {- "stats": {- "stats": {- "field": "score"- }- }- }- }))- .unwrap()- };-- let agg_req_1: Aggregations = serde_json::from_value(json!({- "stats_i64": {- "stats": {- "field": "score_i64",- },- },- "stats_f64": {- "stats": {- "field": "score_f64",- },- },- "stats": {- "stats": {- "field": "score",- },- },- "range": range_agg- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(agg_req_1, Default::default());-- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&term_query, &collector).unwrap();-- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- assert_eq!(- res["stats"],- json!({- "avg": 12.142857142857142,- "count": 7,- "max": 44.0,- "min": 1.0,- "sum": 85.0- })- );-- assert_eq!(- res["stats_i64"],- json!({- "avg": 12.142857142857142,- "count": 7,- "max": 44.0,- "min": 1.0,- "sum": 85.0- })- );-- assert_eq!(- res["stats_f64"],- json!({- "avg": 12.214285714285714,- "count": 7,- "max": 44.5,- "min": 1.0,- "sum": 85.5- })- );-- assert_eq!(- res["range"]["buckets"][2]["stats"],- json!({- "avg": 10.666666666666666,- "count": 3,- "max": 14.0,- "min": 7.0,- "sum": 32.0- })- );-- assert_eq!(- res["range"]["buckets"][3]["stats"],- json!({- "avg": serde_json::Value::Null,- "count": 0,- "max": serde_json::Value::Null,- "min": serde_json::Value::Null,- "sum": 0.0,- })- );-- Ok(())- }-- #[test]- fn test_stats_json() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with empty json- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();- // => Segment with json, but no field partially_empty- index_writer- .add_document(doc!(json => json!({"different_field": "blue"})))- .unwrap();- index_writer.commit().unwrap();- //// => Segment with field partially_empty- index_writer- .add_document(doc!(json => json!({"partially_empty": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_stats": {- "stats": {- "field": "json.partially_empty"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res["my_stats"],- json!({- "avg": 10.0,- "count": 1,- "max": 10.0,- "min": 10.0,- "sum": 10.0- })- );-- Ok(())- }-- #[test]- fn test_stats_json_missing() -> crate::Result<()> {- let mut schema_builder = Schema::builder();- let json = schema_builder.add_json_field("json", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);- let mut index_writer: IndexWriter = index.writer_for_tests().unwrap();- // => Segment with empty json- index_writer.add_document(doc!()).unwrap();- index_writer.commit().unwrap();- // => Segment with json, but no field partially_empty- index_writer- .add_document(doc!(json => json!({"different_field": "blue"})))- .unwrap();- index_writer.commit().unwrap();- //// => Segment with field partially_empty- index_writer- .add_document(doc!(json => json!({"partially_empty": 10.0})))- .unwrap();- index_writer.add_document(doc!())?;- index_writer.commit().unwrap();-- let agg_req: Aggregations = serde_json::from_value(json!({- "my_stats": {- "stats": {- "field": "json.partially_empty",- "missing": 0.0- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res["my_stats"],- json!({- "avg": 2.5,- "count": 4,- "max": 10.0,- "min": 0.0,- "sum": 10.0- })- );-- // From string- let agg_req: Aggregations = serde_json::from_value(json!({- "my_stats": {- "stats": {- "field": "json.partially_empty",- "missing": "0.0"- },- }- }))- .unwrap();-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res["my_stats"],- json!({- "avg": 2.5,- "count": 4,- "max": 10.0,- "min": 0.0,- "sum": 10.0- })- );-- Ok(())- }-- #[test]- fn test_stats_json_missing_sub_agg() -> crate::Result<()> {- // This test verifies the `collect` method (in contrast to `collect_block`), which is- // called when the sub-aggregations are flushed.- let mut schema_builder = Schema::builder();- let text_field = schema_builder.add_text_field("texts", FAST);- let score_field_f64 = schema_builder.add_f64_field("score", FAST);- let schema = schema_builder.build();- let index = Index::create_in_ram(schema);-- {- let mut index_writer = index.writer_for_tests()?;- // writing the segment- index_writer.add_document(doc!(- score_field_f64 => 10.0f64,- text_field => "a"- ))?;-- index_writer.add_document(doc!(text_field => "a"))?;-- index_writer.commit()?;- }-- let agg_req: Aggregations = {- serde_json::from_value(json!({- "range_with_stats": {- "terms": {- "field": "texts"- },- "aggs": {- "my_stats": {- "stats": {- "field": "score",- "missing": 0.0- }- }- }- }- }))- .unwrap()- };-- let res = exec_request_with_query(agg_req, &index, None)?;-- assert_eq!(- res["range_with_stats"]["buckets"][0]["my_stats"]["count"],- 2- );- assert_eq!(- res["range_with_stats"]["buckets"][0]["my_stats"]["min"],- 0.0- );- assert_eq!(- res["range_with_stats"]["buckets"][0]["my_stats"]["avg"],- 5.0- );-- Ok(())- }-}diff --git a/crates/tantivy/src/aggregation/metric/sum.rs b/crates/tantivy/src/aggregation/metric/sum.rsdeleted file mode 100644index 86f66167..00000000--- a/crates/tantivy/src/aggregation/metric/sum.rs+++ /dev/null@@ -1,68 +0,0 @@-use std::fmt::Debug;--use serde::{Deserialize, Serialize};--use super::*;-use crate::aggregation::*;--/// A single-value metric aggregation that sums up numeric values that are-/// extracted from the aggregated documents.-/// See [super::SingleMetricResult] for return value.-///-/// # JSON Format-/// ```json-/// {-/// "sum": {-/// "field": "score"-/// }-/// }-/// ```-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct SumAggregation {- /// The field name to compute the minimum on.- pub field: String,- /// The missing parameter defines how documents that are missing a value should be treated.- /// By default they will be ignored but it is also possible to treat them as if they had a- /// value. Examples in JSON format:- /// { "field": "my_numbers", "missing": "10.0" }- #[serde(default, deserialize_with = "deserialize_option_f64")]- pub missing: Option<f64>,-}--impl SumAggregation {- /// Creates a new [`SumAggregation`] instance from a field name.- pub fn from_field_name(field_name: String) -> Self {- Self {- field: field_name,- missing: None,- }- }- /// Returns the field name the aggregation is computed on.- pub fn field_name(&self) -> &str {- &self.field- }-}--/// Intermediate result of the minimum aggregation that can be combined with other intermediate-/// results.-#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize)]-pub struct IntermediateSum {- stats: IntermediateStats,-}--impl IntermediateSum {- /// Creates a new [`IntermediateSum`] instance from a [`SegmentStatsCollector`].- pub(crate) fn from_collector(collector: SegmentStatsCollector) -> Self {- Self {- stats: collector.stats,- }- }- /// Merges the other intermediate result into self.- pub fn merge_fruits(&mut self, other: IntermediateSum) {- self.stats.merge_fruits(other.stats);- }- /// Computes the final minimum value.- pub fn finalize(&self) -> Option<f64> {- Some(self.stats.finalize().sum)- }-}diff --git a/crates/tantivy/src/aggregation/metric/top_hits.rs b/crates/tantivy/src/aggregation/metric/top_hits.rsdeleted file mode 100644index 72c8e8ce..00000000--- a/crates/tantivy/src/aggregation/metric/top_hits.rs+++ /dev/null@@ -1,918 +0,0 @@-use std::collections::HashMap;-use std::net::Ipv6Addr;--use crate::columnar::{Column, ColumnType, ColumnarReader, DynamicColumn};-use crate::common::json_path_writer::JSON_PATH_SEGMENT_SEP_STR;-use crate::common::DateTime;-use regex::Regex;-use serde::ser::SerializeMap;-use serde::{Deserialize, Deserializer, Serialize, Serializer};--use super::{TopHitsMetricResult, TopHitsVecEntry};-use crate::aggregation::bucket::Order;-use crate::aggregation::intermediate_agg_result::{- IntermediateAggregationResult, IntermediateMetricResult,-};-use crate::aggregation::segment_agg_result::SegmentAggregationCollector;-use crate::aggregation::AggregationError;-use crate::collector::TopNComputer;-use crate::schema::OwnedValue;-use crate::{DocAddress, DocId, SegmentOrdinal};--/// # Top Hits-///-/// The top hits aggregation is a useful tool to answer questions like:-/// - "What are the most recent posts by each author?"-/// - "What are the most popular items in each category?"-///-/// It does so by keeping track of the most relevant document being aggregated,-/// in terms of a sort criterion that can consist of multiple fields and their-/// sort-orders (ascending or descending).-///-/// `top_hits` should not be used as a top-level aggregation. It is intended to be-/// used as a sub-aggregation, inside a `terms` aggregation or a `filters` aggregation,-/// for example.-///-/// Note that this aggregator does not return the actual document addresses, but-/// rather a list of the values of the fields that were requested to be retrieved.-/// These values can be specified in the `docvalue_fields` parameter, which can include-/// a list of fast fields to be retrieved. At the moment, only fast fields are supported-/// but it is possible that we support the `fields` parameter to retrieve any stored-/// field in the future.-///-/// The following example demonstrates a request for the top_hits aggregation:-/// ```JSON-/// {-/// "aggs": {-/// "top_authors": {-/// "terms": {-/// "field": "author",-/// "size": 5-/// }-/// },-/// "aggs": {-/// "top_hits": {-/// "size": 2,-/// "from": 0-/// "sort": [-/// { "date": "desc" }-/// ]-/// "docvalue_fields": ["date", "title", "iden"]-/// }-/// }-/// }-/// ```-///-/// This request will return an object containing the top two documents, sorted-/// by the `date` field in descending order. You can also sort by multiple fields, which-/// helps to resolve ties. The aggregation object for each bucket will look like:-/// ```JSON-/// {-/// "hits": [-/// {-/// "score": [<time_u64>],-/// "docvalue_fields": {-/// "date": "<date_RFC3339>",-/// "title": "<title>",-/// "iden": "<iden>"-/// }-/// },-/// {-/// "score": [<time_u64>]-/// "docvalue_fields": {-/// "date": "<date_RFC3339>",-/// "title": "<title>",-/// "iden": "<iden>"-/// }-/// }-/// ]-/// }-/// ```-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]-pub struct TopHitsAggregation {- sort: Vec<KeyOrder>,- size: usize,- from: Option<usize>,-- #[serde(rename = "docvalue_fields")]- #[serde(default)]- doc_value_fields: Vec<String>,-- // Not supported- _source: Option<serde_json::Value>,- fields: Option<serde_json::Value>,- script_fields: Option<serde_json::Value>,- highlight: Option<serde_json::Value>,- explain: Option<serde_json::Value>,- version: Option<serde_json::Value>,-}--#[derive(Debug, Clone, PartialEq, Default)]-struct KeyOrder {- field: String,- order: Order,-}--impl Serialize for KeyOrder {- fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {- let KeyOrder { field, order } = self;- let mut map = serializer.serialize_map(Some(1))?;- map.serialize_entry(field, order)?;- map.end()- }-}--impl<'de> Deserialize<'de> for KeyOrder {- fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>- where- D: Deserializer<'de>,- {- let mut key_order = <HashMap<String, Order>>::deserialize(deserializer)?.into_iter();- let (field, order) = key_order.next().ok_or(serde::de::Error::custom(- "Expected exactly one key-value pair in sort parameter of top_hits, found none",- ))?;- if key_order.next().is_some() {- return Err(serde::de::Error::custom(format!(- "Expected exactly one key-value pair in sort parameter of top_hits, found \- {key_order:?}"- )));- }- Ok(Self { field, order })- }-}--// Tranform a glob (`pattern*`, for example) into a regex::Regex (`^pattern.*$`)-fn globbed_string_to_regex(glob: &str) -> Result<Regex, crate::TantivyError> {- // Replace `*` glob with `.*` regex- let sanitized = format!("^{}$", regex::escape(glob).replace(r"\*", ".*"));- Regex::new(&sanitized.replace('*', ".*")).map_err(|e| {- crate::TantivyError::SchemaError(format!("Invalid regex '{glob}' in docvalue_fields: {e}"))- })-}--fn use_doc_value_fields_err(parameter: &str) -> crate::Result<()> {- Err(crate::TantivyError::AggregationError(- AggregationError::InvalidRequest(format!(- "The `{parameter}` parameter is not supported, only `docvalue_fields` is supported in \- `top_hits` aggregation"- )),- ))-}-fn unsupported_err(parameter: &str) -> crate::Result<()> {- Err(crate::TantivyError::AggregationError(- AggregationError::InvalidRequest(format!(- "The `{parameter}` parameter is not supported in the `top_hits` aggregation"- )),- ))-}--impl TopHitsAggregation {- /// Validate and resolve field retrieval parameters- pub fn validate_and_resolve_field_names(- &mut self,- reader: &ColumnarReader,- ) -> crate::Result<()> {- if self._source.is_some() {- use_doc_value_fields_err("_source")?;- }- if self.fields.is_some() {- use_doc_value_fields_err("fields")?;- }- if self.script_fields.is_some() {- use_doc_value_fields_err("script_fields")?;- }- if self.explain.is_some() {- unsupported_err("explain")?;- }- if self.highlight.is_some() {- unsupported_err("highlight")?;- }- if self.version.is_some() {- unsupported_err("version")?;- }-- self.doc_value_fields = self- .doc_value_fields- .iter()- .map(|field| {- if !field.contains('*')- && reader- .iter_columns()?- .any(|(name, _)| name.as_str() == field)- {- return Ok(vec![field.to_owned()]);- }-- let pattern = globbed_string_to_regex(field)?;- let fields = reader- .iter_columns()?- .map(|(name, _)| {- // normalize path from internal fast field repr- name.replace(JSON_PATH_SEGMENT_SEP_STR, ".")- })- .filter(|name| pattern.is_match(name))- .collect::<Vec<_>>();- assert!(- !fields.is_empty(),- "No fields matched the glob '{field}' in docvalue_fields"- );- Ok(fields)- })- .collect::<crate::Result<Vec<_>>>()?- .into_iter()- .flatten()- .collect();-- Ok(())- }-- /// Return fields accessed by the aggregator, in order.- pub fn field_names(&self) -> Vec<&str> {- self.sort- .iter()- .map(|KeyOrder { field, .. }| field.as_str())- .collect()- }-- /// Return fields accessed by the aggregator's value retrieval.- pub fn value_field_names(&self) -> Vec<&str> {- self.doc_value_fields.iter().map(|s| s.as_str()).collect()- }-- fn get_document_field_data(- &self,- accessors: &HashMap<String, Vec<DynamicColumn>>,- doc_id: DocId,- ) -> HashMap<String, FastFieldValue> {- let doc_value_fields = self- .doc_value_fields- .iter()- .map(|field| {- let accessors = accessors- .get(field)- .unwrap_or_else(|| panic!("field '{field}' not found in accessors"));-- let values: Vec<FastFieldValue> = accessors- .iter()- .flat_map(|accessor| match accessor {- DynamicColumn::U64(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::U64)- .collect::<Vec<_>>(),- DynamicColumn::I64(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::I64)- .collect::<Vec<_>>(),- DynamicColumn::F64(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::F64)- .collect::<Vec<_>>(),- DynamicColumn::Bytes(accessor) => accessor- .term_ords(doc_id)- .map(|term_ord| {- let mut buffer = vec![];- assert!(- accessor- .ord_to_bytes(term_ord, &mut buffer)- .expect("could not read term dictionary"),- "term corresponding to term_ord does not exist"- );- FastFieldValue::Bytes(buffer)- })- .collect::<Vec<_>>(),- DynamicColumn::Str(accessor) => accessor- .term_ords(doc_id)- .map(|term_ord| {- let mut buffer = vec![];- assert!(- accessor- .ord_to_bytes(term_ord, &mut buffer)- .expect("could not read term dictionary"),- "term corresponding to term_ord does not exist"- );- FastFieldValue::Str(String::from_utf8(buffer).unwrap())- })- .collect::<Vec<_>>(),- DynamicColumn::Bool(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::Bool)- .collect::<Vec<_>>(),- DynamicColumn::IpAddr(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::IpAddr)- .collect::<Vec<_>>(),- DynamicColumn::DateTime(accessor) => accessor- .values_for_doc(doc_id)- .map(FastFieldValue::Date)- .collect::<Vec<_>>(),- })- .collect();-- (field.to_owned(), FastFieldValue::Array(values))- })- .collect();- doc_value_fields- }-}--/// A retrieved value from a fast field.-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]-pub enum FastFieldValue {- /// The str type is used for any text information.- Str(String),- /// Unsigned 64-bits Integer `u64`- U64(u64),- /// Signed 64-bits Integer `i64`- I64(i64),- /// 64-bits Float `f64`- F64(f64),- /// Bool value- Bool(bool),- /// Date/time with nanoseconds precision- Date(DateTime),- /// Arbitrarily sized byte array- Bytes(Vec<u8>),- /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.- IpAddr(Ipv6Addr),- /// A list of values.- Array(Vec<Self>),-}--impl From<FastFieldValue> for OwnedValue {- fn from(value: FastFieldValue) -> Self {- match value {- FastFieldValue::Str(s) => OwnedValue::Str(s),- FastFieldValue::U64(u) => OwnedValue::U64(u),- FastFieldValue::I64(i) => OwnedValue::I64(i),- FastFieldValue::F64(f) => OwnedValue::F64(f),- FastFieldValue::Bool(b) => OwnedValue::Bool(b),- FastFieldValue::Date(d) => OwnedValue::Date(d),- FastFieldValue::Bytes(b) => OwnedValue::Bytes(b),- FastFieldValue::IpAddr(ip) => OwnedValue::IpAddr(ip),- FastFieldValue::Array(a) => {- OwnedValue::Array(a.into_iter().map(OwnedValue::from).collect())- }- }- }-}--/// Holds a fast field value in its u64 representation, and the order in which it should be sorted.-#[derive(Clone, Serialize, Deserialize, Debug)]-struct DocValueAndOrder {- /// A fast field value in its u64 representation.- value: Option<u64>,- /// Sort order for the value- order: Order,-}--impl Ord for DocValueAndOrder {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- let invert = |cmp: std::cmp::Ordering| match self.order {- Order::Asc => cmp,- Order::Desc => cmp.reverse(),- };-- match (self.value, other.value) {- (Some(self_value), Some(other_value)) => invert(self_value.cmp(&other_value)),- (Some(_), None) => std::cmp::Ordering::Greater,- (None, Some(_)) => std::cmp::Ordering::Less,- (None, None) => std::cmp::Ordering::Equal,- }- }-}--impl PartialOrd for DocValueAndOrder {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl PartialEq for DocValueAndOrder {- fn eq(&self, other: &Self) -> bool {- self.value.cmp(&other.value) == std::cmp::Ordering::Equal- }-}--impl Eq for DocValueAndOrder {}--#[derive(Clone, Serialize, Deserialize, Debug)]-struct DocSortValuesAndFields {- sorts: Vec<DocValueAndOrder>,-- #[serde(rename = "docvalue_fields")]- #[serde(skip_serializing_if = "HashMap::is_empty")]- doc_value_fields: HashMap<String, FastFieldValue>,-}--impl Ord for DocSortValuesAndFields {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- for (self_feature, other_feature) in self.sorts.iter().zip(other.sorts.iter()) {- let cmp = self_feature.cmp(other_feature);- if cmp != std::cmp::Ordering::Equal {- return cmp;- }- }- std::cmp::Ordering::Equal- }-}--impl PartialOrd for DocSortValuesAndFields {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl PartialEq for DocSortValuesAndFields {- fn eq(&self, other: &Self) -> bool {- self.cmp(other) == std::cmp::Ordering::Equal- }-}--impl Eq for DocSortValuesAndFields {}--/// The TopHitsCollector used for collecting over segments and merging results.-#[derive(Clone, Serialize, Deserialize, Debug)]-pub struct TopHitsTopNComputer {- req: TopHitsAggregation,- top_n: TopNComputer<DocSortValuesAndFields, DocAddress, false>,-}--impl std::cmp::PartialEq for TopHitsTopNComputer {- fn eq(&self, _other: &Self) -> bool {- false- }-}--impl TopHitsTopNComputer {- /// Create a new TopHitsCollector- pub fn new(req: &TopHitsAggregation) -> Self {- Self {- top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),- req: req.clone(),- }- }-- fn collect(&mut self, features: DocSortValuesAndFields, doc: DocAddress) {- self.top_n.push(features, doc);- }-- pub(crate) fn merge_fruits(&mut self, other_fruit: Self) -> crate::Result<()> {- for doc in other_fruit.top_n.into_vec() {- self.collect(doc.feature, doc.doc);- }- Ok(())- }-- /// Finalize by converting self into the final result form- pub fn into_final_result(self) -> TopHitsMetricResult {- let mut hits: Vec<TopHitsVecEntry> = self- .top_n- .into_sorted_vec()- .into_iter()- .map(|doc| TopHitsVecEntry {- sort: doc.feature.sorts.iter().map(|f| f.value).collect(),- doc_value_fields: doc- .feature- .doc_value_fields- .into_iter()- .map(|(k, v)| (k, v.into()))- .collect(),- })- .collect();-- // Remove the first `from` elements- // Truncating from end would be more efficient, but we need to truncate from the front- // because `into_sorted_vec` gives us a descending order because of the inverted- // `Ord` semantics of the heap elements.- hits.drain(..self.req.from.unwrap_or(0));- TopHitsMetricResult { hits }- }-}--#[derive(Clone, Debug)]-pub(crate) struct TopHitsSegmentCollector {- segment_ordinal: SegmentOrdinal,- accessor_idx: usize,- top_n: TopNComputer<Vec<DocValueAndOrder>, DocAddress, false>,-}--impl TopHitsSegmentCollector {- pub fn from_req(- req: &TopHitsAggregation,- accessor_idx: usize,- segment_ordinal: SegmentOrdinal,- ) -> Self {- Self {- top_n: TopNComputer::new(req.size + req.from.unwrap_or(0)),- segment_ordinal,- accessor_idx,- }- }- fn into_top_hits_collector(- self,- value_accessors: &HashMap<String, Vec<DynamicColumn>>,- req: &TopHitsAggregation,- ) -> TopHitsTopNComputer {- let mut top_hits_computer = TopHitsTopNComputer::new(req);- let top_results = self.top_n.into_vec();-- for res in top_results {- let doc_value_fields = req.get_document_field_data(value_accessors, res.doc.doc_id);- top_hits_computer.collect(- DocSortValuesAndFields {- sorts: res.feature,- doc_value_fields,- },- res.doc,- );- }-- top_hits_computer- }-- /// TODO add a specialized variant for a single sort field- fn collect_with(- &mut self,- doc_id: crate::DocId,- req: &TopHitsAggregation,- accessors: &[(Column<u64>, ColumnType)],- ) -> crate::Result<()> {- let sorts: Vec<DocValueAndOrder> = req- .sort- .iter()- .enumerate()- .map(|(idx, KeyOrder { order, .. })| {- let order = *order;- let value = accessors- .get(idx)- .expect("could not find field in accessors")- .0- .values_for_doc(doc_id)- .next();- DocValueAndOrder { value, order }- })- .collect();-- self.top_n.push(- sorts,- DocAddress {- segment_ord: self.segment_ordinal,- doc_id,- },- );- Ok(())- }-}--impl SegmentAggregationCollector for TopHitsSegmentCollector {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,- results: &mut crate::aggregation::intermediate_agg_result::IntermediateAggregationResults,- ) -> crate::Result<()> {- let name = agg_with_accessor.aggs.keys[self.accessor_idx].to_string();-- let value_accessors = &agg_with_accessor.aggs.values[self.accessor_idx].value_accessors;- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]- .agg- .agg- .as_top_hits()- .expect("aggregation request must be of type top hits");-- let intermediate_result = IntermediateMetricResult::TopHits(- self.into_top_hits_collector(value_accessors, tophits_req),- );- results.push(- name,- IntermediateAggregationResult::Metric(intermediate_result),- )- }-- /// TODO: Consider a caching layer to reduce the call overhead- fn collect(- &mut self,- doc_id: crate::DocId,- agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,- ) -> crate::Result<()> {- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]- .agg- .agg- .as_top_hits()- .expect("aggregation request must be of type top hits");- let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;- self.collect_with(doc_id, tophits_req, accessors)?;- Ok(())- }-- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut crate::aggregation::agg_req_with_accessor::AggregationsWithAccessor,- ) -> crate::Result<()> {- let tophits_req = &agg_with_accessor.aggs.values[self.accessor_idx]- .agg- .agg- .as_top_hits()- .expect("aggregation request must be of type top hits");- let accessors = &agg_with_accessor.aggs.values[self.accessor_idx].accessors;- // TODO: Consider getting fields with the column block accessor.- for doc in docs {- self.collect_with(*doc, tophits_req, accessors)?;- }- Ok(())- }-}--#[cfg(test)]-mod tests {- use crate::common::DateTime;- use pretty_assertions::assert_eq;- use serde_json::Value;- use time::macros::datetime;-- use super::{DocSortValuesAndFields, DocValueAndOrder, Order};- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::bucket::tests::get_test_index_from_docs;- use crate::aggregation::tests::get_test_index_from_values;- use crate::aggregation::AggregationCollector;- use crate::collector::ComparableDoc;- use crate::query::AllQuery;- use crate::schema::OwnedValue;-- fn invert_order(cmp_feature: DocValueAndOrder) -> DocValueAndOrder {- let DocValueAndOrder { value, order } = cmp_feature;- let order = match order {- Order::Asc => Order::Desc,- Order::Desc => Order::Asc,- };- DocValueAndOrder { value, order }- }-- fn collector_with_capacity(capacity: usize) -> super::TopHitsTopNComputer {- super::TopHitsTopNComputer {- top_n: super::TopNComputer::new(capacity),- req: Default::default(),- }- }-- fn invert_order_features(mut cmp_features: DocSortValuesAndFields) -> DocSortValuesAndFields {- cmp_features.sorts = cmp_features- .sorts- .into_iter()- .map(invert_order)- .collect::<Vec<_>>();- cmp_features- }-- #[test]- fn test_comparable_doc_feature() -> crate::Result<()> {- let small = DocValueAndOrder {- value: Some(1),- order: Order::Asc,- };- let big = DocValueAndOrder {- value: Some(2),- order: Order::Asc,- };- let none = DocValueAndOrder {- value: None,- order: Order::Asc,- };-- assert!(small < big);- assert!(none < small);- assert!(none < big);-- let small = invert_order(small);- let big = invert_order(big);- let none = invert_order(none);-- assert!(small > big);- assert!(none < small);- assert!(none < big);-- Ok(())- }-- #[test]- fn test_comparable_doc_features() -> crate::Result<()> {- let features_1 = DocSortValuesAndFields {- sorts: vec![DocValueAndOrder {- value: Some(1),- order: Order::Asc,- }],- doc_value_fields: Default::default(),- };-- let features_2 = DocSortValuesAndFields {- sorts: vec![DocValueAndOrder {- value: Some(2),- order: Order::Asc,- }],- doc_value_fields: Default::default(),- };-- assert!(features_1 < features_2);-- assert!(invert_order_features(features_1.clone()) > invert_order_features(features_2));-- Ok(())- }-- #[test]- fn test_aggregation_top_hits_empty_index() -> crate::Result<()> {- let values = vec![];-- let index = get_test_index_from_values(false, &values)?;-- let d: Aggregations = serde_json::from_value(json!({- "top_hits_req": {- "top_hits": {- "size": 2,- "sort": [- { "date": "desc" }- ],- "from": 0,- }- }- }))- .unwrap();-- let collector = AggregationCollector::from_aggs(d, Default::default());-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();-- let res: Value = serde_json::from_str(- &serde_json::to_string(&agg_res).expect("JSON serialization failed"),- )- .expect("JSON parsing failed");-- assert_eq!(- res,- json!({- "top_hits_req": {- "hits": []- }- })- );-- Ok(())- }-- #[test]- fn test_top_hits_collector_single_feature() -> crate::Result<()> {- let docs = vec![- ComparableDoc::<_, _, false> {- doc: crate::DocAddress {- segment_ord: 0,- doc_id: 0,- },- feature: DocSortValuesAndFields {- sorts: vec![DocValueAndOrder {- value: Some(1),- order: Order::Asc,- }],- doc_value_fields: Default::default(),- },- },- ComparableDoc {- doc: crate::DocAddress {- segment_ord: 0,- doc_id: 2,- },- feature: DocSortValuesAndFields {- sorts: vec![DocValueAndOrder {- value: Some(3),- order: Order::Asc,- }],- doc_value_fields: Default::default(),- },- },- ComparableDoc {- doc: crate::DocAddress {- segment_ord: 0,- doc_id: 1,- },- feature: DocSortValuesAndFields {- sorts: vec![DocValueAndOrder {- value: Some(5),- order: Order::Asc,- }],- doc_value_fields: Default::default(),- },- },- ];-- let mut collector = collector_with_capacity(3);- for doc in docs.clone() {- collector.collect(doc.feature, doc.doc);- }-- let res = collector.into_final_result();-- assert_eq!(- res,- super::TopHitsMetricResult {- hits: vec![- super::TopHitsVecEntry {- sort: vec![docs[0].feature.sorts[0].value],- doc_value_fields: Default::default(),- },- super::TopHitsVecEntry {- sort: vec![docs[1].feature.sorts[0].value],- doc_value_fields: Default::default(),- },- super::TopHitsVecEntry {- sort: vec![docs[2].feature.sorts[0].value],- doc_value_fields: Default::default(),- },- ]- }- );-- Ok(())- }-- fn test_aggregation_top_hits(merge_segments: bool) -> crate::Result<()> {- let docs = vec![- vec![- r#"{ "date": "2015-01-02T00:00:00Z", "text": "bbb", "text2": "bbb", "mixed": { "dyn_arr": [1, "2"] } }"#,- r#"{ "date": "2017-06-15T00:00:00Z", "text": "ccc", "text2": "ddd", "mixed": { "dyn_arr": [3, "4"] } }"#,- ],- vec![- r#"{ "text": "aaa", "text2": "bbb", "date": "2018-01-02T00:00:00Z", "mixed": { "dyn_arr": ["9", 8] } }"#,- r#"{ "text": "aaa", "text2": "bbb", "date": "2016-01-02T00:00:00Z", "mixed": { "dyn_arr": ["7", 6] } }"#,- ],- ];-- let index = get_test_index_from_docs(merge_segments, &docs)?;-- let d: Aggregations = serde_json::from_value(json!({- "top_hits_req": {- "top_hits": {- "size": 2,- "sort": [- { "date": "desc" }- ],- "from": 1,- "docvalue_fields": [- "date",- "tex*",- "mixed.*",- ],- }- }- }))?;-- let collector = AggregationCollector::from_aggs(d, Default::default());- let reader = index.reader()?;- let searcher = reader.searcher();-- let agg_res =- serde_json::to_value(searcher.search(&AllQuery, &collector).unwrap()).unwrap();-- let date_2017 = datetime!(2017-06-15 00:00:00 UTC);- let date_2016 = datetime!(2016-01-02 00:00:00 UTC);-- assert_eq!(- agg_res["top_hits_req"],- json!({- "hits": [- {- "sort": [crate::common::i64_to_u64(date_2017.unix_timestamp_nanos() as i64)],- "docvalue_fields": {- "date": [ OwnedValue::Date(DateTime::from_utc(date_2017)) ],- "text": [ "ccc" ],- "text2": [ "ddd" ],- "mixed.dyn_arr": [ 3, "4" ],- }- },- {- "sort": [crate::common::i64_to_u64(date_2016.unix_timestamp_nanos() as i64)],- "docvalue_fields": {- "date": [ OwnedValue::Date(DateTime::from_utc(date_2016)) ],- "text": [ "aaa" ],- "text2": [ "bbb" ],- "mixed.dyn_arr": [ 6, "7" ],- }- }- ]- }),- );-- Ok(())- }-- #[test]- fn test_aggregation_top_hits_single_segment() -> crate::Result<()> {- test_aggregation_top_hits(true)- }-- #[test]- fn test_aggregation_top_hits_multi_segment() -> crate::Result<()> {- test_aggregation_top_hits(false)- }-}diff --git a/crates/tantivy/src/aggregation/mod.rs b/crates/tantivy/src/aggregation/mod.rsdeleted file mode 100644index 8c0a4b00..00000000--- a/crates/tantivy/src/aggregation/mod.rs+++ /dev/null@@ -1,695 +0,0 @@-//! # Aggregations-//!-//! An aggregation summarizes your data as statistics on buckets or metrics.-//!-//! Aggregations can provide answer to questions like:-//! - What is the average price of all sold articles?-//! - How many errors with status code 500 do we have per day?-//! - What is the average listing price of cars grouped by color?-//!-//! There are two categories: [Metrics](metric) and [Buckets](bucket).-//!-//! ## Prerequisite-//! Currently aggregations work only on [fast fields](`crate::fastfield`). Fast fields-//! of type `u64`, `f64`, `i64`, `date` and fast fields on text fields.-//!-//! ## Usage-//! To use aggregations, build an aggregation request by constructing-//! [`Aggregations`](agg_req::Aggregations).-//! Create an [`AggregationCollector`] from this request. `AggregationCollector` implements the-//! [`Collector`](crate::collector::Collector) trait and can be passed as collector into-//! [`Searcher::search()`](crate::Searcher::search).-//!-//!-//! ## JSON Format-//! Aggregations request and result structures de/serialize into elasticsearch compatible JSON.-//!-//! Notice: Intermediate aggregation results should not be de/serialized via JSON format.-//! Postcard is a good choice.-//!-//! ```verbatim-//! let agg_req: Aggregations = serde_json::from_str(json_request_string).unwrap();-//! let collector = AggregationCollector::from_aggs(agg_req, None);-//! let searcher = reader.searcher();-//! let agg_res = searcher.search(&term_query, &collector).unwrap_err();-//! let json_response_string: String = &serde_json::to_string(&agg_res)?;-//! ```-//!-//! ## Supported Aggregations-//! - [Bucket](bucket)-//! - [Histogram](bucket::HistogramAggregation)-//! - [DateHistogram](bucket::DateHistogramAggregationReq)-//! - [Range](bucket::RangeAggregation)-//! - [Terms](bucket::TermsAggregation)-//! - [Metric](metric)-//! - [Average](metric::AverageAggregation)-//! - [Stats](metric::StatsAggregation)-//! - [Min](metric::MinAggregation)-//! - [Max](metric::MaxAggregation)-//! - [Sum](metric::SumAggregation)-//! - [Count](metric::CountAggregation)-//! - [Percentiles](metric::PercentilesAggregationReq)-//!-//! # Example-//! Compute the average metric, by building [`agg_req::Aggregations`], which is built from an-//! `(String, agg_req::Aggregation)` iterator.-//!-//! Requests are compatible with the elasticsearch JSON request format.-//!-//! ```-//! use tantivy::aggregation::agg_req::Aggregations;-//!-//! let elasticsearch_compatible_json_req = r#"-//! {-//! "average": {-//! "avg": { "field": "score" }-//! },-//! "range": {-//! "range": {-//! "field": "score",-//! "ranges": [-//! { "to": 3.0 },-//! { "from": 3.0, "to": 7.0 },-//! { "from": 7.0, "to": 20.0 },-//! { "from": 20.0 }-//! ]-//! },-//! "aggs": {-//! "average_in_range": { "avg": { "field": "score" } }-//! }-//! }-//! }-//! "#;-//! let agg_req: Aggregations =-//! serde_json::from_str(elasticsearch_compatible_json_req).unwrap();-//! ```-//! # Code Organization-//!-//! Check the [README](https://github.com/quickwit-oss/tantivy/tree/main/src/aggregation#readme) on github to see how the code is organized.-//!-//! # Nested Aggregation-//!-//! Buckets can contain sub-aggregations. In this example we create buckets with the range-//! aggregation and then calculate the average on each bucket.-//! ```-//! use tantivy::aggregation::agg_req::*;-//! use serde_json::json;-//!-//! let agg_req_1: Aggregations = serde_json::from_value(json!({-//! "rangef64": {-//! "range": {-//! "field": "score",-//! "ranges": [-//! { "from": 3, "to": 7000 },-//! { "from": 7000, "to": 20000 },-//! { "from": 50000, "to": 60000 }-//! ]-//! },-//! "aggs": {-//! "average_in_range": { "avg": { "field": "score" } }-//! }-//! },-//! }))-//! .unwrap();-//! ```-//!-//! # Distributed Aggregation-//! When the data is distributed on different [`Index`](crate::Index) instances, the-//! [`DistributedAggregationCollector`] provides functionality to merge data between independent-//! search calls by returning-//! [`IntermediateAggregationResults`](intermediate_agg_result::IntermediateAggregationResults).-//! `IntermediateAggregationResults` provides the-//! [`merge_fruits`](intermediate_agg_result::IntermediateAggregationResults::merge_fruits) method-//! to merge multiple results. The merged result can then be converted into-//! [`AggregationResults`](agg_result::AggregationResults) via the-//! [`into_final_result`](intermediate_agg_result::IntermediateAggregationResults::into_final_result) method.--mod agg_limits;-pub mod agg_req;-mod agg_req_with_accessor;-pub mod agg_result;-pub mod bucket;-mod buf_collector;-mod collector;-mod date;-mod error;-pub mod intermediate_agg_result;-pub mod metric;--mod segment_agg_result;-use std::collections::HashMap;-use std::fmt::Display;--use core::fmt;--use crate::columnar::{ColumnType, MonotonicallyMappableToU64};-pub use agg_limits::AggregationLimits;-pub use collector::{- AggregationCollector, AggregationSegmentCollector, DistributedAggregationCollector,- DEFAULT_BUCKET_LIMIT,-};-pub(crate) use date::format_date;-pub use error::AggregationError;-use itertools::Itertools;-use serde::de::{self, Visitor};-use serde::{Deserialize, Deserializer, Serialize};--fn parse_str_into_f64<E: de::Error>(value: &str) -> Result<f64, E> {- let parsed = value- .parse::<f64>()- .map_err(|_err| de::Error::custom(format!("Failed to parse f64 from string: {value:?}")))?;-- // Check if the parsed value is NaN or infinity- if parsed.is_nan() || parsed.is_infinite() {- Err(de::Error::custom(format!(- "Value is not a valid f64 (NaN or Infinity): {value:?}"- )))- } else {- Ok(parsed)- }-}--/// deserialize Option<f64> from string or float-pub(crate) fn deserialize_option_f64<'de, D>(deserializer: D) -> Result<Option<f64>, D::Error>-where- D: Deserializer<'de>,-{- struct StringOrFloatVisitor;-- impl<'de> Visitor<'de> for StringOrFloatVisitor {- type Value = Option<f64>;-- fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {- formatter.write_str("a string or a float")- }-- fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>- where- E: de::Error,- {- parse_str_into_f64(value).map(Some)- }-- fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(Some(value))- }-- fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(Some(value as f64))- }-- fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(Some(value as f64))- }-- fn visit_none<E>(self) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(None)- }-- fn visit_unit<E>(self) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(None)- }- }-- deserializer.deserialize_any(StringOrFloatVisitor)-}--/// deserialize f64 from string or float-pub(crate) fn deserialize_f64<'de, D>(deserializer: D) -> Result<f64, D::Error>-where- D: Deserializer<'de>,-{- struct StringOrFloatVisitor;-- impl<'de> Visitor<'de> for StringOrFloatVisitor {- type Value = f64;-- fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {- formatter.write_str("a string or a float")- }-- fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>- where- E: de::Error,- {- parse_str_into_f64(value)- }-- fn visit_f64<E>(self, value: f64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(value)- }-- fn visit_i64<E>(self, value: i64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(value as f64)- }-- fn visit_u64<E>(self, value: u64) -> Result<Self::Value, E>- where- E: de::Error,- {- Ok(value as f64)- }- }-- deserializer.deserialize_any(StringOrFloatVisitor)-}--/// Represents an associative array `(key => values)` in a very efficient manner.-#[derive(PartialEq, Serialize, Deserialize)]-pub(crate) struct VecWithNames<T> {- pub(crate) values: Vec<T>,- keys: Vec<String>,-}--impl<T: Clone> Clone for VecWithNames<T> {- fn clone(&self) -> Self {- Self {- values: self.values.clone(),- keys: self.keys.clone(),- }- }-}--impl<T> Default for VecWithNames<T> {- fn default() -> Self {- Self {- values: Default::default(),- keys: Default::default(),- }- }-}--impl<T: std::fmt::Debug> std::fmt::Debug for VecWithNames<T> {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_map().entries(self.iter()).finish()- }-}--impl<T> From<HashMap<String, T>> for VecWithNames<T> {- fn from(map: HashMap<String, T>) -> Self {- VecWithNames::from_entries(map.into_iter().collect_vec())- }-}--impl<T> VecWithNames<T> {- fn from_entries(mut entries: Vec<(String, T)>) -> Self {- // Sort to ensure order of elements match across multiple instances- entries.sort_by(|left, right| left.0.cmp(&right.0));- let mut data = Vec::with_capacity(entries.len());- let mut data_names = Vec::with_capacity(entries.len());- for entry in entries {- data_names.push(entry.0);- data.push(entry.1);- }- VecWithNames {- values: data,- keys: data_names,- }- }- fn iter(&self) -> impl Iterator<Item = (&str, &T)> + '_ {- self.keys().zip(self.values.iter())- }- fn keys(&self) -> impl Iterator<Item = &str> + '_ {- self.keys.iter().map(|key| key.as_str())- }- fn values_mut(&mut self) -> impl Iterator<Item = &mut T> + '_ {- self.values.iter_mut()- }- fn is_empty(&self) -> bool {- self.keys.is_empty()- }- fn len(&self) -> usize {- self.keys.len()- }- fn get(&self, name: &str) -> Option<&T> {- self.keys()- .position(|key| key == name)- .map(|pos| &self.values[pos])- }-}--/// The serialized key is used in a `HashMap`.-pub type SerializedKey = String;--#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd)]-/// The key to identify a bucket.-#[serde(untagged)]-pub enum Key {- /// String key- Str(String),- /// `f64` key- F64(f64),-}-impl Eq for Key {}-impl std::hash::Hash for Key {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- core::mem::discriminant(self).hash(state);- match self {- Key::Str(text) => text.hash(state),- Key::F64(val) => val.to_bits().hash(state),- }- }-}--impl PartialEq for Key {- fn eq(&self, other: &Self) -> bool {- match (self, other) {- (Self::Str(l), Self::Str(r)) => l == r,- (Self::F64(l), Self::F64(r)) => l == r,- _ => false,- }- }-}--impl Display for Key {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- Key::Str(val) => f.write_str(val),- Key::F64(val) => f.write_str(&val.to_string()),- }- }-}--/// Inverse of `to_fastfield_u64`. Used to convert to `f64` for metrics.-///-/// # Panics-/// Only `u64`, `f64`, `date`, and `i64` are supported.-pub(crate) fn f64_from_fastfield_u64(val: u64, field_type: &ColumnType) -> f64 {- match field_type {- ColumnType::U64 => val as f64,- ColumnType::I64 | ColumnType::DateTime => i64::from_u64(val) as f64,- ColumnType::F64 => f64::from_u64(val),- ColumnType::Bool => val as f64,- _ => {- panic!("unexpected type {field_type:?}. This should not happen")- }- }-}--/// Converts the `f64` value to fast field value space, which is always u64.-///-/// If the fast field has `u64`, values are stored unchanged as `u64` in the fast field.-///-/// If the fast field has `f64` values are converted and stored to `u64` using a-/// monotonic mapping.-/// A `f64` value of e.g. `2.0` needs to be converted using the same monotonic-/// conversion function, so that the value matches the `u64` value stored in the fast-/// field.-pub(crate) fn f64_to_fastfield_u64(val: f64, field_type: &ColumnType) -> Option<u64> {- match field_type {- ColumnType::U64 => Some(val as u64),- ColumnType::I64 | ColumnType::DateTime => Some((val as i64).to_u64()),- ColumnType::F64 => Some(val.to_u64()),- ColumnType::Bool => Some(val as u64),- _ => None,- }-}--#[cfg(test)]-mod tests {- use std::net::Ipv6Addr;-- use crate::columnar::DateTime;- use serde_json::Value;- use time::OffsetDateTime;-- use super::agg_req::Aggregations;- use super::*;- use crate::indexer::NoMergePolicy;- use crate::query::{AllQuery, TermQuery};- use crate::schema::{IndexRecordOption, Schema, TextFieldIndexing, FAST, STRING};- use crate::{Index, IndexWriter, Term};-- pub fn get_test_index_with_num_docs(- merge_segments: bool,- num_docs: usize,- ) -> crate::Result<Index> {- get_test_index_from_values(- merge_segments,- &(0..num_docs).map(|el| el as f64).collect::<Vec<f64>>(),- )- }-- pub fn exec_request(agg_req: Aggregations, index: &Index) -> crate::Result<Value> {- exec_request_with_query(agg_req, index, None)- }- pub fn exec_request_with_query(- agg_req: Aggregations,- index: &Index,- query: Option<(&str, &str)>,- ) -> crate::Result<Value> {- exec_request_with_query_and_memory_limit(agg_req, index, query, Default::default())- }-- pub fn exec_request_with_query_and_memory_limit(- agg_req: Aggregations,- index: &Index,- query: Option<(&str, &str)>,- limits: AggregationLimits,- ) -> crate::Result<Value> {- let collector = AggregationCollector::from_aggs(agg_req, limits);-- let reader = index.reader()?;- let searcher = reader.searcher();- let agg_res = if let Some((field, term)) = query {- let text_field = reader.searcher().schema().get_field(field).unwrap();-- let term_query = TermQuery::new(- Term::from_field_text(text_field, term),- IndexRecordOption::Basic,- );-- searcher.search(&term_query, &collector)?- } else {- searcher.search(&AllQuery, &collector)?- };-- // Test serialization/deserialization roundtrip- let res: Value = serde_json::from_str(&serde_json::to_string(&agg_res)?)?;- Ok(res)- }-- pub fn get_test_index_from_values(- merge_segments: bool,- values: &[f64],- ) -> crate::Result<Index> {- // Every value gets its own segment- let mut segment_and_values = vec![];- for value in values {- segment_and_values.push(vec![(*value, value.to_string())]);- }- get_test_index_from_values_and_terms(merge_segments, &segment_and_values)- }-- pub fn get_test_index_from_terms(- merge_segments: bool,- values: &[Vec<&str>],- ) -> crate::Result<Index> {- // Every value gets its own segment- let segment_and_values = values- .iter()- .map(|terms| {- terms- .iter()- .enumerate()- .map(|(i, term)| (i as f64, term.to_string()))- .collect()- })- .collect::<Vec<_>>();- get_test_index_from_values_and_terms(merge_segments, &segment_and_values)- }-- pub fn get_test_index_from_values_and_terms(- merge_segments: bool,- segment_and_values: &[Vec<(f64, String)>],- ) -> crate::Result<Index> {- let mut schema_builder = Schema::builder();- let text_fieldtype = crate::schema::TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default()- .set_index_option(IndexRecordOption::Basic)- .set_fieldnorms(false),- )- .set_fast(None)- .set_stored();- let text_field = schema_builder.add_text_field("text", text_fieldtype.clone());- let text_field_id = schema_builder.add_text_field("text_id", text_fieldtype);- let string_field_id = schema_builder.add_text_field("string_id", STRING | FAST);- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();- let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());- let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());- let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);- let fraction_field = schema_builder.add_f64_field(- "fraction_f64",- crate::schema::NumericOptions::default().set_fast(),- );- let index = Index::create_in_ram(schema_builder.build());- {- // let mut index_writer = index.writer_for_tests()?;- let mut index_writer = index.writer_with_num_threads(1, 20_000_000)?;- index_writer.set_merge_policy(Box::new(NoMergePolicy));- for values in segment_and_values {- for (i, term) in values {- let i = *i;- // writing the segment- index_writer.add_document(doc!(- text_field => "cool",- text_field_id => term.to_string(),- string_field_id => term.to_string(),- score_field => i as u64,- score_field_f64 => i,- score_field_i64 => i as i64,- fraction_field => i/100.0,- ))?;- }- index_writer.commit()?;- }- }- if merge_segments {- let segment_ids = index- .searchable_segment_ids()- .expect("Searchable segments failed.");- if segment_ids.len() > 1 {- let mut index_writer: IndexWriter = index.writer_for_tests()?;- index_writer.merge(&segment_ids).wait()?;- index_writer.wait_merging_threads()?;- }- }-- Ok(index)- }-- pub fn get_test_index_2_segments(merge_segments: bool) -> crate::Result<Index> {- let mut schema_builder = Schema::builder();- let text_fieldtype = crate::schema::TextOptions::default()- .set_indexing_options(- TextFieldIndexing::default().set_index_option(IndexRecordOption::WithFreqs),- )- .set_fast(None)- .set_stored();- let text_field = schema_builder.add_text_field("text", text_fieldtype);- let date_field = schema_builder.add_date_field("date", FAST);- schema_builder.add_text_field("dummy_text", STRING);- let score_fieldtype = crate::schema::NumericOptions::default().set_fast();- let score_field = schema_builder.add_u64_field("score", score_fieldtype.clone());- let score_field_f64 = schema_builder.add_f64_field("score_f64", score_fieldtype.clone());- let ip_addr_field = schema_builder.add_ip_addr_field("ip_addr", FAST);-- let multivalue = crate::schema::NumericOptions::default().set_fast();- let scores_field_i64 = schema_builder.add_i64_field("scores_i64", multivalue);-- let score_field_i64 = schema_builder.add_i64_field("score_i64", score_fieldtype);- let index = Index::create_in_ram(schema_builder.build());- {- let mut index_writer = index.writer_for_tests()?;- // writing the segment- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800).unwrap()),- score_field => 1u64,- ip_addr_field => Ipv6Addr::from(1u128),- score_field_f64 => 1f64,- score_field_i64 => 1i64,- scores_field_i64 => 1i64,- scores_field_i64 => 2i64,- ))?;- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),- score_field => 3u64,- score_field_f64 => 3f64,- score_field_i64 => 3i64,- scores_field_i64 => 5i64,- scores_field_i64 => 5i64,- ))?;- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),- score_field => 5u64,- score_field_f64 => 5f64,- score_field_i64 => 5i64,- ))?;- index_writer.add_document(doc!(- text_field => "nohit",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),- score_field => 6u64,- score_field_f64 => 6f64,- score_field_i64 => 6i64,- ))?;- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),- score_field => 7u64,- score_field_f64 => 7f64,- score_field_i64 => 7i64,- ))?;- index_writer.commit()?;- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400).unwrap()),- score_field => 11u64,- score_field_f64 => 11f64,- score_field_i64 => 11i64,- ))?;- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),- score_field => 14u64,- score_field_f64 => 14f64,- score_field_i64 => 14i64,- ))?;-- index_writer.add_document(doc!(- text_field => "cool",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),- score_field => 44u64,- score_field_f64 => 44.5f64,- score_field_i64 => 44i64,- ))?;-- index_writer.commit()?;-- // no hits segment- index_writer.add_document(doc!(- text_field => "nohit",- date_field => DateTime::from_utc(OffsetDateTime::from_unix_timestamp(1_546_300_800 + 86400 + 86400).unwrap()),- score_field => 44u64,- score_field_f64 => 44.5f64,- score_field_i64 => 44i64,- ))?;-- index_writer.commit()?;- }- if merge_segments {- let segment_ids = index- .searchable_segment_ids()- .expect("Searchable segments failed.");- let mut index_writer: IndexWriter = index.writer_for_tests()?;- index_writer.merge(&segment_ids).wait()?;- index_writer.wait_merging_threads()?;- }-- Ok(index)- }-}diff --git a/crates/tantivy/src/aggregation/segment_agg_result.rs b/crates/tantivy/src/aggregation/segment_agg_result.rsdeleted file mode 100644index cc0aab5a..00000000--- a/crates/tantivy/src/aggregation/segment_agg_result.rs+++ /dev/null@@ -1,245 +0,0 @@-//! Contains aggregation trees which is used during collection in a segment.-//! This tree contains datastructrues optimized for fast collection.-//! The tree can be converted to an intermediate tree, which contains datastructrues optimized for-//! merging.--use std::fmt::Debug;--pub(crate) use super::agg_limits::AggregationLimits;-use super::agg_req::AggregationVariants;-use super::agg_req_with_accessor::{AggregationWithAccessor, AggregationsWithAccessor};-use super::bucket::{SegmentHistogramCollector, SegmentRangeCollector, SegmentTermCollector};-use super::intermediate_agg_result::IntermediateAggregationResults;-use super::metric::{- AverageAggregation, CountAggregation, ExtendedStatsAggregation, MaxAggregation, MinAggregation,- SegmentPercentilesCollector, SegmentStatsCollector, SegmentStatsType, StatsAggregation,- SumAggregation,-};-use crate::aggregation::bucket::TermMissingAgg;-use crate::aggregation::metric::{SegmentExtendedStatsCollector, TopHitsSegmentCollector};--pub(crate) trait SegmentAggregationCollector: CollectorClone + Debug {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()>;-- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()>;-- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()>;-- /// Finalize method. Some Aggregator collect blocks of docs before calling `collect_block`.- /// This method ensures those staged docs will be collected.- fn flush(&mut self, _agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- Ok(())- }-}--pub(crate) trait CollectorClone {- fn clone_box(&self) -> Box<dyn SegmentAggregationCollector>;-}--impl<T> CollectorClone for T-where T: 'static + SegmentAggregationCollector + Clone-{- fn clone_box(&self) -> Box<dyn SegmentAggregationCollector> {- Box::new(self.clone())- }-}--impl Clone for Box<dyn SegmentAggregationCollector> {- fn clone(&self) -> Box<dyn SegmentAggregationCollector> {- self.clone_box()- }-}--pub(crate) fn build_segment_agg_collector(- req: &mut AggregationsWithAccessor,-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {- // Single collector special case- if req.aggs.len() == 1 {- let req = &mut req.aggs.values[0];- let accessor_idx = 0;- return build_single_agg_segment_collector(req, accessor_idx);- }-- let agg = GenericSegmentAggregationResultsCollector::from_req_and_validate(req)?;- Ok(Box::new(agg))-}--pub(crate) fn build_single_agg_segment_collector(- req: &mut AggregationWithAccessor,- accessor_idx: usize,-) -> crate::Result<Box<dyn SegmentAggregationCollector>> {- use AggregationVariants::*;- match &req.agg.agg {- Terms(terms_req) => {- if req.accessors.is_empty() {- Ok(Box::new(SegmentTermCollector::from_req_and_validate(- terms_req,- &mut req.sub_aggregation,- req.field_type,- accessor_idx,- )?))- } else {- Ok(Box::new(TermMissingAgg::new(- accessor_idx,- &mut req.sub_aggregation,- )?))- }- }- Range(range_req) => Ok(Box::new(SegmentRangeCollector::from_req_and_validate(- range_req,- &mut req.sub_aggregation,- &req.limits,- req.field_type,- accessor_idx,- )?)),- Histogram(histogram) => Ok(Box::new(SegmentHistogramCollector::from_req_and_validate(- histogram.clone(),- &mut req.sub_aggregation,- req.field_type,- accessor_idx,- )?)),- DateHistogram(histogram) => Ok(Box::new(SegmentHistogramCollector::from_req_and_validate(- histogram.to_histogram_req()?,- &mut req.sub_aggregation,- req.field_type,- accessor_idx,- )?)),- Average(AverageAggregation { missing, .. }) => {- Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Average,- accessor_idx,- *missing,- )))- }- Count(CountAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Count,- accessor_idx,- *missing,- ))),- Max(MaxAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Max,- accessor_idx,- *missing,- ))),- Min(MinAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Min,- accessor_idx,- *missing,- ))),- Stats(StatsAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Stats,- accessor_idx,- *missing,- ))),- ExtendedStats(ExtendedStatsAggregation { missing, sigma, .. }) => Ok(Box::new(- SegmentExtendedStatsCollector::from_req(req.field_type, *sigma, accessor_idx, *missing),- )),- Sum(SumAggregation { missing, .. }) => Ok(Box::new(SegmentStatsCollector::from_req(- req.field_type,- SegmentStatsType::Sum,- accessor_idx,- *missing,- ))),- Percentiles(percentiles_req) => Ok(Box::new(- SegmentPercentilesCollector::from_req_and_validate(- percentiles_req,- req.field_type,- accessor_idx,- )?,- )),- TopHits(top_hits_req) => Ok(Box::new(TopHitsSegmentCollector::from_req(- top_hits_req,- accessor_idx,- req.segment_ordinal,- ))),- }-}--#[derive(Clone, Default)]-/// The GenericSegmentAggregationResultsCollector is the generic version of the collector, which-/// can handle arbitrary complexity of sub-aggregations. Ideally we never have to pick this one-/// and can provide specialized versions instead, that remove some of its overhead.-pub(crate) struct GenericSegmentAggregationResultsCollector {- pub(crate) aggs: Vec<Box<dyn SegmentAggregationCollector>>,-}--impl Debug for GenericSegmentAggregationResultsCollector {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- f.debug_struct("SegmentAggregationResultsCollector")- .field("aggs", &self.aggs)- .finish()- }-}--impl SegmentAggregationCollector for GenericSegmentAggregationResultsCollector {- fn add_intermediate_aggregation_result(- self: Box<Self>,- agg_with_accessor: &AggregationsWithAccessor,- results: &mut IntermediateAggregationResults,- ) -> crate::Result<()> {- for agg in self.aggs {- agg.add_intermediate_aggregation_result(agg_with_accessor, results)?;- }-- Ok(())- }-- fn collect(- &mut self,- doc: crate::DocId,- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- self.collect_block(&[doc], agg_with_accessor)?;-- Ok(())- }-- fn collect_block(- &mut self,- docs: &[crate::DocId],- agg_with_accessor: &mut AggregationsWithAccessor,- ) -> crate::Result<()> {- for collector in &mut self.aggs {- collector.collect_block(docs, agg_with_accessor)?;- }-- Ok(())- }-- fn flush(&mut self, agg_with_accessor: &mut AggregationsWithAccessor) -> crate::Result<()> {- for collector in &mut self.aggs {- collector.flush(agg_with_accessor)?;- }- Ok(())- }-}--impl GenericSegmentAggregationResultsCollector {- pub(crate) fn from_req_and_validate(req: &mut AggregationsWithAccessor) -> crate::Result<Self> {- let aggs = req- .aggs- .values_mut()- .enumerate()- .map(|(accessor_idx, req)| build_single_agg_segment_collector(req, accessor_idx))- .collect::<crate::Result<Vec<Box<dyn SegmentAggregationCollector>>>>()?;-- Ok(GenericSegmentAggregationResultsCollector { aggs })- }-}diff --git a/crates/tantivy/src/error.rs b/crates/tantivy/src/error.rsindex 487cc21a..ade94d54 100644--- a/crates/tantivy/src/error.rs+++ b/crates/tantivy/src/error.rs@@ -6,7 +6,6 @@ use std::{fmt, io};use thiserror::Error;-use crate::aggregation::AggregationError;use crate::directory::error::{Incompatibility, LockError, OpenDirectoryError, OpenReadError, OpenWriteError,};@@ -55,9 +54,6 @@ impl fmt::Debug for DataCorruption {/// The library's error enum#[derive(Debug, Clone, Error)]pub enum TantivyError {- /// Error when handling aggregations.- #[error(transparent)]- AggregationError(#[from] AggregationError),/// Failed to open the directory.#[error("Failed to open the directory: '{0:?}'")]OpenDirectoryError(#[from] OpenDirectoryError),diff --git a/crates/tantivy/src/indexer/mod.rs b/crates/tantivy/src/indexer/mod.rsindex 64b121d4..60bed2cc 100644--- a/crates/tantivy/src/indexer/mod.rs+++ b/crates/tantivy/src/indexer/mod.rs@@ -61,12 +61,9 @@ type AddBatchReceiver<D> = channel::Receiver<AddBatch<D>>;#[cfg(test)]mod tests_mmap {- use crate::aggregation::agg_req::Aggregations;- use crate::aggregation::agg_result::AggregationResults;- use crate::aggregation::AggregationCollector;use crate::collector::{Count, TopDocs};use crate::index::FieldMetadata;- use crate::query::{AllQuery, QueryParser};+ use crate::query::QueryParser;use crate::schema::{JsonObjectOptions, Schema, Type, FAST, INDEXED, STORED, TEXT};use crate::{Index, IndexWriter, Term};@@ -226,41 +223,6 @@ mod tests_mmap {test_query(format!("json.a{field_name_out}.a{field_name_out}:test6").as_str());test_query(format!("json.{field_name_out}a:test7").as_str());- let test_agg = |field_name: &str, expected: &str| {- let agg_req_str = json!(- {- "termagg": {- "terms": {- "field": field_name,- }- }- });-- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();- let collector = AggregationCollector::from_aggs(agg_req, Default::default());- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let res = serde_json::to_value(agg_res).unwrap();- assert_eq!(res["termagg"]["buckets"][0]["doc_count"], 1);- assert_eq!(res["termagg"]["buckets"][0]["key"], expected);- };-- test_agg(format!("json.{field_name_out}").as_str(), "test1");- test_agg(format!("json.a{field_name_out}").as_str(), "test2");- test_agg(format!("json.a{field_name_out}a").as_str(), "test3");- test_agg(- format!("json.a{field_name_out}a{field_name_out}").as_str(),- "test4",- );- test_agg(- format!("json.a{field_name_out}.ab{field_name_out}").as_str(),- "test5",- );- test_agg(- format!("json.a{field_name_out}.a{field_name_out}").as_str(),- "test6",- );- test_agg(format!("json.{field_name_out}a").as_str(), "test7");-// `.` is stored as `\u{0001}` internally in tantivylet field_name_out_internal = if field_name_out == "." {"\u{0001}"@@ -530,94 +492,5 @@ mod tests_mmap {assert!(!count_docs.is_empty(), "{}", indexed_field.field_name);}}- // Test if returned field name can be used for aggregation- for fast_field in fields_metadata.iter().filter(|meta| meta.fast) {- let agg_req_str = json!(- {- "termagg": {- "terms": {- "field": fast_field.field_name,- }- }- });-- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();- let collector = AggregationCollector::from_aggs(agg_req, Default::default());- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let res = serde_json::to_value(agg_res).unwrap();- if !fast_field.field_name.contains("empty") && fast_field.typ != Type::Json {- assert!(- !res["termagg"]["buckets"].as_array().unwrap().is_empty(),- "{}",- fast_field.field_name- );- }- }- }-- #[test]- fn test_json_field_shadowing_field_name_bug() {- /// This test is only there to display a bug on addressing a field if it gets shadowed- /// The issues only occurs if the field name that shadows contains a dot.- ///- /// Happens independently of the `expand_dots` option. Since that option does not- /// affect the field name itself.- use pretty_assertions::assert_eq;- let mut schema_builder = Schema::builder();- let json_options: JsonObjectOptions =- JsonObjectOptions::from(TEXT).set_fast(None).set_stored();- // let json_options = json_options.set_expand_dots_enabled();- let json_field_shadow = schema_builder.add_json_field("json.shadow", json_options.clone());- let json_field = schema_builder.add_json_field("json", json_options.clone());- let index = Index::create_in_ram(schema_builder.build());- let mut index_writer = index.writer_for_tests().unwrap();- index_writer- .add_document(- doc!(json_field_shadow=>json!({"val": "b"}), json_field=>json!({"shadow": {"val": "a"}})),- )- .unwrap();- index_writer.commit().unwrap();- let reader = index.reader().unwrap();-- let searcher = reader.searcher();-- let fields_and_vals = [- ("json.shadow\u{1}val".to_string(), "a"), // Succeeds- //("json.shadow.val".to_string(), "a"), // Fails- ("json.shadow.val".to_string(), "b"),- ];-- let query_parser = QueryParser::for_index(&index, vec![]);- // Test if field name can be queried- for (indexed_field, val) in fields_and_vals.iter() {- let query_str = &format!("{indexed_field}:{val}");- let query = query_parser.parse_query(query_str).unwrap();- let count_docs = searcher.search(&*query, &TopDocs::with_limit(2)).unwrap();- assert!(!count_docs.is_empty(), "{indexed_field}:{val}");- }- // Test if field name can be used for aggregation- for (field_name, val) in fields_and_vals.iter() {- let agg_req_str = json!(- {- "termagg": {- "terms": {- "field": field_name,- }- }- });-- let agg_req: Aggregations = serde_json::from_value(agg_req_str).unwrap();- let collector = AggregationCollector::from_aggs(agg_req, Default::default());- let agg_res: AggregationResults = searcher.search(&AllQuery, &collector).unwrap();- let res = serde_json::to_value(agg_res).unwrap();- assert_eq!(- res["termagg"]["buckets"].as_array().unwrap()[0]["key"]- .as_str()- .unwrap(),- *val,- "{}",- field_name- );- }}}diff --git a/crates/tantivy/src/lib.rs b/crates/tantivy/src/lib.rsindex 726a2008..14e52c50 100644--- a/crates/tantivy/src/lib.rs+++ b/crates/tantivy/src/lib.rs@@ -191,7 +191,6 @@ mod sstable;pub mod error;pub mod tokenizer;-pub mod aggregation;pub mod collector;pub mod directory;pub mod fastfield;