Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

62.1 KB
Raw Blame
#include <gtest/gtest.h>
#include <string>
#include <vector>
#include <fstream>
#include <algorithm>
#include <collection_manager.h>
#include "collection.h"

class CollectionGroupingTest : public ::testing::Test {
protected:
    Store *store;
    CollectionManager & collectionManager = CollectionManager::get_instance();
    std::atomic<bool> quit = false;
    Collection *coll_group;

    void setupCollection() {
        std::string state_dir_path = "/tmp/typesense_test/collection_grouping";
        LOG(INFO) << "Truncating and creating: " << state_dir_path;
        system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());

        store = new Store(state_dir_path);
        collectionManager.init(store, 1.0, "auth_key", quit);
        collectionManager.load(8, 1000);

        std::vector<field> fields = {
            field("title", field_types::STRING, false),
            field("brand", field_types::STRING, true, true),
            field("size", field_types::INT32, true, false),
            field("colors", field_types::STRING_ARRAY, true, false),
            field("rating", field_types::FLOAT, true, false)
        };

        coll_group = collectionManager.get_collection("coll_group").get();
        if(coll_group == nullptr) {
            coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get();
        }

        std::ifstream infile(std::string(ROOT_DIR)+"test/group_documents.jsonl");

        std::string json_line;

        while (std::getline(infile, json_line)) {
            auto add_op = coll_group->add(json_line);
            if(!add_op.ok()) {
                std::cout << add_op.error() << std::endl;
            }
            ASSERT_TRUE(add_op.ok());
        }

        infile.close();
    }

    virtual void SetUp() {
        setupCollection();
    }

    virtual void TearDown() {
        collectionManager.dispose();
        delete store;
    }
};

TEST_F(CollectionGroupingTest, GroupingBasics) {
    // group by size (int32)
    auto res = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(3, res["found"].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"].size());
    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());

    ASSERT_EQ(2, res["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ(11, res["grouped_hits"][0]["hits"][0]["document"]["size"].get<size_t>());
    ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("1", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(7, res["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(3, res["grouped_hits"][2]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("2", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
    ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());

    // group by rating (float) and sort by size
    std::vector<sort_by> sort_size = {sort_by("size", "DESC")};
    res = coll_group->search("*", {}, "", {"brand"}, sort_size, {0}, 50, 1, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "brand: omeg", 30, 5,
                             "", 10,
                             {}, {}, {"rating"}, 2).get();

    // 7 unique ratings
    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(7, res["found"].get<size_t>());
    ASSERT_EQ(7, res["grouped_hits"].size());
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][0]["group_key"][0].get<float>());

    ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_EQ(12, res["grouped_hits"][0]["hits"][0]["document"]["size"].get<uint32_t>());
    ASSERT_STREQ("8", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());

    ASSERT_EQ(4, res["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_EQ(12, res["grouped_hits"][1]["hits"][0]["document"]["size"].get<uint32_t>());
    ASSERT_STREQ("6", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
    
    ASSERT_EQ(11, res["grouped_hits"][1]["hits"][1]["document"]["size"].get<uint32_t>());
    ASSERT_STREQ("1", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());

    ASSERT_EQ(1, res["grouped_hits"][5]["found"].get<int32_t>());
    ASSERT_EQ(10, res["grouped_hits"][5]["hits"][0]["document"]["size"].get<uint32_t>());
    ASSERT_STREQ("9", res["grouped_hits"][5]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.1, res["grouped_hits"][5]["hits"][0]["document"]["rating"].get<float>());

    ASSERT_EQ(1, res["grouped_hits"][6]["found"].get<int32_t>());
    ASSERT_EQ(10, res["grouped_hits"][6]["hits"][0]["document"]["size"].get<uint32_t>());
    ASSERT_STREQ("0", res["grouped_hits"][6]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][6]["hits"][0]["document"]["rating"].get<float>());

    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());
    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
    ASSERT_STREQ("<mark>Omeg</mark>a", res["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());

    // Wildcard group_by is not allowed
    auto error = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"foo*"}, 2).error();
    ASSERT_EQ("Pattern `foo*` is not allowed.",  error);

    // typo_tokens_threshold should respect num_groups
    res = coll_group->search("beta", {"brand"}, "", {"brand"}, {}, {2}, 50, 1, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 2,
                             {}, {}, {"brand"}, 1).get();

    ASSERT_EQ(4, res["found_docs"].get<size_t>());
    ASSERT_EQ(2, res["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"].size());
    ASSERT_EQ("Beta", res["grouped_hits"][0]["group_key"][0]);
    ASSERT_EQ("Zeta", res["grouped_hits"][1]["group_key"][0]);

    res = coll_group->search("beta", {"brand"}, "", {"brand"}, {}, {2}, 50, 1, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 1,
                             {}, {}, {"brand"}, 1).get();

    ASSERT_EQ(3, res["found_docs"].get<size_t>());
    ASSERT_EQ(1, res["found"].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"].size());
    ASSERT_EQ("Beta", res["grouped_hits"][0]["group_key"][0]);
}

TEST_F(CollectionGroupingTest, GroupingCompoundKey) {
    // group by size+brand (int32, string)
    auto res = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"size", "brand"}, 2).get();
    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(10, res["found"].get<size_t>());
    ASSERT_EQ(10, res["grouped_hits"].size());

    ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_STREQ("Beta", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());

    // optional field should have no value in the group key component
    ASSERT_EQ(1, res["grouped_hits"][5]["group_key"].size());
    ASSERT_STREQ("10", res["grouped_hits"][5]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("11", res["grouped_hits"][5]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(1, res["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());
    ASSERT_EQ(2, res["grouped_hits"][2]["hits"].size());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("3", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
    ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());

    // pagination with page=2, per_page=2
    res = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 2, 2, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"size", "brand"}, 2).get();


    // 3rd result from previous assertion will be in the first position
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("3", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.5, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("0", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());

    // total count and facet counts should be the same
    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(10, res["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"].size());
    ASSERT_EQ(10, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_STREQ("Omega", res["grouped_hits"][0]["group_key"][1].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
    ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());

    // respect min and max grouping limit (greater than 0 and less than 99)
    auto res_op = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "brand: omeg", 30, 5,
                             "", 10,
                             {}, {}, {"rating"}, 100);

    ASSERT_FALSE(res_op.ok());
    ASSERT_STREQ("Value of `group_limit` must be between 1 and 99.", res_op.error().c_str());

    res_op = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                {false}, Index::DROP_TOKENS_THRESHOLD,
                                spp::sparse_hash_set<std::string>(),
                                spp::sparse_hash_set<std::string>(), 10, "brand: omeg", 30, 5,
                                "", 10,
                                {}, {}, {"rating"}, 0);

    ASSERT_FALSE(res_op.ok());
    ASSERT_STREQ("Value of `group_limit` must be between 1 and 99.", res_op.error().c_str());
}

TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
    Collection *coll1;

    std::vector<field> fields = {field("title", field_types::STRING, false),
                                 field("artist", field_types::STRING, false),
                                 field("genre", field_types::STRING, true),
                                 field("points", field_types::INT32, false),};

    coll1 = collectionManager.get_collection("coll1").get();
    if(coll1 == nullptr) {
        coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
    }

    std::vector<std::vector<std::string>> records = {
        {"Train or Highway", "Gord Downie", "rock"},
        {"Down There by the Train", "Dustin Kensrue", "pop"},
        {"In the Train", "Dustin Kensrue", "pop"},
        {"State Trooper", "Dustin Kensrue", "country"},
        {"Down There Somewhere", "Dustin Kensrue", "pop"},
        {"Down There by the Train", "Gord Downie", "rock"},
        {"Down and Outside", "Gord Downie", "rock"},
        {"Let it be", "Downie Kensrue", "country"},
        {"There was a Train", "Gord Kensrue", "country"},
    };

    for(size_t i=0; i<records.size(); i++) {
        nlohmann::json doc;

        doc["id"] = std::to_string(i);
        doc["title"] = records[i][0];
        doc["artist"] = records[i][1];
        doc["genre"] = records[i][2];
        doc["points"] = i;

        ASSERT_TRUE(coll1->add(doc.dump()).ok());
    }

    auto results = coll1->search("Dustin Kensrue Down There by the Train",
                                 {"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                                 {false}, 10,
                                 spp::sparse_hash_set<std::string>(),
                                 spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                 "", 10,
                                 {}, {}, {"genre"}, 2).get();

    ASSERT_EQ(7, results["found_docs"].get<size_t>());
    ASSERT_EQ(3, results["found"].get<size_t>());
    ASSERT_EQ(3, results["grouped_hits"].size());

    ASSERT_EQ(3, results["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_STREQ("pop", results["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
    ASSERT_EQ(2, results["grouped_hits"][0]["hits"].size());
    ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(2, results["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
    ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());
    ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("0", results["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(2, results["grouped_hits"][2]["found"].get<int32_t>());
    ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
    ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());
    ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());

    collectionManager.drop_collection("coll1");
}

TEST_F(CollectionGroupingTest, GroupingWithGropLimitOfOne) {
    auto res = coll_group->search("*", {}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"brand"}, 1).get();

    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(5, res["found"].get<size_t>());
    ASSERT_EQ(5, res["grouped_hits"].size());

    // all hits array must be of size 1
    for(auto i=0; i<5; i++) {
        ASSERT_EQ(1, res["grouped_hits"][i]["hits"].size());
    }
    
    ASSERT_EQ(3, res["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    
    ASSERT_EQ(4, res["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    
    ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());
    ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    
    ASSERT_EQ(2, res["grouped_hits"][3]["found"].get<int32_t>());
    ASSERT_STREQ("10", res["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str()); // unbranded
    
    ASSERT_EQ(1, res["grouped_hits"][4]["found"].get<int32_t>());
    ASSERT_STREQ("9", res["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>().c_str());

    // facet counts should each be 1, including unbranded
    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    for(size_t i=0; i < 4; i++) {
        ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][i]["count"]);
    }
}

TEST_F(CollectionGroupingTest, GroupingWithArrayFieldAndOverride) {
    nlohmann::json override_json_include = {
        {"id", "include-rule"},
        {
            "rule", {
               {"query", "shirt"},
               {"match", override_t::MATCH_EXACT}
            }
        },
        {"stop_processing", false}
    };

    override_json_include["includes"] = nlohmann::json::array();
    override_json_include["includes"][0] = nlohmann::json::object();
    override_json_include["includes"][0]["id"] = "11";
    override_json_include["includes"][0]["position"] = 1;

    override_json_include["includes"][1] = nlohmann::json::object();
    override_json_include["includes"][1]["id"] = "10";
    override_json_include["includes"][1]["position"] = 1;

    nlohmann::json override_json_exclude = {
        {"id",   "exclude-rule"},
        {
            "rule", {
                 {"query", "shirt"},
                 {"match", override_t::MATCH_EXACT}
            }
        },
        {"stop_processing", false}
    };
    override_json_exclude["excludes"] = nlohmann::json::array();
    override_json_exclude["excludes"][0] = nlohmann::json::object();
    override_json_exclude["excludes"][0]["id"] = "2";

    override_t override1;
    override_t override2;

    override_t::parse(override_json_include, "", override1);
    override_t::parse(override_json_exclude, "", override2);

    Option<uint32_t> ov1_op = coll_group->add_override(override1);
    Option<uint32_t> ov2_op = coll_group->add_override(override2);

    ASSERT_TRUE(ov1_op.ok());
    ASSERT_TRUE(ov2_op.ok());

    auto res = coll_group->search("shirt", {"title"}, "", {"brand"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"colors"}, 2).get();

    ASSERT_EQ(9, res["found_docs"].get<size_t>());
    ASSERT_EQ(4, res["found"].get<size_t>());
    ASSERT_EQ(4, res["grouped_hits"].size());

    ASSERT_EQ(1, res["grouped_hits"][0]["group_key"][0].size());
    ASSERT_STREQ("white", res["grouped_hits"][0]["group_key"][0][0].get<std::string>().c_str());

    ASSERT_STREQ("11", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("10", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_STREQ("5", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_STREQ("4", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_STREQ("0", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(1, res["grouped_hits"][3]["hits"].size());
    ASSERT_STREQ("8", res["grouped_hits"][3]["hits"][0]["document"]["id"].get<std::string>().c_str());

    // assert facet counts
    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
    
    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
    ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());
}

TEST_F(CollectionGroupingTest, GroupOrderIndependence) {
    Collection *coll1;

    std::vector<field> fields = {field("group", field_types::STRING, true),
                                 field("points", field_types::INT32, false),};

    coll1 = collectionManager.get_collection("coll1").get();
    if(coll1 == nullptr) {
        coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
    }

    nlohmann::json doc;

    for(size_t i = 0; i < 256; i++) {
        int64_t points = 100 + i;
        doc["id"] = std::to_string(i);
        doc["group"] = std::to_string(i);
        doc["points"] = points;

        ASSERT_TRUE(coll1->add(doc.dump()).ok());
    }

    // doc id "255" will have points of 255
    // try to insert doc id "256" with group "256" but having lesser points than all records

    doc["id"] = "256";
    doc["group"] = "256";
    doc["points"] = 50;
    ASSERT_TRUE(coll1->add(doc.dump()).ok());

    // insert doc id "257" of same group "256" with greatest point

    doc["id"] = "257";
    doc["group"] = "256";
    doc["points"] = 500;
    ASSERT_TRUE(coll1->add(doc.dump()).ok());

    // when we search by grouping records, sorting descending on points, both records of group "256" should show up

    std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};

    auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"group"}, 10).get();

    ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());
    ASSERT_STREQ("256", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
}

TEST_F(CollectionGroupingTest, UseHighestValueInGroupForOrdering) {
    Collection *coll1;

    std::vector<field> fields = {field("group", field_types::STRING, true),
                                 field("points", field_types::INT32, false),};

    coll1 = collectionManager.get_collection("coll1").get();
    if(coll1 == nullptr) {
        coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
    }

    nlohmann::json doc;

    for(size_t i = 0; i < 250; i++) {
        int64_t points = 100 + i;
        doc["id"] = std::to_string(i);
        doc["group"] = std::to_string(i);
        doc["points"] = points;

        ASSERT_TRUE(coll1->add(doc.dump()).ok());
    }

    // points: 100 -> 349

    // group with highest point is "249" with 349 points
    // insert another document for that group with 50 points
    doc["id"] = "250";
    doc["group"] = "249";
    doc["points"] = 50;
    ASSERT_TRUE(coll1->add(doc.dump()).ok());

    // now insert another new group whose points is greater than 50
    doc["id"] = "251";
    doc["group"] = "1000";
    doc["points"] = 60;
    ASSERT_TRUE(coll1->add(doc.dump()).ok());

    std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};

    auto res = coll1->search("*", {}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"group"}, 10).get();

    ASSERT_EQ(1, res["grouped_hits"][0]["group_key"].size());
    ASSERT_STREQ("249", res["grouped_hits"][0]["group_key"][0].get<std::string>().c_str());
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
}


TEST_F(CollectionGroupingTest, RepeatedFieldNameGroupHitCount) {
    std::vector<field> fields = {
            field("title", field_types::STRING, false),
            field("brand", field_types::STRING, true, true),
            field("colors", field_types::STRING, true, false),
    };

    Collection* coll2 = collectionManager.get_collection("coll2").get();
    if(coll2 == nullptr) {
        coll2 = collectionManager.create_collection("coll2", 1, fields).get();
    }

    nlohmann::json doc;
    doc["id"] = "0";
    doc["title"] = "foobar";
    doc["brand"] = "Omega";
    doc["colors"] = "foo";

    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    auto res = coll2->search("f", {"title", "colors"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                                   {true}, 10,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"brand"}, 2).get();

    ASSERT_EQ(1, res["grouped_hits"].size());
    ASSERT_EQ(1, res["grouped_hits"][0]["found"].get<int32_t>());
}

TEST_F(CollectionGroupingTest, ControlMissingValues) {
    std::vector<field> fields = {
        field("brand", field_types::STRING, true, true),
    };

    Collection* coll2 = collectionManager.get_collection("coll2").get();
    if(coll2 == nullptr) {
        coll2 = collectionManager.create_collection("coll2", 1, fields).get();
    }

    LOG(INFO) << "----------------------";

    nlohmann::json doc;
    doc["id"] = "0";
    doc["brand"] = "Omega";
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "1";
    doc["brand"] = nullptr;
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "2";
    doc["brand"] = nullptr;
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "3";
    doc["brand"] = "Omega";
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    // disable null value aggregation

    auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                             {true}, 10,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"brand"}, 2,
                             "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
                             4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
                             100, 0, 0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left", true, false).get();

    ASSERT_EQ(3, res["grouped_hits"].size());
    ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());

    ASSERT_EQ(0, res["grouped_hits"][2]["group_key"].size());
    ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
    ASSERT_EQ("1", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>());

    // with null value aggregation (default)
    res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                        {true}, 10,
                        spp::sparse_hash_set<std::string>(),
                        spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                        "", 10,
                        {}, {}, {"brand"}, 2,
                        "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
                        4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
                        100, 0, 0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left", true, true).get();

    ASSERT_EQ(2, res["grouped_hits"].size());

    ASSERT_EQ("Omega", res["grouped_hits"][0]["group_key"][0].get<std::string>());
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
    ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
}

TEST_F(CollectionGroupingTest, SortingOnGroupCount) {

    std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};
    
    auto res = coll_group->search("*", {}, "", {"brand"}, sort_fields, {0}, 50, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(3, res["found"].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"].size());

    ASSERT_EQ(10, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(7, res["grouped_hits"][0]["found"].get<int32_t>());

    ASSERT_EQ(12, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"][1]["found"].get<int32_t>());

    ASSERT_EQ(11, res["grouped_hits"][2]["group_key"][0].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<int32_t>());


    //search in asc order

    std::vector<sort_by> sort_fields2 = {sort_by("_group_found", "ASC")};
    
    auto res2 = coll_group->search("*", {}, "", {"brand"}, sort_fields2, {0}, 50, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(12, res2["found_docs"].get<size_t>());
    ASSERT_EQ(3, res2["found"].get<size_t>());
    ASSERT_EQ(3, res2["grouped_hits"].size());

    ASSERT_EQ(11, res2["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(2, res2["grouped_hits"][0]["found"].get<int32_t>());

    ASSERT_EQ(12, res2["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(3, res2["grouped_hits"][1]["found"].get<int32_t>());

    ASSERT_EQ(10, res2["grouped_hits"][2]["group_key"][0].get<size_t>());
    ASSERT_EQ(7, res2["grouped_hits"][2]["found"].get<int32_t>());
}

TEST_F(CollectionGroupingTest, SortingMoreThanMaxTopsterSize) {

    std::vector<field> fields = {
            field("title", field_types::STRING, false),
            field("brand", field_types::STRING, true, true),
            field("size", field_types::INT32, true, false),
            field("colors", field_types::STRING, true, false),
            field("rating", field_types::FLOAT, true, false)
    };

    Collection* coll3 = collectionManager.get_collection("coll3").get();
    if(coll3 == nullptr) {
        coll3 = collectionManager.create_collection("coll3", 4, fields, "rating").get();
    }

    for(auto i = 0; i < 150; i++) {
        auto group_id = i;
        for(auto j = 0; j < 4; j++) {
            nlohmann::json doc;
            doc["title"] = "Omega Casual Poplin Shirt";
            doc["brand"] = "Omega";
            doc["size"] = group_id;
            doc["colors"] = "blue";
            doc["rating"] = 4.5;

            ASSERT_TRUE(coll3->add(doc.dump()).ok());
        } 
    }

    for(auto i = 150; i < 250; i++) {
        auto group_id = i;
        for(auto j = 0; j < 3; j++) {
            nlohmann::json doc;
            doc["title"] = "Beta Casual Poplin Shirt";
            doc["brand"] = "Beta";
            doc["size"] = group_id;
            doc["colors"] = "white";
            doc["rating"] = 4.3;

            ASSERT_TRUE(coll3->add(doc.dump()).ok());
        } 
    }

    for(auto i = 250; i < 300; i++) {
        auto group_id = i;
        for(auto j = 0; j < 2; j++) {
            nlohmann::json doc;
            doc["title"] = "Zeta Casual Poplin Shirt";
            doc["brand"] = "Zeta";
            doc["size"] = group_id;
            doc["colors"] = "red";
            doc["rating"] = 4.6;

            ASSERT_TRUE(coll3->add(doc.dump()).ok());
        } 
    }

    //first search in desc order
    std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};
    
    auto res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 2, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(1000, res["found_docs"].get<size_t>());
    ASSERT_EQ(300, res["found"].get<size_t>());
    ASSERT_EQ(100, res["grouped_hits"].size());

    ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());

    ASSERT_EQ(4, res["grouped_hits"][4]["found"].get<int32_t>());

    ASSERT_EQ(3, res["grouped_hits"][50]["found"].get<int32_t>());

    ASSERT_EQ(3, res["grouped_hits"][99]["found"].get<int32_t>());


    res = coll3->search("*", {}, "", {"brand"}, sort_fields, {0}, 100, 3, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(1000, res["found_docs"].get<size_t>());
    ASSERT_EQ(300, res["found"].get<size_t>());
    ASSERT_EQ(100, res["grouped_hits"].size());

    ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());

    ASSERT_EQ(3, res["grouped_hits"][4]["found"].get<int32_t>());

    ASSERT_EQ(2, res["grouped_hits"][50]["found"].get<int32_t>());

    ASSERT_EQ(2, res["grouped_hits"][99]["found"].get<int32_t>());


    //search in asc order

    std::vector<sort_by> sort_fields2 = {sort_by("_group_found", "ASC")};
    
    auto res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(1000, res2["found_docs"].get<size_t>());
    ASSERT_EQ(300, res2["found"].get<size_t>());
    ASSERT_EQ(100, res2["grouped_hits"].size());

    ASSERT_EQ(2, res2["grouped_hits"][0]["found"].get<int32_t>());

    ASSERT_EQ(2, res2["grouped_hits"][1]["found"].get<int32_t>());

    ASSERT_EQ(3, res2["grouped_hits"][50]["found"].get<int32_t>());

    ASSERT_EQ(3, res2["grouped_hits"][99]["found"].get<int32_t>());

    res2 = coll3->search("*", {}, "", {"brand"}, sort_fields2, {0}, 100, 2, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {"size"}, 2).get();

    ASSERT_EQ(1000, res2["found_docs"].get<size_t>());
    ASSERT_EQ(300, res2["found"].get<size_t>());
    ASSERT_EQ(100, res2["grouped_hits"].size());

    ASSERT_EQ(3, res2["grouped_hits"][0]["found"].get<int32_t>());

    ASSERT_EQ(3, res2["grouped_hits"][1]["found"].get<int32_t>());

    ASSERT_EQ(4, res2["grouped_hits"][50]["found"].get<int32_t>());

    ASSERT_EQ(4, res2["grouped_hits"][99]["found"].get<int32_t>());
}

TEST_F(CollectionGroupingTest, GroupSortingWithoutGroupingFields) {
    
    std::vector<sort_by> sort_fields = {sort_by("_group_found", "DESC")};
    
    auto res = coll_group->search("*", {}, "", {"brand"}, sort_fields, {0}, 50, 1, FREQUENCY,
                                   {false}, Index::DROP_TOKENS_THRESHOLD,
                                   spp::sparse_hash_set<std::string>(),
                                   spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                   "", 10,
                                   {}, {}, {});

    ASSERT_EQ(res.ok(), false);
    ASSERT_EQ(res.error(), "group_by parameters should not be empty when using sort_by group_found");
}

TEST_F(CollectionGroupingTest, SkipToReverseGroupBy) {
    std::vector<field> fields = {
            field("brand", field_types::STRING, true, true),
    };

    Collection* coll2 = collectionManager.get_collection("coll2").get();
    if(coll2 == nullptr) {
        coll2 = collectionManager.create_collection("coll2", 1, fields).get();
    }

    nlohmann::json doc;

    doc["id"] = "0";
    doc["brand"] = nullptr;
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    auto res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                             {true}, 10,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"brand"}, 2,
                             "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
                             4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
                             100, 0, 0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left", true, false).get();

    ASSERT_EQ(1, res["grouped_hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][0]["group_key"].size());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("0", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>());

    doc["id"] = "1";
    doc["brand"] = "adidas";
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "2";
    doc["brand"] = "puma";
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "3";
    doc["brand"] = nullptr;
    ASSERT_TRUE(coll2->add(doc.dump()).ok());

    doc["id"] = "4";
    doc["brand"] = "nike";
    ASSERT_TRUE(coll2->add(doc.dump()).ok());


    res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                             {true}, 10,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"brand"}, 2,
                             "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
                             4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
                             100, 0, 0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left", true, false).get();

    ASSERT_EQ(5, res["grouped_hits"].size());

    ASSERT_EQ("nike", res["grouped_hits"][0]["group_key"][0].get<std::string>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());

    ASSERT_EQ("puma", res["grouped_hits"][2]["group_key"][0].get<std::string>());
    ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());

    ASSERT_EQ("adidas", res["grouped_hits"][3]["group_key"][0].get<std::string>());
    ASSERT_EQ(1, res["grouped_hits"][3]["hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][4]["group_key"].size());
    ASSERT_EQ(1, res["grouped_hits"][4]["hits"].size());
    ASSERT_EQ("0", res["grouped_hits"][4]["hits"][0]["document"]["id"].get<std::string>());

    res = coll2->search("*", {}, "", {}, {}, {0}, 10, 1, FREQUENCY,
                             {true}, 10,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                             "", 10,
                             {}, {}, {"brand"}, 2,
                             "<mark>", "</mark>", {3,3}, 1000, true, false, true, "", false, 6000 * 1000, 4, 7, fallback,
                             4, {off}, 0, 0, 0, 2, false, "", true, 0, max_score,
                             100, 0, 0, "exhaustive", 30000, 2, "", {}, {}, "right_to_left", true, true).get();

    ASSERT_EQ(4, res["grouped_hits"].size());

    ASSERT_EQ("nike", res["grouped_hits"][0]["group_key"][0].get<std::string>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());

    ASSERT_EQ(0, res["grouped_hits"][1]["group_key"].size());
    ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("3", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>());

    ASSERT_EQ("puma", res["grouped_hits"][2]["group_key"][0].get<std::string>());
    ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
}

TEST_F(CollectionGroupingTest, GroupByMultipleFacetFields) {
    auto res = coll_group->search("*", {}, "", {"brand", "colors"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"size"}, 2).get();

    ASSERT_EQ(12, res["found_docs"].get<size_t>());
    ASSERT_EQ(3, res["found"].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"].size());
    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());

    ASSERT_EQ(2, res["grouped_hits"][0]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ(11, res["grouped_hits"][0]["hits"][0]["document"]["size"].get<size_t>());
    ASSERT_STREQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("1", res["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(7, res["grouped_hits"][1]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("3", res["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_EQ(3, res["grouped_hits"][2]["found"].get<int32_t>());
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_STREQ("2", res["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());
    ASSERT_STREQ("8", res["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][3]["count"]);
    ASSERT_STREQ("Zeta", res["facet_counts"][0]["counts"][3]["value"].get<std::string>().c_str());


    ASSERT_STREQ("colors", res["facet_counts"][1]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][0]["count"]);
    ASSERT_STREQ("blue", res["facet_counts"][1]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][1]["count"]);
    ASSERT_STREQ("white", res["facet_counts"][1]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][1]["counts"][2]["count"]);
    ASSERT_STREQ("red", res["facet_counts"][1]["counts"][2]["value"].get<std::string>().c_str());
}

TEST_F(CollectionGroupingTest, GroupByMultipleFacetFieldsWithFilter) {
    auto res = coll_group->search("*", {}, "size:>10", {"colors", "brand"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {}, {}, {"size"}, 2).get();

    ASSERT_EQ(5, res["found_docs"].get<size_t>());
    ASSERT_EQ(2, res["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"].size());

    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][0]["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ("1", res["grouped_hits"][0]["hits"][1]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][0]["hits"][1]["document"]["rating"].get<float>());

    ASSERT_EQ(12, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"][1]["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("2", res["grouped_hits"][1]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ("8", res["grouped_hits"][1]["hits"][1]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][1]["hits"][1]["document"]["rating"].get<float>());

    ASSERT_STREQ("colors", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("blue", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("white", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("red", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][1]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][1]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][1]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(2, (int) res["facet_counts"][1]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][1]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][1]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][1]["counts"][2]["value"].get<std::string>().c_str());
}

TEST_F(CollectionGroupingTest, GroupByMultipleFacetFieldsWithPinning) {
    auto res = coll_group->search("*", {}, "size:>10", {"colors", "brand"}, {}, {0}, 50, 1, FREQUENCY,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 5,
                                  "", 10,
                                  {"3:1,4:2"}, {}, {"size"}, 2).get();

    ASSERT_EQ(5, res["found_docs"].get<size_t>());
    ASSERT_EQ(4, res["found"].get<size_t>());
    ASSERT_EQ(4, res["grouped_hits"].size());

    ASSERT_EQ(10, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("3", res["grouped_hits"][0]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][0]["hits"][0]["document"]["rating"].get<float>());

    ASSERT_EQ(10, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("4", res["grouped_hits"][1]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][1]["hits"][0]["document"]["rating"].get<float>());

    ASSERT_EQ(11, res["grouped_hits"][2]["group_key"][0].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][2]["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][2]["hits"].size());
    ASSERT_EQ("5", res["grouped_hits"][2]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.8, res["grouped_hits"][2]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ("1", res["grouped_hits"][2]["hits"][1]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.3, res["grouped_hits"][2]["hits"][1]["document"]["rating"].get<float>());

    ASSERT_EQ(12, res["grouped_hits"][3]["group_key"][0].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"][3]["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"][3]["hits"].size());
    ASSERT_EQ("2", res["grouped_hits"][3]["hits"][0]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.6, res["grouped_hits"][3]["hits"][0]["document"]["rating"].get<float>());
    ASSERT_EQ("8", res["grouped_hits"][3]["hits"][1]["document"]["id"]);
    ASSERT_FLOAT_EQ(4.4, res["grouped_hits"][3]["hits"][1]["document"]["rating"].get<float>());

    ASSERT_STREQ("colors", res["facet_counts"][0]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][0]["count"]);
    ASSERT_STREQ("blue", res["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][0]["counts"][1]["count"]);
    ASSERT_STREQ("white", res["facet_counts"][0]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][0]["counts"][2]["count"]);
    ASSERT_STREQ("red", res["facet_counts"][0]["counts"][2]["value"].get<std::string>().c_str());

    ASSERT_STREQ("brand", res["facet_counts"][1]["field_name"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][0]["count"]);
    ASSERT_STREQ("Beta", res["facet_counts"][1]["counts"][0]["value"].get<std::string>().c_str());

    ASSERT_EQ(3, (int) res["facet_counts"][1]["counts"][1]["count"]);
    ASSERT_STREQ("Omega", res["facet_counts"][1]["counts"][1]["value"].get<std::string>().c_str());

    ASSERT_EQ(1, (int) res["facet_counts"][1]["counts"][2]["count"]);
    ASSERT_STREQ("Xorp", res["facet_counts"][1]["counts"][2]["value"].get<std::string>().c_str());
}

TEST_F(CollectionGroupingTest, GroupByPinnedHitsOrder) {
    auto res = coll_group->search("*", {"title"}, "size:=[12,11]", {}, {}, {0}, 50, 0, NOT_SET,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 4,
                                  "", 1,
                                  {"6:1,1:2"}, {}, {"size"}, 1).get();

    ASSERT_EQ(2, res["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"].size());

    ASSERT_EQ(12, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("6", res["grouped_hits"][0]["hits"][0]["document"]["id"]);

    ASSERT_EQ(11, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("1", res["grouped_hits"][1]["hits"][0]["document"]["id"]);

    //try with pinned hits in other order
    res = coll_group->search("*", {"title"}, "size:=[12,11]", {}, {}, {0}, 50, 0, NOT_SET,
                                  {false}, Index::DROP_TOKENS_THRESHOLD,
                                  spp::sparse_hash_set<std::string>(),
                                  spp::sparse_hash_set<std::string>(), 10, "", 30, 4,
                                  "", 1,
                                  {"5:1,8:2"}, {}, {"size"}, 1).get();

    ASSERT_EQ(2, res["found"].get<size_t>());
    ASSERT_EQ(2, res["grouped_hits"].size());

    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"]);

    ASSERT_EQ(12, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("8", res["grouped_hits"][1]["hits"][0]["document"]["id"]);

    //random order
    res = coll_group->search("*", {"title"}, "size:=[12,11,10]", {}, {}, {0}, 50, 0, NOT_SET,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 4,
                             "", 1,
                             {"5:1,8:2,0:3"}, {}, {"size"}, 1).get();

    ASSERT_EQ(3, res["found"].get<size_t>());
    ASSERT_EQ(3, res["grouped_hits"].size());

    ASSERT_EQ(11, res["grouped_hits"][0]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][0]["hits"].size());
    ASSERT_EQ("5", res["grouped_hits"][0]["hits"][0]["document"]["id"]);

    ASSERT_EQ(12, res["grouped_hits"][1]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][1]["hits"].size());
    ASSERT_EQ("8", res["grouped_hits"][1]["hits"][0]["document"]["id"]);

    ASSERT_EQ(10, res["grouped_hits"][2]["group_key"][0].get<size_t>());
    ASSERT_EQ(1, res["grouped_hits"][2]["hits"].size());
    ASSERT_EQ("0", res["grouped_hits"][2]["hits"][0]["document"]["id"]);
}

TEST_F(CollectionGroupingTest, GroupByPerPage) {
    std::vector<field> fields = {
            field("name", field_types::STRING, false, false),
            field("id", field_types::STRING, true, true),
    };

    Collection* fabric = collectionManager.get_collection("fabric").get();
    if(fabric == nullptr) {
        fabric = collectionManager.create_collection("fabric", 1, fields).get();
    }


    nlohmann::json doc;

    doc["id"] = "1001";
    doc["name"] = "Cotton";
    ASSERT_TRUE(fabric->add(doc.dump()).ok());

    doc["id"] = "1002";
    doc["name"] = "Nylon";
    ASSERT_TRUE(fabric->add(doc.dump()).ok());

    doc["id"] = "1003";
    doc["name"] = "Polyester";
    ASSERT_TRUE(fabric->add(doc.dump()).ok());

    doc["id"] = "1004";
    doc["name"] = "Linen";
    ASSERT_TRUE(fabric->add(doc.dump()).ok());

    doc["id"] = "1005";
    doc["name"] = "Silk";
    ASSERT_TRUE(fabric->add(doc.dump()).ok());

    fields = {
            field("name", field_types::STRING, false, false),
            field("fabric_id", field_types::STRING, true, false, true,
            "", -1, -1, false, 0, 0, cosine, "fabric.id"),
            field("size", field_types::STRING, false, false),
    };

    Collection* garments = collectionManager.get_collection("garments").get();
    if(garments == nullptr) {
        garments = collectionManager.create_collection("garments", 1, fields).get();
    }

    nlohmann::json doc2;

    doc2["name"] = "Tshirt";
    doc2["fabric_id"] = "1001";
    doc2["size"] = "Medium";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Tshirt";
    doc2["fabric_id"] = "1003";
    doc2["size"] = "Large";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Shirt";
    doc2["fabric_id"] = "1004";
    doc2["size"] = "Xtra Large";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Trouser";
    doc2["fabric_id"] = "1002";
    doc2["size"] = "Small";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Veshti";
    doc2["fabric_id"] = "1005";
    doc2["size"] = "Free";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Shorts";
    doc2["fabric_id"] = "1002";
    doc2["size"] = "Medium";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    doc2["name"] = "Shirt";
    doc2["fabric_id"] = "1005";
    doc2["size"] = "Large";
    ASSERT_TRUE(garments->add(doc2.dump()).ok());

    //limit per page to 4
    auto res = garments->search("*", {"name"}, "", {}, {}, {0}, 4, 1, NOT_SET,
                             {false}, Index::DROP_TOKENS_THRESHOLD,
                             spp::sparse_hash_set<std::string>(),
                             spp::sparse_hash_set<std::string>(), 10, "", 30, 4,
                             "", 1,
                             {}, {}, {"fabric_id"}, 1).get();

    ASSERT_EQ(4, res["found"].get<size_t>());
    ASSERT_EQ(4, res["grouped_hits"].size());
    ASSERT_EQ(7, res["found_docs"].get<size_t>());

    ASSERT_EQ("1005", res["grouped_hits"][0]["group_key"][0]);
    ASSERT_EQ("1002", res["grouped_hits"][1]["group_key"][0]);
    ASSERT_EQ("1004", res["grouped_hits"][2]["group_key"][0]);
    ASSERT_EQ("1003", res["grouped_hits"][3]["group_key"][0]);

    //per page 10
    res = garments->search("*", {"name"}, "", {}, {}, {0}, 10, 1, NOT_SET,
                                {false}, Index::DROP_TOKENS_THRESHOLD,
                                spp::sparse_hash_set<std::string>(),
                                spp::sparse_hash_set<std::string>(), 10, "", 30, 4,
                                "", 1,
                                {}, {}, {"fabric_id"}, 1).get();

    ASSERT_EQ(5, res["found"].get<size_t>());
    ASSERT_EQ(5, res["grouped_hits"].size());
    ASSERT_EQ(7, res["found_docs"].get<size_t>());

    ASSERT_EQ("1005", res["grouped_hits"][0]["group_key"][0]);
    ASSERT_EQ("1002", res["grouped_hits"][1]["group_key"][0]);
    ASSERT_EQ("1004", res["grouped_hits"][2]["group_key"][0]);
    ASSERT_EQ("1003", res["grouped_hits"][3]["group_key"][0]);
    ASSERT_EQ("1001", res["grouped_hits"][4]["group_key"][0]);
}